v0.1.0 — Réécriture complète de OhmStreaming
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
This commit is contained in:
Roman
@@ -0,0 +1,499 @@
|
||||
"""Gestionnaire de téléchargements : file asyncio, parallélisme limité,
|
||||
pause/reprise (Range HTTP), anti-doublons, persistance, progression temps réel.
|
||||
|
||||
Les statuts : pending → downloading → done | failed | cancelled
|
||||
↕ paused
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import re
|
||||
import signal
|
||||
import time
|
||||
import unicodedata
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ACTIVE_STATUSES = ("pending", "downloading", "paused")
|
||||
|
||||
_FFMPEG_TIME_RE = re.compile(r"time=(\d+:\d+:\d+(?:\.\d+)?)")
|
||||
_EXTINF_RE = re.compile(r"#EXTINF:([\d.]+)")
|
||||
_BANDWIDTH_RE = re.compile(r"#EXT-X-STREAM-INF:[^\n]*BANDWIDTH=(\d+)[^\n]*\n(\S+)")
|
||||
|
||||
|
||||
def _parse_ffmpeg_time(value: str) -> float:
|
||||
parts = value.split(":")
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
||||
|
||||
|
||||
def _best_variant(master_body: str, base_url: str) -> str | None:
|
||||
"""URL de la variante au plus haut débit d'une playlist maître HLS."""
|
||||
variants = [
|
||||
(int(bw), urljoin(base_url, uri)) for bw, uri in _BANDWIDTH_RE.findall(master_body)
|
||||
]
|
||||
return max(variants)[1] if variants else None
|
||||
|
||||
_STATUS_LABELS = {
|
||||
"pending": "en attente",
|
||||
"downloading": "en cours",
|
||||
"paused": "en pause",
|
||||
"done": "terminé",
|
||||
"failed": "échec",
|
||||
"cancelled": "annulé",
|
||||
}
|
||||
|
||||
|
||||
def sanitize_filename(name: str) -> str:
|
||||
"""Nettoie un nom de fichier : caractères interdits retirés, anti-traversée."""
|
||||
name = unicodedata.normalize("NFKC", name)
|
||||
name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', " ", name)
|
||||
name = re.sub(r"\s+", " ", name).strip(" .")
|
||||
if not name or name in (".", ".."):
|
||||
name = "video"
|
||||
return name[:150]
|
||||
|
||||
|
||||
class DownloadManager:
|
||||
"""File d'attente de téléchargements, injectée dans les routes via app.state."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._queue: asyncio.Queue[int] # créée dans start() (affinité avec la boucle)
|
||||
self._tasks: dict[int, asyncio.Task] = {} # download_id → tâche asyncio
|
||||
self._pause_events: dict[int, asyncio.Event] = {} # set = peut tourner
|
||||
self._progress: dict[int, dict[str, Any]] = {} # progression temps réel en mémoire
|
||||
self._listeners: list[asyncio.Queue] = []
|
||||
self._workers: list[asyncio.Task] = []
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
self._hls_processes: dict[int, asyncio.subprocess.Process] = {}
|
||||
|
||||
# ------------------------------------------------------------ cycle de vie
|
||||
|
||||
async def start(self) -> None:
|
||||
self._queue = asyncio.Queue()
|
||||
settings = get_settings()
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(30.0, read=300.0),
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"User-Agent": settings.user_agent,
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
||||
},
|
||||
)
|
||||
# Restaure les téléchargements interrompus (crash/arrêt) en 'pending'
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = 'pending', updated_at = datetime('now') "
|
||||
"WHERE status = 'downloading'"
|
||||
)
|
||||
await self._scan_download_dir()
|
||||
for _ in range(settings.max_parallel_downloads):
|
||||
self._workers.append(asyncio.create_task(self._worker()))
|
||||
logger.info("DownloadManager démarré (%d workers)", settings.max_parallel_downloads)
|
||||
|
||||
async def stop(self) -> None:
|
||||
for worker in self._workers:
|
||||
worker.cancel()
|
||||
for task in self._tasks.values():
|
||||
task.cancel()
|
||||
for proc in self._hls_processes.values():
|
||||
if proc.returncode is None:
|
||||
proc.kill()
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._workers.clear()
|
||||
self._tasks.clear()
|
||||
|
||||
async def _scan_download_dir(self) -> None:
|
||||
"""Restaure en 'done' les fichiers présents sur disque mais inconnus de la DB."""
|
||||
download_dir = get_settings().download_dir
|
||||
rows = await db.fetchall("SELECT file_path FROM downloads WHERE file_path IS NOT NULL")
|
||||
known = {row["file_path"] for row in rows}
|
||||
for path in download_dir.iterdir():
|
||||
if path.is_file() and path.suffix != ".part" and path.name not in known:
|
||||
size = path.stat().st_size
|
||||
await db.execute(
|
||||
"INSERT INTO downloads "
|
||||
"(source_key, video_url, page_url, title, file_path, status, "
|
||||
" total_bytes, downloaded_bytes) "
|
||||
"VALUES (?, ?, ?, ?, ?, 'done', ?, ?)",
|
||||
(
|
||||
f"file:{path.name}",
|
||||
"",
|
||||
"",
|
||||
path.stem,
|
||||
path.name,
|
||||
size,
|
||||
size,
|
||||
),
|
||||
)
|
||||
logger.info("Fichier restauré depuis le disque : %s", path.name)
|
||||
|
||||
# ------------------------------------------------------------ API publique
|
||||
|
||||
async def enqueue(self, video_url: str, page_url: str, title: str) -> dict:
|
||||
"""Ajoute un téléchargement ; retourne la tâche existante si doublon actif."""
|
||||
source_key = video_url
|
||||
existing = await db.fetchone(
|
||||
f"SELECT * FROM downloads WHERE source_key = ? AND status IN "
|
||||
f"({','.join('?' * len(ACTIVE_STATUSES))})",
|
||||
(source_key, *ACTIVE_STATUSES),
|
||||
)
|
||||
if existing:
|
||||
logger.info("Anti-doublon : %s déjà en file (id=%s)", title, existing["id"])
|
||||
return self._to_dict(existing, duplicate=True)
|
||||
|
||||
filename = sanitize_filename(title) + self._guess_extension(video_url)
|
||||
cursor = await db.execute(
|
||||
"INSERT INTO downloads (source_key, video_url, page_url, title, file_path) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
(source_key, video_url, page_url, title, filename),
|
||||
)
|
||||
download_id = cursor.lastrowid
|
||||
await self._queue.put(download_id)
|
||||
await self._emit(download_id)
|
||||
logger.info("Téléchargement ajouté : %s (id=%s)", title, download_id)
|
||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
||||
return self._to_dict(row)
|
||||
|
||||
async def pause(self, download_id: int) -> bool:
|
||||
event = self._pause_events.get(download_id)
|
||||
if event:
|
||||
event.clear()
|
||||
await self._set_status(download_id, "paused")
|
||||
return True
|
||||
|
||||
async def resume(self, download_id: int) -> bool:
|
||||
row = await self._get_row(download_id)
|
||||
if row["status"] != "paused":
|
||||
return False
|
||||
await self._set_status(download_id, "pending")
|
||||
await self._queue.put(download_id)
|
||||
return True
|
||||
|
||||
async def retry(self, download_id: int) -> bool:
|
||||
row = await self._get_row(download_id)
|
||||
if row["status"] not in ("failed", "cancelled"):
|
||||
return False
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = 'pending', error = NULL, downloaded_bytes = 0, "
|
||||
"updated_at = datetime('now') WHERE id = ?",
|
||||
(download_id,),
|
||||
)
|
||||
part = self._part_path(row["file_path"])
|
||||
part.unlink(missing_ok=True)
|
||||
await self._queue.put(download_id)
|
||||
await self._emit(download_id)
|
||||
return True
|
||||
|
||||
async def cancel(self, download_id: int) -> bool:
|
||||
task = self._tasks.get(download_id)
|
||||
if task:
|
||||
task.cancel()
|
||||
proc = self._hls_processes.get(download_id)
|
||||
if proc and proc.returncode is None:
|
||||
proc.kill()
|
||||
await self._set_status(download_id, "cancelled")
|
||||
row = await self._get_row(download_id)
|
||||
self._part_path(row["file_path"]).unlink(missing_ok=True)
|
||||
await self._emit(download_id)
|
||||
return True
|
||||
|
||||
async def cancel_all(self) -> int:
|
||||
rows = await db.fetchall(
|
||||
f"SELECT id FROM downloads WHERE status IN ({','.join('?' * len(ACTIVE_STATUSES))})",
|
||||
ACTIVE_STATUSES,
|
||||
)
|
||||
for row in rows:
|
||||
await self.cancel(row["id"])
|
||||
return len(rows)
|
||||
|
||||
async def clear_finished(self) -> int:
|
||||
"""Supprime de la file les tâches terminées/échouées/annulées (fichiers gardés)."""
|
||||
cursor = await db.execute(
|
||||
"DELETE FROM downloads WHERE status IN ('done', 'failed', 'cancelled')"
|
||||
)
|
||||
return cursor.rowcount or 0
|
||||
|
||||
async def list_all(self, limit: int = 200) -> list[dict]:
|
||||
rows = await db.fetchall(
|
||||
"SELECT * FROM downloads ORDER BY "
|
||||
"CASE status WHEN 'downloading' THEN 0 WHEN 'pending' THEN 1 WHEN 'paused' THEN 2 "
|
||||
"ELSE 3 END, updated_at DESC LIMIT ?",
|
||||
(limit,),
|
||||
)
|
||||
return [self._to_dict(row) for row in rows]
|
||||
|
||||
async def get(self, download_id: int) -> dict | None:
|
||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
||||
return self._to_dict(row) if row else None
|
||||
|
||||
# ------------------------------------------------------------ événements SSE
|
||||
|
||||
async def subscribe(self) -> AsyncIterator[dict]:
|
||||
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
|
||||
self._listeners.append(queue)
|
||||
try:
|
||||
while True:
|
||||
yield await queue.get()
|
||||
finally:
|
||||
self._listeners.remove(queue)
|
||||
|
||||
async def _emit(self, download_id: int) -> None:
|
||||
data = await self.get(download_id)
|
||||
if data is None:
|
||||
return
|
||||
for queue in self._listeners:
|
||||
with contextlib.suppress(asyncio.QueueFull):
|
||||
queue.put_nowait(data)
|
||||
|
||||
# ------------------------------------------------------------ worker interne
|
||||
|
||||
async def _worker(self) -> None:
|
||||
while True:
|
||||
download_id = await self._queue.get()
|
||||
row = await db.fetchone("SELECT status FROM downloads WHERE id = ?", (download_id,))
|
||||
if row is None or row["status"] != "pending":
|
||||
continue # annulé/pausé entre-temps
|
||||
# Tâche dédiée : annuler un téléchargement ne doit pas tuer le worker
|
||||
task = asyncio.create_task(self._download(download_id))
|
||||
self._tasks[download_id] = task
|
||||
self._pause_events[download_id] = asyncio.Event()
|
||||
self._pause_events[download_id].set()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
if asyncio.current_task().cancelling() > 0:
|
||||
raise # le worker lui-même s'arrête (stop())
|
||||
finally:
|
||||
self._tasks.pop(download_id, None)
|
||||
self._pause_events.pop(download_id, None)
|
||||
self._progress.pop(download_id, None)
|
||||
|
||||
async def _download(self, download_id: int) -> None:
|
||||
"""Dispatche HTTP/HLS ; gestion d'erreurs centralisée ici."""
|
||||
row = await self._get_row(download_id)
|
||||
try:
|
||||
if ".m3u8" in row["video_url"]:
|
||||
await self._download_hls(download_id, row)
|
||||
else:
|
||||
await self._download_http(download_id, row)
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Téléchargement annulé : %s", row["file_path"])
|
||||
raise
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
# Échec réseau/disque : journalisé, statut 'failed' visible dans l'UI
|
||||
logger.error("Échec du téléchargement de %s : %s", row["file_path"], exc)
|
||||
part = self._part_path(row["file_path"])
|
||||
downloaded = part.stat().st_size if part.exists() else 0
|
||||
await self._fail(download_id, exc, downloaded)
|
||||
|
||||
async def _download_http(self, download_id: int, row: Any) -> None:
|
||||
"""Téléchargement HTTP direct avec reprise via Range."""
|
||||
video_url, file_path = row["video_url"], row["file_path"]
|
||||
target = get_settings().download_dir / file_path
|
||||
part = self._part_path(file_path)
|
||||
downloaded = part.stat().st_size if part.exists() else 0
|
||||
|
||||
headers: dict[str, str] = {}
|
||||
if row["page_url"]:
|
||||
headers["Referer"] = row["page_url"]
|
||||
if downloaded:
|
||||
headers["Range"] = f"bytes={downloaded}-"
|
||||
logger.info("Reprise de %s à %d octets", file_path, downloaded)
|
||||
|
||||
await self._set_status(download_id, "downloading")
|
||||
await self._emit(download_id)
|
||||
started = time.monotonic()
|
||||
last_emit = 0.0
|
||||
|
||||
async with self._client.stream("GET", video_url, headers=headers) as response:
|
||||
if response.status_code == 416: # plage invalide → déjà complet
|
||||
part.rename(target)
|
||||
await self._finish(download_id, downloaded)
|
||||
return
|
||||
response.raise_for_status()
|
||||
if downloaded and response.status_code != 206:
|
||||
downloaded = 0 # serveur sans support Range → on repart de zéro
|
||||
logger.warning("Pas de reprise possible pour %s", file_path)
|
||||
total = int(response.headers.get("content-length") or 0) + downloaded or None
|
||||
await db.execute(
|
||||
"UPDATE downloads SET total_bytes = ? WHERE id = ?", (total, download_id)
|
||||
)
|
||||
|
||||
mode = "ab" if downloaded else "wb"
|
||||
with part.open(mode) as fh:
|
||||
async for chunk in response.aiter_bytes(1 << 16):
|
||||
event = self._pause_events.get(download_id)
|
||||
if event is not None:
|
||||
await event.wait() # pause coopérative
|
||||
fh.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
now = time.monotonic()
|
||||
if now - last_emit >= 1.0:
|
||||
last_emit = now
|
||||
await self._report_progress(download_id, downloaded, total, started)
|
||||
|
||||
part.rename(target)
|
||||
await self._finish(download_id, downloaded)
|
||||
|
||||
async def _download_hls(self, download_id: int, row: Any) -> None:
|
||||
"""Télécharge un flux HLS (.m3u8) via ffmpeg (remux en mp4).
|
||||
|
||||
Pause via SIGSTOP/SIGCONT du processus, annulation via kill.
|
||||
La progression est estimée depuis la durée totale de la playlist.
|
||||
"""
|
||||
video_url, file_path = row["video_url"], row["file_path"]
|
||||
target = get_settings().download_dir / file_path
|
||||
part = self._part_path(file_path)
|
||||
part.unlink(missing_ok=True) # pas de reprise partielle en HLS
|
||||
|
||||
ffmpeg_headers = f"User-Agent: {get_settings().user_agent}\r\n"
|
||||
if row["page_url"]:
|
||||
ffmpeg_headers += f"Referer: {row['page_url']}\r\n"
|
||||
ffmpeg_headers += "Accept-Language: fr-FR,fr;q=0.9,en;q=0.8\r\n"
|
||||
|
||||
total_seconds = await self._hls_duration(video_url, row["page_url"])
|
||||
|
||||
await self._set_status(download_id, "downloading")
|
||||
await self._emit(download_id)
|
||||
started = time.monotonic()
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
"ffmpeg", "-y", "-nostdin", "-v", "error", "-nostats", "-progress", "pipe:2",
|
||||
"-headers", ffmpeg_headers,
|
||||
"-i", video_url,
|
||||
"-c", "copy", "-bsf:a", "aac_adtstoasc", "-f", "mp4",
|
||||
str(part),
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
self._hls_processes[download_id] = process
|
||||
try:
|
||||
assert process.stderr is not None
|
||||
async for raw_line in process.stderr:
|
||||
event = self._pause_events.get(download_id)
|
||||
if event is not None and not event.is_set():
|
||||
process.send_signal(signal.SIGSTOP)
|
||||
await event.wait()
|
||||
process.send_signal(signal.SIGCONT)
|
||||
line = raw_line.decode(errors="replace")
|
||||
if match := _FFMPEG_TIME_RE.search(line):
|
||||
elapsed_video = _parse_ffmpeg_time(match.group(1))
|
||||
size = part.stat().st_size if part.exists() else 0
|
||||
total_est = None
|
||||
if total_seconds and elapsed_video > 0:
|
||||
total_est = int(size / elapsed_video * total_seconds)
|
||||
await self._report_progress(download_id, size, total_est, started)
|
||||
return_code = await process.wait()
|
||||
finally:
|
||||
self._hls_processes.pop(download_id, None)
|
||||
|
||||
if return_code != 0:
|
||||
part.unlink(missing_ok=True)
|
||||
raise OSError(f"ffmpeg a échoué (code {return_code}) sur le flux HLS")
|
||||
size = part.stat().st_size
|
||||
part.rename(target)
|
||||
await self._finish(download_id, size)
|
||||
|
||||
async def _hls_duration(self, playlist_url: str, referer: str | None) -> float | None:
|
||||
"""Durée totale d'une playlist HLS (somme des EXTINF de la variante max)."""
|
||||
headers = {"Referer": referer} if referer else {}
|
||||
try:
|
||||
response = await self._client.get(playlist_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
body = response.text
|
||||
# Playlist maître → on suit la variante de plus haut débit
|
||||
variant = _best_variant(body, playlist_url)
|
||||
if variant and variant != playlist_url:
|
||||
response = await self._client.get(variant, headers=headers)
|
||||
response.raise_for_status()
|
||||
body = response.text
|
||||
durations = [float(m) for m in _EXTINF_RE.findall(body)]
|
||||
return sum(durations) if durations else None
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
logger.warning("Durée HLS indéterminée pour %s : %s", playlist_url, exc)
|
||||
return None
|
||||
|
||||
async def _report_progress(
|
||||
self, download_id: int, downloaded: int, total: int | None, started: float
|
||||
) -> None:
|
||||
elapsed = time.monotonic() - started
|
||||
self._progress[download_id] = {
|
||||
"downloaded_bytes": downloaded,
|
||||
"total_bytes": total,
|
||||
"speed_bps": int(downloaded / elapsed) if elapsed > 0 else 0,
|
||||
}
|
||||
await self._emit(download_id)
|
||||
|
||||
async def _fail(self, download_id: int, exc: Exception, downloaded: int = 0) -> None:
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = 'failed', error = ?, "
|
||||
"downloaded_bytes = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
(str(exc)[:500], downloaded, download_id),
|
||||
)
|
||||
await self._emit(download_id)
|
||||
|
||||
async def _finish(self, download_id: int, size: int) -> None:
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = 'done', total_bytes = ?, downloaded_bytes = ?, "
|
||||
"updated_at = datetime('now') WHERE id = ?",
|
||||
(size, size, download_id),
|
||||
)
|
||||
await self._emit(download_id)
|
||||
logger.info("Téléchargement terminé (id=%s, %d octets)", download_id, size)
|
||||
|
||||
# ------------------------------------------------------------ helpers
|
||||
|
||||
async def _get_row(self, download_id: int) -> Any:
|
||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
||||
if row is None:
|
||||
raise KeyError(f"Téléchargement introuvable : {download_id}")
|
||||
return row
|
||||
|
||||
async def _set_status(self, download_id: int, status: str) -> None:
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
(status, download_id),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _part_path(file_path: str | None) -> Path:
|
||||
name = file_path or "video"
|
||||
return get_settings().download_dir / (name + ".part")
|
||||
|
||||
@staticmethod
|
||||
def _guess_extension(url: str) -> str:
|
||||
match = re.search(r"\.(mp4|mkv|webm|avi|m3u8)(?:\?|$)", url)
|
||||
ext = match.group(1) if match else "mp4"
|
||||
return ".mp4" if ext == "m3u8" else f".{ext}"
|
||||
|
||||
def _to_dict(self, row: Any, duplicate: bool = False) -> dict:
|
||||
data = dict(row)
|
||||
live = self._progress.get(data["id"], {})
|
||||
downloaded = live.get("downloaded_bytes", data["downloaded_bytes"])
|
||||
total = live.get("total_bytes", data["total_bytes"])
|
||||
speed = live.get("speed_bps", 0)
|
||||
percent = round(downloaded / total * 100, 1) if total else None
|
||||
eta = int((total - downloaded) / speed) if total and speed else None
|
||||
data.update(
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=total,
|
||||
percent=percent,
|
||||
speed_bps=speed,
|
||||
eta_seconds=eta,
|
||||
status_label=_STATUS_LABELS.get(data["status"], data["status"]),
|
||||
duplicate=duplicate,
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
download_manager = DownloadManager()
|
||||
Reference in New Issue
Block a user