Roman
41566ab5fb
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
500 lines
20 KiB
Python
500 lines
20 KiB
Python
"""Gestionnaire de téléchargements : file asyncio, parallélisme limité,
|
|
pause/reprise (Range HTTP), anti-doublons, persistance, progression temps réel.
|
|
|
|
Les statuts : pending → downloading → done | failed | cancelled
|
|
↕ paused
|
|
"""
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import logging
|
|
import re
|
|
import signal
|
|
import time
|
|
import unicodedata
|
|
from collections.abc import AsyncIterator
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import urljoin
|
|
|
|
import httpx
|
|
|
|
from app.config import get_settings
|
|
from app.db import db
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ACTIVE_STATUSES = ("pending", "downloading", "paused")
|
|
|
|
_FFMPEG_TIME_RE = re.compile(r"time=(\d+:\d+:\d+(?:\.\d+)?)")
|
|
_EXTINF_RE = re.compile(r"#EXTINF:([\d.]+)")
|
|
_BANDWIDTH_RE = re.compile(r"#EXT-X-STREAM-INF:[^\n]*BANDWIDTH=(\d+)[^\n]*\n(\S+)")
|
|
|
|
|
|
def _parse_ffmpeg_time(value: str) -> float:
|
|
parts = value.split(":")
|
|
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
|
|
|
|
|
def _best_variant(master_body: str, base_url: str) -> str | None:
|
|
"""URL de la variante au plus haut débit d'une playlist maître HLS."""
|
|
variants = [
|
|
(int(bw), urljoin(base_url, uri)) for bw, uri in _BANDWIDTH_RE.findall(master_body)
|
|
]
|
|
return max(variants)[1] if variants else None
|
|
|
|
_STATUS_LABELS = {
|
|
"pending": "en attente",
|
|
"downloading": "en cours",
|
|
"paused": "en pause",
|
|
"done": "terminé",
|
|
"failed": "échec",
|
|
"cancelled": "annulé",
|
|
}
|
|
|
|
|
|
def sanitize_filename(name: str) -> str:
|
|
"""Nettoie un nom de fichier : caractères interdits retirés, anti-traversée."""
|
|
name = unicodedata.normalize("NFKC", name)
|
|
name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', " ", name)
|
|
name = re.sub(r"\s+", " ", name).strip(" .")
|
|
if not name or name in (".", ".."):
|
|
name = "video"
|
|
return name[:150]
|
|
|
|
|
|
class DownloadManager:
|
|
"""File d'attente de téléchargements, injectée dans les routes via app.state."""
|
|
|
|
def __init__(self) -> None:
|
|
self._queue: asyncio.Queue[int] # créée dans start() (affinité avec la boucle)
|
|
self._tasks: dict[int, asyncio.Task] = {} # download_id → tâche asyncio
|
|
self._pause_events: dict[int, asyncio.Event] = {} # set = peut tourner
|
|
self._progress: dict[int, dict[str, Any]] = {} # progression temps réel en mémoire
|
|
self._listeners: list[asyncio.Queue] = []
|
|
self._workers: list[asyncio.Task] = []
|
|
self._client: httpx.AsyncClient | None = None
|
|
self._hls_processes: dict[int, asyncio.subprocess.Process] = {}
|
|
|
|
# ------------------------------------------------------------ cycle de vie
|
|
|
|
async def start(self) -> None:
|
|
self._queue = asyncio.Queue()
|
|
settings = get_settings()
|
|
self._client = httpx.AsyncClient(
|
|
timeout=httpx.Timeout(30.0, read=300.0),
|
|
follow_redirects=True,
|
|
headers={
|
|
"User-Agent": settings.user_agent,
|
|
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
|
},
|
|
)
|
|
# Restaure les téléchargements interrompus (crash/arrêt) en 'pending'
|
|
await db.execute(
|
|
"UPDATE downloads SET status = 'pending', updated_at = datetime('now') "
|
|
"WHERE status = 'downloading'"
|
|
)
|
|
await self._scan_download_dir()
|
|
for _ in range(settings.max_parallel_downloads):
|
|
self._workers.append(asyncio.create_task(self._worker()))
|
|
logger.info("DownloadManager démarré (%d workers)", settings.max_parallel_downloads)
|
|
|
|
async def stop(self) -> None:
|
|
for worker in self._workers:
|
|
worker.cancel()
|
|
for task in self._tasks.values():
|
|
task.cancel()
|
|
for proc in self._hls_processes.values():
|
|
if proc.returncode is None:
|
|
proc.kill()
|
|
if self._client:
|
|
await self._client.aclose()
|
|
self._workers.clear()
|
|
self._tasks.clear()
|
|
|
|
async def _scan_download_dir(self) -> None:
|
|
"""Restaure en 'done' les fichiers présents sur disque mais inconnus de la DB."""
|
|
download_dir = get_settings().download_dir
|
|
rows = await db.fetchall("SELECT file_path FROM downloads WHERE file_path IS NOT NULL")
|
|
known = {row["file_path"] for row in rows}
|
|
for path in download_dir.iterdir():
|
|
if path.is_file() and path.suffix != ".part" and path.name not in known:
|
|
size = path.stat().st_size
|
|
await db.execute(
|
|
"INSERT INTO downloads "
|
|
"(source_key, video_url, page_url, title, file_path, status, "
|
|
" total_bytes, downloaded_bytes) "
|
|
"VALUES (?, ?, ?, ?, ?, 'done', ?, ?)",
|
|
(
|
|
f"file:{path.name}",
|
|
"",
|
|
"",
|
|
path.stem,
|
|
path.name,
|
|
size,
|
|
size,
|
|
),
|
|
)
|
|
logger.info("Fichier restauré depuis le disque : %s", path.name)
|
|
|
|
# ------------------------------------------------------------ API publique
|
|
|
|
async def enqueue(self, video_url: str, page_url: str, title: str) -> dict:
|
|
"""Ajoute un téléchargement ; retourne la tâche existante si doublon actif."""
|
|
source_key = video_url
|
|
existing = await db.fetchone(
|
|
f"SELECT * FROM downloads WHERE source_key = ? AND status IN "
|
|
f"({','.join('?' * len(ACTIVE_STATUSES))})",
|
|
(source_key, *ACTIVE_STATUSES),
|
|
)
|
|
if existing:
|
|
logger.info("Anti-doublon : %s déjà en file (id=%s)", title, existing["id"])
|
|
return self._to_dict(existing, duplicate=True)
|
|
|
|
filename = sanitize_filename(title) + self._guess_extension(video_url)
|
|
cursor = await db.execute(
|
|
"INSERT INTO downloads (source_key, video_url, page_url, title, file_path) "
|
|
"VALUES (?, ?, ?, ?, ?)",
|
|
(source_key, video_url, page_url, title, filename),
|
|
)
|
|
download_id = cursor.lastrowid
|
|
await self._queue.put(download_id)
|
|
await self._emit(download_id)
|
|
logger.info("Téléchargement ajouté : %s (id=%s)", title, download_id)
|
|
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
|
return self._to_dict(row)
|
|
|
|
async def pause(self, download_id: int) -> bool:
|
|
event = self._pause_events.get(download_id)
|
|
if event:
|
|
event.clear()
|
|
await self._set_status(download_id, "paused")
|
|
return True
|
|
|
|
async def resume(self, download_id: int) -> bool:
|
|
row = await self._get_row(download_id)
|
|
if row["status"] != "paused":
|
|
return False
|
|
await self._set_status(download_id, "pending")
|
|
await self._queue.put(download_id)
|
|
return True
|
|
|
|
async def retry(self, download_id: int) -> bool:
|
|
row = await self._get_row(download_id)
|
|
if row["status"] not in ("failed", "cancelled"):
|
|
return False
|
|
await db.execute(
|
|
"UPDATE downloads SET status = 'pending', error = NULL, downloaded_bytes = 0, "
|
|
"updated_at = datetime('now') WHERE id = ?",
|
|
(download_id,),
|
|
)
|
|
part = self._part_path(row["file_path"])
|
|
part.unlink(missing_ok=True)
|
|
await self._queue.put(download_id)
|
|
await self._emit(download_id)
|
|
return True
|
|
|
|
async def cancel(self, download_id: int) -> bool:
|
|
task = self._tasks.get(download_id)
|
|
if task:
|
|
task.cancel()
|
|
proc = self._hls_processes.get(download_id)
|
|
if proc and proc.returncode is None:
|
|
proc.kill()
|
|
await self._set_status(download_id, "cancelled")
|
|
row = await self._get_row(download_id)
|
|
self._part_path(row["file_path"]).unlink(missing_ok=True)
|
|
await self._emit(download_id)
|
|
return True
|
|
|
|
async def cancel_all(self) -> int:
|
|
rows = await db.fetchall(
|
|
f"SELECT id FROM downloads WHERE status IN ({','.join('?' * len(ACTIVE_STATUSES))})",
|
|
ACTIVE_STATUSES,
|
|
)
|
|
for row in rows:
|
|
await self.cancel(row["id"])
|
|
return len(rows)
|
|
|
|
async def clear_finished(self) -> int:
|
|
"""Supprime de la file les tâches terminées/échouées/annulées (fichiers gardés)."""
|
|
cursor = await db.execute(
|
|
"DELETE FROM downloads WHERE status IN ('done', 'failed', 'cancelled')"
|
|
)
|
|
return cursor.rowcount or 0
|
|
|
|
async def list_all(self, limit: int = 200) -> list[dict]:
|
|
rows = await db.fetchall(
|
|
"SELECT * FROM downloads ORDER BY "
|
|
"CASE status WHEN 'downloading' THEN 0 WHEN 'pending' THEN 1 WHEN 'paused' THEN 2 "
|
|
"ELSE 3 END, updated_at DESC LIMIT ?",
|
|
(limit,),
|
|
)
|
|
return [self._to_dict(row) for row in rows]
|
|
|
|
async def get(self, download_id: int) -> dict | None:
|
|
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
|
return self._to_dict(row) if row else None
|
|
|
|
# ------------------------------------------------------------ événements SSE
|
|
|
|
async def subscribe(self) -> AsyncIterator[dict]:
|
|
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
|
|
self._listeners.append(queue)
|
|
try:
|
|
while True:
|
|
yield await queue.get()
|
|
finally:
|
|
self._listeners.remove(queue)
|
|
|
|
async def _emit(self, download_id: int) -> None:
|
|
data = await self.get(download_id)
|
|
if data is None:
|
|
return
|
|
for queue in self._listeners:
|
|
with contextlib.suppress(asyncio.QueueFull):
|
|
queue.put_nowait(data)
|
|
|
|
# ------------------------------------------------------------ worker interne
|
|
|
|
async def _worker(self) -> None:
|
|
while True:
|
|
download_id = await self._queue.get()
|
|
row = await db.fetchone("SELECT status FROM downloads WHERE id = ?", (download_id,))
|
|
if row is None or row["status"] != "pending":
|
|
continue # annulé/pausé entre-temps
|
|
# Tâche dédiée : annuler un téléchargement ne doit pas tuer le worker
|
|
task = asyncio.create_task(self._download(download_id))
|
|
self._tasks[download_id] = task
|
|
self._pause_events[download_id] = asyncio.Event()
|
|
self._pause_events[download_id].set()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
if asyncio.current_task().cancelling() > 0:
|
|
raise # le worker lui-même s'arrête (stop())
|
|
finally:
|
|
self._tasks.pop(download_id, None)
|
|
self._pause_events.pop(download_id, None)
|
|
self._progress.pop(download_id, None)
|
|
|
|
async def _download(self, download_id: int) -> None:
|
|
"""Dispatche HTTP/HLS ; gestion d'erreurs centralisée ici."""
|
|
row = await self._get_row(download_id)
|
|
try:
|
|
if ".m3u8" in row["video_url"]:
|
|
await self._download_hls(download_id, row)
|
|
else:
|
|
await self._download_http(download_id, row)
|
|
except asyncio.CancelledError:
|
|
logger.info("Téléchargement annulé : %s", row["file_path"])
|
|
raise
|
|
except (httpx.HTTPError, OSError) as exc:
|
|
# Échec réseau/disque : journalisé, statut 'failed' visible dans l'UI
|
|
logger.error("Échec du téléchargement de %s : %s", row["file_path"], exc)
|
|
part = self._part_path(row["file_path"])
|
|
downloaded = part.stat().st_size if part.exists() else 0
|
|
await self._fail(download_id, exc, downloaded)
|
|
|
|
async def _download_http(self, download_id: int, row: Any) -> None:
|
|
"""Téléchargement HTTP direct avec reprise via Range."""
|
|
video_url, file_path = row["video_url"], row["file_path"]
|
|
target = get_settings().download_dir / file_path
|
|
part = self._part_path(file_path)
|
|
downloaded = part.stat().st_size if part.exists() else 0
|
|
|
|
headers: dict[str, str] = {}
|
|
if row["page_url"]:
|
|
headers["Referer"] = row["page_url"]
|
|
if downloaded:
|
|
headers["Range"] = f"bytes={downloaded}-"
|
|
logger.info("Reprise de %s à %d octets", file_path, downloaded)
|
|
|
|
await self._set_status(download_id, "downloading")
|
|
await self._emit(download_id)
|
|
started = time.monotonic()
|
|
last_emit = 0.0
|
|
|
|
async with self._client.stream("GET", video_url, headers=headers) as response:
|
|
if response.status_code == 416: # plage invalide → déjà complet
|
|
part.rename(target)
|
|
await self._finish(download_id, downloaded)
|
|
return
|
|
response.raise_for_status()
|
|
if downloaded and response.status_code != 206:
|
|
downloaded = 0 # serveur sans support Range → on repart de zéro
|
|
logger.warning("Pas de reprise possible pour %s", file_path)
|
|
total = int(response.headers.get("content-length") or 0) + downloaded or None
|
|
await db.execute(
|
|
"UPDATE downloads SET total_bytes = ? WHERE id = ?", (total, download_id)
|
|
)
|
|
|
|
mode = "ab" if downloaded else "wb"
|
|
with part.open(mode) as fh:
|
|
async for chunk in response.aiter_bytes(1 << 16):
|
|
event = self._pause_events.get(download_id)
|
|
if event is not None:
|
|
await event.wait() # pause coopérative
|
|
fh.write(chunk)
|
|
downloaded += len(chunk)
|
|
now = time.monotonic()
|
|
if now - last_emit >= 1.0:
|
|
last_emit = now
|
|
await self._report_progress(download_id, downloaded, total, started)
|
|
|
|
part.rename(target)
|
|
await self._finish(download_id, downloaded)
|
|
|
|
async def _download_hls(self, download_id: int, row: Any) -> None:
|
|
"""Télécharge un flux HLS (.m3u8) via ffmpeg (remux en mp4).
|
|
|
|
Pause via SIGSTOP/SIGCONT du processus, annulation via kill.
|
|
La progression est estimée depuis la durée totale de la playlist.
|
|
"""
|
|
video_url, file_path = row["video_url"], row["file_path"]
|
|
target = get_settings().download_dir / file_path
|
|
part = self._part_path(file_path)
|
|
part.unlink(missing_ok=True) # pas de reprise partielle en HLS
|
|
|
|
ffmpeg_headers = f"User-Agent: {get_settings().user_agent}\r\n"
|
|
if row["page_url"]:
|
|
ffmpeg_headers += f"Referer: {row['page_url']}\r\n"
|
|
ffmpeg_headers += "Accept-Language: fr-FR,fr;q=0.9,en;q=0.8\r\n"
|
|
|
|
total_seconds = await self._hls_duration(video_url, row["page_url"])
|
|
|
|
await self._set_status(download_id, "downloading")
|
|
await self._emit(download_id)
|
|
started = time.monotonic()
|
|
|
|
process = await asyncio.create_subprocess_exec(
|
|
"ffmpeg", "-y", "-nostdin", "-v", "error", "-nostats", "-progress", "pipe:2",
|
|
"-headers", ffmpeg_headers,
|
|
"-i", video_url,
|
|
"-c", "copy", "-bsf:a", "aac_adtstoasc", "-f", "mp4",
|
|
str(part),
|
|
stdout=asyncio.subprocess.DEVNULL,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
self._hls_processes[download_id] = process
|
|
try:
|
|
assert process.stderr is not None
|
|
async for raw_line in process.stderr:
|
|
event = self._pause_events.get(download_id)
|
|
if event is not None and not event.is_set():
|
|
process.send_signal(signal.SIGSTOP)
|
|
await event.wait()
|
|
process.send_signal(signal.SIGCONT)
|
|
line = raw_line.decode(errors="replace")
|
|
if match := _FFMPEG_TIME_RE.search(line):
|
|
elapsed_video = _parse_ffmpeg_time(match.group(1))
|
|
size = part.stat().st_size if part.exists() else 0
|
|
total_est = None
|
|
if total_seconds and elapsed_video > 0:
|
|
total_est = int(size / elapsed_video * total_seconds)
|
|
await self._report_progress(download_id, size, total_est, started)
|
|
return_code = await process.wait()
|
|
finally:
|
|
self._hls_processes.pop(download_id, None)
|
|
|
|
if return_code != 0:
|
|
part.unlink(missing_ok=True)
|
|
raise OSError(f"ffmpeg a échoué (code {return_code}) sur le flux HLS")
|
|
size = part.stat().st_size
|
|
part.rename(target)
|
|
await self._finish(download_id, size)
|
|
|
|
async def _hls_duration(self, playlist_url: str, referer: str | None) -> float | None:
|
|
"""Durée totale d'une playlist HLS (somme des EXTINF de la variante max)."""
|
|
headers = {"Referer": referer} if referer else {}
|
|
try:
|
|
response = await self._client.get(playlist_url, headers=headers)
|
|
response.raise_for_status()
|
|
body = response.text
|
|
# Playlist maître → on suit la variante de plus haut débit
|
|
variant = _best_variant(body, playlist_url)
|
|
if variant and variant != playlist_url:
|
|
response = await self._client.get(variant, headers=headers)
|
|
response.raise_for_status()
|
|
body = response.text
|
|
durations = [float(m) for m in _EXTINF_RE.findall(body)]
|
|
return sum(durations) if durations else None
|
|
except (httpx.HTTPError, ValueError) as exc:
|
|
logger.warning("Durée HLS indéterminée pour %s : %s", playlist_url, exc)
|
|
return None
|
|
|
|
async def _report_progress(
|
|
self, download_id: int, downloaded: int, total: int | None, started: float
|
|
) -> None:
|
|
elapsed = time.monotonic() - started
|
|
self._progress[download_id] = {
|
|
"downloaded_bytes": downloaded,
|
|
"total_bytes": total,
|
|
"speed_bps": int(downloaded / elapsed) if elapsed > 0 else 0,
|
|
}
|
|
await self._emit(download_id)
|
|
|
|
async def _fail(self, download_id: int, exc: Exception, downloaded: int = 0) -> None:
|
|
await db.execute(
|
|
"UPDATE downloads SET status = 'failed', error = ?, "
|
|
"downloaded_bytes = ?, updated_at = datetime('now') WHERE id = ?",
|
|
(str(exc)[:500], downloaded, download_id),
|
|
)
|
|
await self._emit(download_id)
|
|
|
|
async def _finish(self, download_id: int, size: int) -> None:
|
|
await db.execute(
|
|
"UPDATE downloads SET status = 'done', total_bytes = ?, downloaded_bytes = ?, "
|
|
"updated_at = datetime('now') WHERE id = ?",
|
|
(size, size, download_id),
|
|
)
|
|
await self._emit(download_id)
|
|
logger.info("Téléchargement terminé (id=%s, %d octets)", download_id, size)
|
|
|
|
# ------------------------------------------------------------ helpers
|
|
|
|
async def _get_row(self, download_id: int) -> Any:
|
|
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
|
if row is None:
|
|
raise KeyError(f"Téléchargement introuvable : {download_id}")
|
|
return row
|
|
|
|
async def _set_status(self, download_id: int, status: str) -> None:
|
|
await db.execute(
|
|
"UPDATE downloads SET status = ?, updated_at = datetime('now') WHERE id = ?",
|
|
(status, download_id),
|
|
)
|
|
|
|
@staticmethod
|
|
def _part_path(file_path: str | None) -> Path:
|
|
name = file_path or "video"
|
|
return get_settings().download_dir / (name + ".part")
|
|
|
|
@staticmethod
|
|
def _guess_extension(url: str) -> str:
|
|
match = re.search(r"\.(mp4|mkv|webm|avi|m3u8)(?:\?|$)", url)
|
|
ext = match.group(1) if match else "mp4"
|
|
return ".mp4" if ext == "m3u8" else f".{ext}"
|
|
|
|
def _to_dict(self, row: Any, duplicate: bool = False) -> dict:
|
|
data = dict(row)
|
|
live = self._progress.get(data["id"], {})
|
|
downloaded = live.get("downloaded_bytes", data["downloaded_bytes"])
|
|
total = live.get("total_bytes", data["total_bytes"])
|
|
speed = live.get("speed_bps", 0)
|
|
percent = round(downloaded / total * 100, 1) if total else None
|
|
eta = int((total - downloaded) / speed) if total and speed else None
|
|
data.update(
|
|
downloaded_bytes=downloaded,
|
|
total_bytes=total,
|
|
percent=percent,
|
|
speed_bps=speed,
|
|
eta_seconds=eta,
|
|
status_label=_STATUS_LABELS.get(data["status"], data["status"]),
|
|
duplicate=duplicate,
|
|
)
|
|
return data
|
|
|
|
|
|
download_manager = DownloadManager()
|