Files
ohm_streaming/app/services/downloads.py
T
Roman d5fa546dd7 Pour toi : fallback sans Sonarr et amorçage sans historique
- Sans Sonarr, les séries/films téléchargés donnent leurs genres via leur
  fiche French-Stream (page_url des téléchargements), convertis vers les
  catégories Kitsu (mapping FR→EN ; genres sans équivalent ignorés)
- Aucun historique (ni téléchargement, ni favori, ni Sonarr) : réponse
  cold_start non cachée → carte d'amorçage « téléchargez un premier titre »
- Fin de téléchargement → invalidation immédiate des recommandations
- genres_of_page ignore les fiches hors French-Stream (URLs d'animés)
2026-09-25 15:14:14 +00:00

587 lines
24 KiB
Python

"""Gestionnaire de téléchargements : file asyncio, parallélisme limité,
pause/reprise (Range HTTP), anti-doublons, persistance, progression temps réel.
Les statuts : pending → downloading → done | failed | cancelled
↕ paused
"""
import asyncio
import contextlib
import logging
import re
import signal
import time
import unicodedata
from collections.abc import AsyncIterator
from pathlib import Path
from typing import Any
from urllib.parse import urljoin
import httpx
from app.config import get_settings
from app.db import db
logger = logging.getLogger(__name__)
ACTIVE_STATUSES = ("pending", "downloading", "paused")
_FFMPEG_TIME_RE = re.compile(r"time=(\d+:\d+:\d+(?:\.\d+)?)")
_EXTINF_RE = re.compile(r"#EXTINF:([\d.]+)")
_BANDWIDTH_RE = re.compile(r"#EXT-X-STREAM-INF:[^\n]*BANDWIDTH=(\d+)[^\n]*\n(\S+)")
def _parse_ffmpeg_time(value: str) -> float:
parts = value.split(":")
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
def _best_variant(master_body: str, base_url: str) -> str | None:
"""URL de la variante au plus haut débit d'une playlist maître HLS."""
variants = [
(int(bw), urljoin(base_url, uri)) for bw, uri in _BANDWIDTH_RE.findall(master_body)
]
return max(variants)[1] if variants else None
_STATUS_LABELS = {
"pending": "en attente",
"downloading": "en cours",
"paused": "en pause",
"done": "terminé",
"failed": "échec",
"cancelled": "annulé",
}
def sanitize_filename(name: str) -> str:
"""Nettoie un nom de fichier : caractères interdits retirés, anti-traversée."""
name = unicodedata.normalize("NFKC", name)
name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', " ", name)
name = re.sub(r"\s+", " ", name).strip(" .")
if not name or name in (".", ".."):
name = "video"
return name[:150]
_SERIES_RE = re.compile(r"^(.*?)[\s\-–—]*(?:épisode|episode|ep|e)\s*\d", re.IGNORECASE)
def series_dirname(title: str) -> str | None:
"""Nom du sous-dossier d'un titre (« One Piece - E12 » → « One Piece »).
Range les épisodes par animé sur le disque (bien meilleur parsing Plex).
None si aucun marqueur d'épisode détecté → fichier à plat.
"""
match = _SERIES_RE.match(title)
base = sanitize_filename(match.group(1).strip()) if match else ""
return base or None
_EPISODE_RE = re.compile(
r"^(?P<series>.+?)\s*-\s*(?:Saison\s+(?P<season>\d+)\s*-\s*)?"
r"E(?P<ep>\d+(?:[.,]\d+)?)(?P<tail>\s*\([^)]*\))?\s*$",
re.IGNORECASE,
)
def plex_filename(title: str, extension: str) -> str | None:
"""Nom de fichier compatible Plex (« S01E09 ») pour un titre d'épisode.
Les scrapers produisent « Série - Saison 1 - E9 (VF) » ; le scanner Plex
exige « Série - S01E09 (VF) » (les épisodes 1-9 sont sinon ignorés).
None si le titre ne suit pas le format épisode → nommage inchangé.
"""
match = _EPISODE_RE.match(title)
if not match:
return None
series = match.group("series")
season = int(match.group("season") or 1)
ep_raw = match.group("ep").replace(",", ".")
ep = float(ep_raw)
ep_label = f"{int(ep):02d}" if ep.is_integer() else ep_raw
tail = match.group("tail") or ""
return sanitize_filename(f"{series} - S{season:02d}E{ep_label}{tail}") + extension
class DownloadManager:
"""File d'attente de téléchargements, injectée dans les routes via app.state."""
def __init__(self) -> None:
self._queue: asyncio.Queue[int] # créée dans start() (affinité avec la boucle)
self._tasks: dict[int, asyncio.Task] = {} # download_id → tâche asyncio
self._pause_events: dict[int, asyncio.Event] = {} # set = peut tourner
self._progress: dict[int, dict[str, Any]] = {} # progression temps réel en mémoire
self._listeners: list[asyncio.Queue] = []
self._workers: list[asyncio.Task] = []
self._client: httpx.AsyncClient | None = None
self._hls_processes: dict[int, asyncio.subprocess.Process] = {}
# ------------------------------------------------------------ cycle de vie
async def start(self) -> None:
self._queue = asyncio.Queue()
settings = get_settings()
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(30.0, read=300.0),
follow_redirects=True,
headers={
"User-Agent": settings.user_agent,
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
},
)
# Restaure les téléchargements interrompus (crash/arrêt) en 'pending'
await db.execute(
"UPDATE downloads SET status = 'pending', updated_at = datetime('now') "
"WHERE status = 'downloading'"
)
await self._scan_download_dir()
for _ in range(settings.max_parallel_downloads):
self._workers.append(asyncio.create_task(self._worker()))
logger.info("DownloadManager démarré (%d workers)", settings.max_parallel_downloads)
async def stop(self) -> None:
for worker in self._workers:
worker.cancel()
for task in self._tasks.values():
task.cancel()
for proc in self._hls_processes.values():
if proc.returncode is None:
proc.kill()
if self._client:
await self._client.aclose()
self._workers.clear()
self._tasks.clear()
async def _scan_download_dir(self) -> None:
"""Restaure en 'done' les fichiers présents sur disque mais inconnus de la DB."""
download_dir = get_settings().download_dir
rows = await db.fetchall("SELECT file_path FROM downloads WHERE file_path IS NOT NULL")
known = {row["file_path"] for row in rows}
for path in download_dir.rglob("*"):
if not path.is_file() or path.suffix == ".part":
continue
rel_path = path.relative_to(download_dir).as_posix()
if rel_path in known:
continue
size = path.stat().st_size
await db.execute(
"INSERT INTO downloads "
"(source_key, video_url, page_url, title, file_path, status, "
" total_bytes, downloaded_bytes) "
"VALUES (?, ?, ?, ?, ?, 'done', ?, ?)",
(
f"file:{rel_path}",
"",
"",
path.stem,
rel_path,
size,
size,
),
)
logger.info("Fichier restauré depuis le disque : %s", rel_path)
# ------------------------------------------------------------ API publique
async def enqueue(
self, video_url: str, page_url: str, title: str, source_key: str | None = None
) -> dict:
"""Ajoute un téléchargement ; retourne la tâche existante si doublon actif.
source_key : clé de déduplication (par défaut l'URL vidéo). Les grabs
Sonarr utilisent « sonarr:<infohash>|<url> » pour rester suivis.
"""
source_key = source_key or video_url
existing = await db.fetchone(
f"SELECT * FROM downloads WHERE source_key = ? AND status IN "
f"({','.join('?' * len(ACTIVE_STATUSES))})",
(source_key, *ACTIVE_STATUSES),
)
if existing:
logger.info("Anti-doublon : %s déjà en file (id=%s)", title, existing["id"])
return self._to_dict(existing, duplicate=True)
extension = self._guess_extension(video_url)
filename = plex_filename(title, extension) or sanitize_filename(title) + extension
series = series_dirname(title)
file_path = f"{series}/{filename}" if series else filename
cursor = await db.execute(
"INSERT INTO downloads (source_key, video_url, page_url, title, file_path) "
"VALUES (?, ?, ?, ?, ?)",
(source_key, video_url, page_url, title, file_path),
)
download_id = cursor.lastrowid
await self._queue.put(download_id)
await self._emit(download_id)
logger.info("Téléchargement ajouté : %s (id=%s)", title, download_id)
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
return self._to_dict(row)
async def pause(self, download_id: int) -> bool:
event = self._pause_events.get(download_id)
if event:
event.clear()
await self._set_status(download_id, "paused")
return True
async def resume(self, download_id: int) -> bool:
row = await self._get_row(download_id)
if row["status"] != "paused":
return False
await self._set_status(download_id, "pending")
await self._queue.put(download_id)
return True
async def retry(self, download_id: int) -> bool:
row = await self._get_row(download_id)
if row["status"] not in ("failed", "cancelled"):
return False
await db.execute(
"UPDATE downloads SET status = 'pending', error = NULL, downloaded_bytes = 0, "
"updated_at = datetime('now') WHERE id = ?",
(download_id,),
)
part = self._part_path(row["file_path"])
part.unlink(missing_ok=True)
await self._queue.put(download_id)
await self._emit(download_id)
return True
async def cancel(self, download_id: int) -> bool:
task = self._tasks.get(download_id)
if task:
task.cancel()
proc = self._hls_processes.get(download_id)
if proc and proc.returncode is None:
proc.kill()
await self._set_status(download_id, "cancelled")
row = await self._get_row(download_id)
self._part_path(row["file_path"]).unlink(missing_ok=True)
await self._emit(download_id)
return True
async def cancel_all(self) -> int:
rows = await db.fetchall(
f"SELECT id FROM downloads WHERE status IN ({','.join('?' * len(ACTIVE_STATUSES))})",
ACTIVE_STATUSES,
)
for row in rows:
await self.cancel(row["id"])
return len(rows)
async def clear_finished(self) -> int:
"""Supprime de la file les tâches terminées/échouées/annulées (fichiers gardés)."""
cursor = await db.execute(
"DELETE FROM downloads WHERE status IN ('done', 'failed', 'cancelled')"
)
return cursor.rowcount or 0
async def delete(self, download_id: int, delete_file: bool = False) -> bool:
"""Supprime une tâche de la file (annulée avant si active).
delete_file=True efface aussi le fichier téléchargé du disque.
"""
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
if row is None:
return False
if row["status"] in ACTIVE_STATUSES:
await self.cancel(download_id)
if delete_file and row["file_path"]:
target = get_settings().download_dir / row["file_path"]
target.unlink(missing_ok=True)
if target.parent != get_settings().download_dir:
with contextlib.suppress(OSError): # dossier de l'animé vide → retiré
target.parent.rmdir()
self._part_path(row["file_path"]).unlink(missing_ok=True)
await db.execute("DELETE FROM downloads WHERE id = ?", (download_id,))
await self._emit_removed(download_id)
logger.info("Téléchargement supprimé (id=%s, fichier=%s)", download_id, delete_file)
return True
async def list_all(self, limit: int = 2000) -> list[dict]:
rows = await db.fetchall(
"SELECT * FROM downloads ORDER BY "
"CASE status WHEN 'downloading' THEN 0 WHEN 'pending' THEN 1 WHEN 'paused' THEN 2 "
"ELSE 3 END, updated_at DESC LIMIT ?",
(limit,),
)
return [self._to_dict(row) for row in rows]
async def get(self, download_id: int) -> dict | None:
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
return self._to_dict(row) if row else None
# ------------------------------------------------------------ événements SSE
async def subscribe(self) -> AsyncIterator[dict]:
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
self._listeners.append(queue)
try:
while True:
yield await queue.get()
finally:
self._listeners.remove(queue)
async def _emit(self, download_id: int) -> None:
data = await self.get(download_id)
if data is None:
return
self._publish({"type": "update", "item": data})
async def _emit_removed(self, download_id: int) -> None:
self._publish({"type": "removed", "id": download_id})
def _publish(self, message: dict) -> None:
for queue in self._listeners:
with contextlib.suppress(asyncio.QueueFull):
queue.put_nowait(message)
# ------------------------------------------------------------ worker interne
async def _worker(self) -> None:
while True:
download_id = await self._queue.get()
row = await db.fetchone("SELECT status FROM downloads WHERE id = ?", (download_id,))
if row is None or row["status"] != "pending":
continue # annulé/pausé entre-temps
# Tâche dédiée : annuler un téléchargement ne doit pas tuer le worker
task = asyncio.create_task(self._download(download_id))
self._tasks[download_id] = task
self._pause_events[download_id] = asyncio.Event()
self._pause_events[download_id].set()
try:
await task
except asyncio.CancelledError:
if asyncio.current_task().cancelling() > 0:
raise # le worker lui-même s'arrête (stop())
finally:
self._tasks.pop(download_id, None)
self._pause_events.pop(download_id, None)
self._progress.pop(download_id, None)
async def _download(self, download_id: int) -> None:
"""Dispatche HTTP/HLS ; gestion d'erreurs centralisée ici."""
row = await self._get_row(download_id)
try:
if ".m3u8" in row["video_url"]:
await self._download_hls(download_id, row)
else:
await self._download_http(download_id, row)
except asyncio.CancelledError:
logger.info("Téléchargement annulé : %s", row["file_path"])
raise
except (httpx.HTTPError, OSError) as exc:
# Échec réseau/disque : journalisé, statut 'failed' visible dans l'UI
logger.error("Échec du téléchargement de %s : %s", row["file_path"], exc)
part = self._part_path(row["file_path"])
downloaded = part.stat().st_size if part.exists() else 0
await self._fail(download_id, exc, downloaded)
async def _download_http(self, download_id: int, row: Any) -> None:
"""Téléchargement HTTP direct avec reprise via Range."""
video_url, file_path = row["video_url"], row["file_path"]
target = get_settings().download_dir / file_path
target.parent.mkdir(parents=True, exist_ok=True)
part = self._part_path(file_path)
downloaded = part.stat().st_size if part.exists() else 0
headers: dict[str, str] = {}
if row["page_url"]:
headers["Referer"] = row["page_url"]
if downloaded:
headers["Range"] = f"bytes={downloaded}-"
logger.info("Reprise de %s à %d octets", file_path, downloaded)
await self._set_status(download_id, "downloading")
await self._emit(download_id)
started = time.monotonic()
last_emit = 0.0
async with self._client.stream("GET", video_url, headers=headers) as response:
if response.status_code == 416: # plage invalide → déjà complet
part.rename(target)
await self._finish(download_id, downloaded)
return
response.raise_for_status()
if downloaded and response.status_code != 206:
downloaded = 0 # serveur sans support Range → on repart de zéro
logger.warning("Pas de reprise possible pour %s", file_path)
total = int(response.headers.get("content-length") or 0) + downloaded or None
await db.execute(
"UPDATE downloads SET total_bytes = ? WHERE id = ?", (total, download_id)
)
mode = "ab" if downloaded else "wb"
with part.open(mode) as fh:
async for chunk in response.aiter_bytes(1 << 16):
event = self._pause_events.get(download_id)
if event is not None:
await event.wait() # pause coopérative
fh.write(chunk)
downloaded += len(chunk)
now = time.monotonic()
if now - last_emit >= 1.0:
last_emit = now
await self._report_progress(download_id, downloaded, total, started)
part.rename(target)
await self._finish(download_id, downloaded)
async def _download_hls(self, download_id: int, row: Any) -> None:
"""Télécharge un flux HLS (.m3u8) via ffmpeg (remux en mp4).
Pause via SIGSTOP/SIGCONT du processus, annulation via kill.
La progression est estimée depuis la durée totale de la playlist.
"""
video_url, file_path = row["video_url"], row["file_path"]
target = get_settings().download_dir / file_path
target.parent.mkdir(parents=True, exist_ok=True)
part = self._part_path(file_path)
part.unlink(missing_ok=True) # pas de reprise partielle en HLS
ffmpeg_headers = f"User-Agent: {get_settings().user_agent}\r\n"
if row["page_url"]:
ffmpeg_headers += f"Referer: {row['page_url']}\r\n"
ffmpeg_headers += "Accept-Language: fr-FR,fr;q=0.9,en;q=0.8\r\n"
total_seconds = await self._hls_duration(video_url, row["page_url"])
await self._set_status(download_id, "downloading")
await self._emit(download_id)
started = time.monotonic()
process = await asyncio.create_subprocess_exec(
"ffmpeg", "-y", "-nostdin", "-v", "error", "-nostats", "-progress", "pipe:2",
"-headers", ffmpeg_headers,
"-i", video_url,
"-c", "copy", "-bsf:a", "aac_adtstoasc", "-f", "mp4",
str(part),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE,
)
self._hls_processes[download_id] = process
try:
assert process.stderr is not None
async for raw_line in process.stderr:
event = self._pause_events.get(download_id)
if event is not None and not event.is_set():
process.send_signal(signal.SIGSTOP)
await event.wait()
process.send_signal(signal.SIGCONT)
line = raw_line.decode(errors="replace")
if match := _FFMPEG_TIME_RE.search(line):
elapsed_video = _parse_ffmpeg_time(match.group(1))
size = part.stat().st_size if part.exists() else 0
total_est = None
if total_seconds and elapsed_video > 0:
total_est = int(size / elapsed_video * total_seconds)
await self._report_progress(download_id, size, total_est, started)
return_code = await process.wait()
finally:
self._hls_processes.pop(download_id, None)
if return_code != 0:
part.unlink(missing_ok=True)
raise OSError(f"ffmpeg a échoué (code {return_code}) sur le flux HLS")
size = part.stat().st_size
part.rename(target)
await self._finish(download_id, size)
async def _hls_duration(self, playlist_url: str, referer: str | None) -> float | None:
"""Durée totale d'une playlist HLS (somme des EXTINF de la variante max)."""
headers = {"Referer": referer} if referer else {}
try:
response = await self._client.get(playlist_url, headers=headers)
response.raise_for_status()
body = response.text
# Playlist maître → on suit la variante de plus haut débit
variant = _best_variant(body, playlist_url)
if variant and variant != playlist_url:
response = await self._client.get(variant, headers=headers)
response.raise_for_status()
body = response.text
durations = [float(m) for m in _EXTINF_RE.findall(body)]
return sum(durations) if durations else None
except (httpx.HTTPError, ValueError) as exc:
logger.warning("Durée HLS indéterminée pour %s : %s", playlist_url, exc)
return None
async def _report_progress(
self, download_id: int, downloaded: int, total: int | None, started: float
) -> None:
elapsed = time.monotonic() - started
self._progress[download_id] = {
"downloaded_bytes": downloaded,
"total_bytes": total,
"speed_bps": int(downloaded / elapsed) if elapsed > 0 else 0,
}
await self._emit(download_id)
async def _fail(self, download_id: int, exc: Exception, downloaded: int = 0) -> None:
await db.execute(
"UPDATE downloads SET status = 'failed', error = ?, "
"downloaded_bytes = ?, updated_at = datetime('now') WHERE id = ?",
(str(exc)[:500], downloaded, download_id),
)
await self._emit(download_id)
async def _finish(self, download_id: int, size: int) -> None:
await db.execute(
"UPDATE downloads SET status = 'done', total_bytes = ?, downloaded_bytes = ?, "
"updated_at = datetime('now') WHERE id = ?",
(size, size, download_id),
)
await self._emit(download_id)
logger.info("Téléchargement terminé (id=%s, %d octets)", download_id, size)
# L'historique a changé : les recommandations « Pour toi » se recalculent
from app.services.discover import discover
discover.invalidate_for_you()
# ------------------------------------------------------------ helpers
async def _get_row(self, download_id: int) -> Any:
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
if row is None:
raise KeyError(f"Téléchargement introuvable : {download_id}")
return row
async def _set_status(self, download_id: int, status: str) -> None:
await db.execute(
"UPDATE downloads SET status = ?, updated_at = datetime('now') WHERE id = ?",
(status, download_id),
)
@staticmethod
def _part_path(file_path: str | None) -> Path:
name = file_path or "video"
return get_settings().download_dir / (name + ".part")
@staticmethod
def _guess_extension(url: str) -> str:
match = re.search(r"\.(mp4|mkv|webm|avi|m3u8)(?:\?|$)", url)
ext = match.group(1) if match else "mp4"
return ".mp4" if ext == "m3u8" else f".{ext}"
def _to_dict(self, row: Any, duplicate: bool = False) -> dict:
data = dict(row)
live = self._progress.get(data["id"], {})
downloaded = live.get("downloaded_bytes", data["downloaded_bytes"])
total = live.get("total_bytes", data["total_bytes"])
speed = live.get("speed_bps", 0)
percent = round(downloaded / total * 100, 1) if total else None
eta = int((total - downloaded) / speed) if total and speed else None
data.update(
downloaded_bytes=downloaded,
total_bytes=total,
percent=percent,
speed_bps=speed,
eta_seconds=eta,
status_label=_STATUS_LABELS.get(data["status"], data["status"]),
duplicate=duplicate,
)
return data
download_manager = DownloadManager()