v0.1.0 — Réécriture complète de OhmStreaming
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
This commit is contained in:
@@ -0,0 +1,352 @@
|
||||
"""Découverte : nouveautés des sources, incontournables et recommandations.
|
||||
|
||||
Trois sections :
|
||||
- **latest** — « récemment ajoutés » scrapés sur chaque source activée (cliquables
|
||||
directement vers la fiche) ;
|
||||
- **must_watch** — titres les plus populaires du catalogue Kitsu (tous temps) ;
|
||||
- **for_you** — recommandations par genres : les genres des titres téléchargés
|
||||
(serveur), des favoris (par utilisateur) et des séries téléchargées sur Sonarr
|
||||
sont agrégés, puis Kitsu est interrogé sur ces catégories en excluant le
|
||||
déjà-possédé.
|
||||
|
||||
Toutes les sources externes sont optionnelles : un échec (réseau, scraping, API)
|
||||
laisse la section vide et n'est jamais remonté au caller (dégradation gracieuse).
|
||||
Un cache mémoire TTL évite de re-scaper à chaque chargement de page.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import unicodedata
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import db
|
||||
from app.scrapers.base import (
|
||||
ScrapeError,
|
||||
SourceScraper,
|
||||
all_sources,
|
||||
import_all_scrapers,
|
||||
)
|
||||
from app.services.kitsu import KitsuService, normalize_title
|
||||
from app.services.settings import is_source_enabled
|
||||
from app.services.sonarr import sonarr
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
import_all_scrapers()
|
||||
|
||||
# Bornes de l'algorithme
|
||||
_MAX_HISTORY_TITLES = 12 # titres récents analysés (téléchargements + favoris)
|
||||
_MAX_GENRES = 4 # genres retenus pour la requête Kitsu
|
||||
_KITSU_PAGE_MAX = 20 # limite dure de l'API Kitsu (page[limit] > 20 → 400)
|
||||
_ENRICH_CONCURRENCY = 6 # enrichissements Kitsu parallèles max (nouveautés)
|
||||
|
||||
_LATEST_TTL_SECONDS = 600 # nouveautés : re-scrape au bout de 10 min
|
||||
_MUST_WATCH_TTL_SECONDS = 21600 # incontournables : quasi statique, 6 h
|
||||
_FOR_YOU_TTL_SECONDS = 3600 # recommandations : 1 h (l'historique évolue lentement)
|
||||
|
||||
|
||||
class _TTLCache:
|
||||
"""Cache mémoire minimal avec expiration (mono-processus, suffisant ici)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._data: dict[str, tuple[float, object]] = {}
|
||||
|
||||
def get(self, key: str) -> object | None:
|
||||
entry = self._data.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
expires_at, value = entry
|
||||
if expires_at <= time.monotonic():
|
||||
del self._data[key]
|
||||
return None
|
||||
return value
|
||||
|
||||
def set(self, key: str, value: object, ttl_seconds: float) -> None:
|
||||
self._data[key] = (time.monotonic() + ttl_seconds, value)
|
||||
|
||||
def clear(self, prefix: str = "") -> None:
|
||||
"""Invalide les clés commençant par prefix (vide = tout le cache)."""
|
||||
for key in [k for k in self._data if k.startswith(prefix)]:
|
||||
del self._data[key]
|
||||
|
||||
|
||||
def category_slug(name: str) -> str:
|
||||
"""Nom de genre → slug de catégorie Kitsu (« Slice of Life » → « slice-of-life »)."""
|
||||
decomposed = unicodedata.normalize("NFKD", name)
|
||||
ascii_only = "".join(char for char in decomposed if not unicodedata.combining(char))
|
||||
return re.sub(r"[^a-z0-9]+", "-", ascii_only.casefold()).strip("-")
|
||||
|
||||
|
||||
class DiscoverService:
|
||||
"""Agrégation des trois sections de découverte, avec cache mémoire."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._cache = _TTLCache()
|
||||
self._kitsu = KitsuService()
|
||||
|
||||
# ------------------------------------------------------------ nouveautés
|
||||
|
||||
async def latest(self, limit: int = 24) -> list[dict]:
|
||||
"""Nouveautés toutes sources confondues, triées par date de sortie réelle.
|
||||
|
||||
Les « récemment ajoutés » de chaque source sont fusionnés (doublons retirés),
|
||||
enrichis via Kitsu (date de début, statut de diffusion) puis triés du plus
|
||||
récent au plus ancien — ce qui sort / vient de sortir en premier.
|
||||
"""
|
||||
cached = self._cache.get(f"latest:{limit}")
|
||||
if cached is not None:
|
||||
return cached # type: ignore[return-value]
|
||||
|
||||
sources = [s for s in all_sources() if await is_source_enabled(s.name)]
|
||||
outcomes = await asyncio.gather(*(self._latest_of(source) for source in sources))
|
||||
items = [item for outcome in outcomes for item in (outcome or [])]
|
||||
|
||||
merged: dict[str, dict] = {}
|
||||
for item in items:
|
||||
key = normalize_title(item["title"]).casefold()
|
||||
existing = merged.get(key)
|
||||
if existing is None or (not existing.get("image_url") and item.get("image_url")):
|
||||
merged[key] = item
|
||||
semaphore = asyncio.Semaphore(_ENRICH_CONCURRENCY)
|
||||
|
||||
async def bounded(item: dict) -> dict:
|
||||
async with semaphore:
|
||||
return await self._with_release_info(item)
|
||||
|
||||
enriched = await asyncio.gather(*(bounded(item) for item in merged.values()))
|
||||
result = sorted(enriched, key=lambda it: it.get("start_date") or "", reverse=True)
|
||||
result = result[:limit]
|
||||
self._cache.set(f"latest:{limit}", result, _LATEST_TTL_SECONDS)
|
||||
return result
|
||||
|
||||
async def _latest_of(self, source: SourceScraper) -> list[dict] | None:
|
||||
"""Items latest() d'une source, aplatis avec les infos de source ([] si KO)."""
|
||||
try:
|
||||
results = await source.latest()
|
||||
except ScrapeError as exc:
|
||||
logger.warning("Nouveautés indisponibles pour %s : %s", source.name, exc)
|
||||
return None
|
||||
return [
|
||||
{**dataclasses.asdict(r), "source": source.name, "label": source.label}
|
||||
for r in results
|
||||
]
|
||||
|
||||
async def _with_release_info(self, item: dict) -> dict:
|
||||
"""Complète un item de nouveauté avec sa date de sortie Kitsu (None si absent)."""
|
||||
item.setdefault("start_date", None)
|
||||
item.setdefault("status", None)
|
||||
item.setdefault("rating", None)
|
||||
match = await self._kitsu_match_for_title(item["title"])
|
||||
if match is None:
|
||||
return item
|
||||
attrs = match.get("attributes", {})
|
||||
item["start_date"] = attrs.get("startDate")
|
||||
item["status"] = attrs.get("status")
|
||||
item["rating"] = KitsuService._to_rating_10(attrs.get("averageRating"))
|
||||
return item
|
||||
|
||||
# --------------------------------------------------------- incontournables
|
||||
|
||||
async def must_watch(self, limit: int = _KITSU_PAGE_MAX) -> list[dict]:
|
||||
"""Titres les plus populaires du catalogue Kitsu (tous temps)."""
|
||||
limit = min(limit, _KITSU_PAGE_MAX)
|
||||
key = f"must_watch:{limit}"
|
||||
cached = self._cache.get(key)
|
||||
if cached is not None:
|
||||
return cached # type: ignore[return-value]
|
||||
items = await self._kitsu_anime({"sort": "-userCount", "page[limit]": limit})
|
||||
self._cache.set(key, items, _MUST_WATCH_TTL_SECONDS)
|
||||
return items
|
||||
|
||||
# ------------------------------------------------------------ pour toi
|
||||
|
||||
async def for_you(self, user_id: int, limit: int = _KITSU_PAGE_MAX) -> dict:
|
||||
"""Recommandations par genres, à partir de l'historique de l'utilisateur.
|
||||
|
||||
Genres = téléchargements du serveur (titres → Kitsu) + favoris de l'utilisateur
|
||||
(genres du payload) + séries téléchargées sur Sonarr (genres fournis par
|
||||
Sonarr). On exclut les titres déjà possédés (local et Sonarr).
|
||||
"""
|
||||
limit = min(limit, _KITSU_PAGE_MAX)
|
||||
cache_key = f"for_you:{user_id}:{limit}"
|
||||
cached = self._cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached # type: ignore[return-value]
|
||||
|
||||
owned, favorite_genres = await self._owned(user_id)
|
||||
genre_counts = await self._genres_from_downloads(owned)
|
||||
for genre, count in favorite_genres.items():
|
||||
genre_counts[genre] = genre_counts.get(genre, 0) + count
|
||||
sonarr_owned, sonarr_genres = await sonarr.profile()
|
||||
for genre, count in sonarr_genres.items():
|
||||
genre_counts[genre] = genre_counts.get(genre, 0) + count
|
||||
owned |= sonarr_owned
|
||||
if not genre_counts:
|
||||
result: dict = {"based_on": [], "items": []}
|
||||
self._cache.set(cache_key, result, _FOR_YOU_TTL_SECONDS)
|
||||
return result
|
||||
|
||||
top_genres = sorted(genre_counts, key=genre_counts.get, reverse=True)[:_MAX_GENRES]
|
||||
slugs = [category_slug(genre) for genre in top_genres]
|
||||
items = await self._kitsu_anime(
|
||||
{
|
||||
"filter[categories]": ",".join(slugs),
|
||||
"sort": "-userCount",
|
||||
"page[limit]": limit, # le déjà-possédé est filtré après
|
||||
}
|
||||
)
|
||||
kept = [item for item in items if item["title"] and item["title"].casefold() not in owned]
|
||||
result = {"based_on": top_genres, "items": kept}
|
||||
self._cache.set(cache_key, result, _FOR_YOU_TTL_SECONDS)
|
||||
return result
|
||||
|
||||
def invalidate_for_you(self) -> None:
|
||||
"""Recommandations recalculées au prochain appel (réglages Sonarr modifiés)."""
|
||||
self._cache.clear("for_you:")
|
||||
|
||||
async def _owned(self, user_id: int) -> tuple[set[str], dict[str, int]]:
|
||||
"""Titres possédés (normalisés) + genres directement connus via les favoris."""
|
||||
rows = await db.fetchall(
|
||||
"SELECT DISTINCT title FROM downloads ORDER BY created_at DESC LIMIT ?",
|
||||
(_MAX_HISTORY_TITLES,),
|
||||
)
|
||||
fav_rows = await db.fetchall(
|
||||
"SELECT payload FROM favorites WHERE user_id = ? ORDER BY created_at DESC LIMIT ?",
|
||||
(user_id, _MAX_HISTORY_TITLES),
|
||||
)
|
||||
owned = {normalize_title(row["title"]).casefold() for row in rows}
|
||||
owned.discard("")
|
||||
genre_counts: dict[str, int] = {}
|
||||
for row in fav_rows:
|
||||
try:
|
||||
payload = json.loads(row["payload"]) if row["payload"] else {}
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
for genre in payload.get("genres") or []:
|
||||
if isinstance(genre, str) and genre.strip():
|
||||
genre_counts[genre.strip()] = genre_counts.get(genre.strip(), 0) + 1
|
||||
return owned, genre_counts
|
||||
|
||||
async def _genres_from_downloads(self, titles: set[str]) -> dict[str, int]:
|
||||
"""Genres Kitsu des titres téléchargés (cache DB puis recherche)."""
|
||||
semaphore = asyncio.Semaphore(_MAX_HISTORY_TITLES)
|
||||
|
||||
async def genres_of(title: str) -> list[str]:
|
||||
async with semaphore:
|
||||
return await self._kitsu_genres_for_title(title)
|
||||
|
||||
outcomes = await asyncio.gather(*(genres_of(t) for t in list(titles)[:_MAX_HISTORY_TITLES]))
|
||||
counts: dict[str, int] = {}
|
||||
for genres in outcomes:
|
||||
for genre in genres:
|
||||
counts[genre] = counts.get(genre, 0) + 1
|
||||
return counts
|
||||
|
||||
async def _kitsu_match_for_title(self, title: str) -> dict | None:
|
||||
"""Match Kitsu d'un titre scrapé (cache DB 72 h via metadata_cache)."""
|
||||
query = normalize_title(title)
|
||||
if not query:
|
||||
return None
|
||||
cache_key = f"kitsu:anime:{query.casefold()}"
|
||||
match = await self._kitsu.get_cached(cache_key)
|
||||
if match is None:
|
||||
match = await self._kitsu.search_anime(title)
|
||||
if match is not None:
|
||||
await self._kitsu.set_cached(cache_key, match)
|
||||
return match
|
||||
|
||||
async def _kitsu_genres_for_title(self, title: str) -> list[str]:
|
||||
"""Genres Kitsu d'un titre scrapé (cache DB 72 h via metadata_cache).
|
||||
|
||||
La recherche Kitsu ne renvoie plus les genres (`include=genres` vide) :
|
||||
on complète avec l'endpoint /anime/<id>/categories.
|
||||
"""
|
||||
match = await self._kitsu_match_for_title(title)
|
||||
if match is None:
|
||||
return []
|
||||
genres = [g for g in match.get("genres", []) if isinstance(g, str)]
|
||||
if not genres:
|
||||
genres = await self._kitsu_categories(match.get("id"))
|
||||
match["genres"] = genres
|
||||
query = normalize_title(title)
|
||||
await self._kitsu.set_cached( # refresh avec les genres
|
||||
f"kitsu:anime:{query.casefold()}", match
|
||||
)
|
||||
return genres
|
||||
|
||||
# -------------------------------------------------------------- Kitsu
|
||||
|
||||
async def _kitsu_anime(self, params: dict) -> list[dict]:
|
||||
"""Requête générique liste Kitsu → items normalisés ([] si échec)."""
|
||||
settings = get_settings()
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=settings.http_timeout,
|
||||
headers={
|
||||
"User-Agent": settings.user_agent,
|
||||
"Accept": "application/vnd.api+json",
|
||||
},
|
||||
) as client:
|
||||
response = await client.get(f"{settings.kitsu_base_url}/anime", params=params)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
logger.warning("Liste Kitsu échouée (%s) : %s", params, exc)
|
||||
return []
|
||||
return [
|
||||
self._normalize_anime(item)
|
||||
for item in payload.get("data", [])
|
||||
if item.get("type") == "anime"
|
||||
]
|
||||
|
||||
async def _kitsu_categories(self, anime_id: object) -> list[str]:
|
||||
"""Titres des catégories Kitsu d'un anime ([] si échec)."""
|
||||
if not anime_id:
|
||||
return []
|
||||
settings = get_settings()
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=settings.http_timeout,
|
||||
headers={
|
||||
"User-Agent": settings.user_agent,
|
||||
"Accept": "application/vnd.api+json",
|
||||
},
|
||||
) as client:
|
||||
response = await client.get(
|
||||
f"{settings.kitsu_base_url}/anime/{anime_id}/categories",
|
||||
params={"page[limit]": _KITSU_PAGE_MAX},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
logger.warning("Catégories Kitsu échouées (anime %s) : %s", anime_id, exc)
|
||||
return []
|
||||
return [
|
||||
attrs["title"]
|
||||
for item in payload.get("data", [])
|
||||
if isinstance(attrs := item.get("attributes", {}), dict) and attrs.get("title")
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _normalize_anime(item: dict) -> dict:
|
||||
attrs = item.get("attributes", {})
|
||||
titles = attrs.get("titles") or {}
|
||||
poster = attrs.get("posterImage") or {}
|
||||
return {
|
||||
"kitsu_id": item.get("id"),
|
||||
"title": attrs.get("canonicalTitle") or titles.get("en_jp"),
|
||||
"image_url": poster.get("large") or poster.get("medium") or poster.get("tiny"),
|
||||
"rating": KitsuService._to_rating_10(attrs.get("averageRating")),
|
||||
"year": KitsuService._extract_year(attrs.get("startDate")),
|
||||
"subtype": attrs.get("subtype"),
|
||||
"user_count": attrs.get("userCount"),
|
||||
}
|
||||
|
||||
|
||||
discover = DiscoverService()
|
||||
@@ -0,0 +1,499 @@
|
||||
"""Gestionnaire de téléchargements : file asyncio, parallélisme limité,
|
||||
pause/reprise (Range HTTP), anti-doublons, persistance, progression temps réel.
|
||||
|
||||
Les statuts : pending → downloading → done | failed | cancelled
|
||||
↕ paused
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import re
|
||||
import signal
|
||||
import time
|
||||
import unicodedata
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ACTIVE_STATUSES = ("pending", "downloading", "paused")
|
||||
|
||||
_FFMPEG_TIME_RE = re.compile(r"time=(\d+:\d+:\d+(?:\.\d+)?)")
|
||||
_EXTINF_RE = re.compile(r"#EXTINF:([\d.]+)")
|
||||
_BANDWIDTH_RE = re.compile(r"#EXT-X-STREAM-INF:[^\n]*BANDWIDTH=(\d+)[^\n]*\n(\S+)")
|
||||
|
||||
|
||||
def _parse_ffmpeg_time(value: str) -> float:
|
||||
parts = value.split(":")
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
||||
|
||||
|
||||
def _best_variant(master_body: str, base_url: str) -> str | None:
|
||||
"""URL de la variante au plus haut débit d'une playlist maître HLS."""
|
||||
variants = [
|
||||
(int(bw), urljoin(base_url, uri)) for bw, uri in _BANDWIDTH_RE.findall(master_body)
|
||||
]
|
||||
return max(variants)[1] if variants else None
|
||||
|
||||
_STATUS_LABELS = {
|
||||
"pending": "en attente",
|
||||
"downloading": "en cours",
|
||||
"paused": "en pause",
|
||||
"done": "terminé",
|
||||
"failed": "échec",
|
||||
"cancelled": "annulé",
|
||||
}
|
||||
|
||||
|
||||
def sanitize_filename(name: str) -> str:
|
||||
"""Nettoie un nom de fichier : caractères interdits retirés, anti-traversée."""
|
||||
name = unicodedata.normalize("NFKC", name)
|
||||
name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', " ", name)
|
||||
name = re.sub(r"\s+", " ", name).strip(" .")
|
||||
if not name or name in (".", ".."):
|
||||
name = "video"
|
||||
return name[:150]
|
||||
|
||||
|
||||
class DownloadManager:
|
||||
"""File d'attente de téléchargements, injectée dans les routes via app.state."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._queue: asyncio.Queue[int] # créée dans start() (affinité avec la boucle)
|
||||
self._tasks: dict[int, asyncio.Task] = {} # download_id → tâche asyncio
|
||||
self._pause_events: dict[int, asyncio.Event] = {} # set = peut tourner
|
||||
self._progress: dict[int, dict[str, Any]] = {} # progression temps réel en mémoire
|
||||
self._listeners: list[asyncio.Queue] = []
|
||||
self._workers: list[asyncio.Task] = []
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
self._hls_processes: dict[int, asyncio.subprocess.Process] = {}
|
||||
|
||||
# ------------------------------------------------------------ cycle de vie
|
||||
|
||||
async def start(self) -> None:
|
||||
self._queue = asyncio.Queue()
|
||||
settings = get_settings()
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(30.0, read=300.0),
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"User-Agent": settings.user_agent,
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
||||
},
|
||||
)
|
||||
# Restaure les téléchargements interrompus (crash/arrêt) en 'pending'
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = 'pending', updated_at = datetime('now') "
|
||||
"WHERE status = 'downloading'"
|
||||
)
|
||||
await self._scan_download_dir()
|
||||
for _ in range(settings.max_parallel_downloads):
|
||||
self._workers.append(asyncio.create_task(self._worker()))
|
||||
logger.info("DownloadManager démarré (%d workers)", settings.max_parallel_downloads)
|
||||
|
||||
async def stop(self) -> None:
|
||||
for worker in self._workers:
|
||||
worker.cancel()
|
||||
for task in self._tasks.values():
|
||||
task.cancel()
|
||||
for proc in self._hls_processes.values():
|
||||
if proc.returncode is None:
|
||||
proc.kill()
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._workers.clear()
|
||||
self._tasks.clear()
|
||||
|
||||
async def _scan_download_dir(self) -> None:
|
||||
"""Restaure en 'done' les fichiers présents sur disque mais inconnus de la DB."""
|
||||
download_dir = get_settings().download_dir
|
||||
rows = await db.fetchall("SELECT file_path FROM downloads WHERE file_path IS NOT NULL")
|
||||
known = {row["file_path"] for row in rows}
|
||||
for path in download_dir.iterdir():
|
||||
if path.is_file() and path.suffix != ".part" and path.name not in known:
|
||||
size = path.stat().st_size
|
||||
await db.execute(
|
||||
"INSERT INTO downloads "
|
||||
"(source_key, video_url, page_url, title, file_path, status, "
|
||||
" total_bytes, downloaded_bytes) "
|
||||
"VALUES (?, ?, ?, ?, ?, 'done', ?, ?)",
|
||||
(
|
||||
f"file:{path.name}",
|
||||
"",
|
||||
"",
|
||||
path.stem,
|
||||
path.name,
|
||||
size,
|
||||
size,
|
||||
),
|
||||
)
|
||||
logger.info("Fichier restauré depuis le disque : %s", path.name)
|
||||
|
||||
# ------------------------------------------------------------ API publique
|
||||
|
||||
async def enqueue(self, video_url: str, page_url: str, title: str) -> dict:
|
||||
"""Ajoute un téléchargement ; retourne la tâche existante si doublon actif."""
|
||||
source_key = video_url
|
||||
existing = await db.fetchone(
|
||||
f"SELECT * FROM downloads WHERE source_key = ? AND status IN "
|
||||
f"({','.join('?' * len(ACTIVE_STATUSES))})",
|
||||
(source_key, *ACTIVE_STATUSES),
|
||||
)
|
||||
if existing:
|
||||
logger.info("Anti-doublon : %s déjà en file (id=%s)", title, existing["id"])
|
||||
return self._to_dict(existing, duplicate=True)
|
||||
|
||||
filename = sanitize_filename(title) + self._guess_extension(video_url)
|
||||
cursor = await db.execute(
|
||||
"INSERT INTO downloads (source_key, video_url, page_url, title, file_path) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
(source_key, video_url, page_url, title, filename),
|
||||
)
|
||||
download_id = cursor.lastrowid
|
||||
await self._queue.put(download_id)
|
||||
await self._emit(download_id)
|
||||
logger.info("Téléchargement ajouté : %s (id=%s)", title, download_id)
|
||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
||||
return self._to_dict(row)
|
||||
|
||||
async def pause(self, download_id: int) -> bool:
|
||||
event = self._pause_events.get(download_id)
|
||||
if event:
|
||||
event.clear()
|
||||
await self._set_status(download_id, "paused")
|
||||
return True
|
||||
|
||||
async def resume(self, download_id: int) -> bool:
|
||||
row = await self._get_row(download_id)
|
||||
if row["status"] != "paused":
|
||||
return False
|
||||
await self._set_status(download_id, "pending")
|
||||
await self._queue.put(download_id)
|
||||
return True
|
||||
|
||||
async def retry(self, download_id: int) -> bool:
|
||||
row = await self._get_row(download_id)
|
||||
if row["status"] not in ("failed", "cancelled"):
|
||||
return False
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = 'pending', error = NULL, downloaded_bytes = 0, "
|
||||
"updated_at = datetime('now') WHERE id = ?",
|
||||
(download_id,),
|
||||
)
|
||||
part = self._part_path(row["file_path"])
|
||||
part.unlink(missing_ok=True)
|
||||
await self._queue.put(download_id)
|
||||
await self._emit(download_id)
|
||||
return True
|
||||
|
||||
async def cancel(self, download_id: int) -> bool:
|
||||
task = self._tasks.get(download_id)
|
||||
if task:
|
||||
task.cancel()
|
||||
proc = self._hls_processes.get(download_id)
|
||||
if proc and proc.returncode is None:
|
||||
proc.kill()
|
||||
await self._set_status(download_id, "cancelled")
|
||||
row = await self._get_row(download_id)
|
||||
self._part_path(row["file_path"]).unlink(missing_ok=True)
|
||||
await self._emit(download_id)
|
||||
return True
|
||||
|
||||
async def cancel_all(self) -> int:
|
||||
rows = await db.fetchall(
|
||||
f"SELECT id FROM downloads WHERE status IN ({','.join('?' * len(ACTIVE_STATUSES))})",
|
||||
ACTIVE_STATUSES,
|
||||
)
|
||||
for row in rows:
|
||||
await self.cancel(row["id"])
|
||||
return len(rows)
|
||||
|
||||
async def clear_finished(self) -> int:
|
||||
"""Supprime de la file les tâches terminées/échouées/annulées (fichiers gardés)."""
|
||||
cursor = await db.execute(
|
||||
"DELETE FROM downloads WHERE status IN ('done', 'failed', 'cancelled')"
|
||||
)
|
||||
return cursor.rowcount or 0
|
||||
|
||||
async def list_all(self, limit: int = 200) -> list[dict]:
|
||||
rows = await db.fetchall(
|
||||
"SELECT * FROM downloads ORDER BY "
|
||||
"CASE status WHEN 'downloading' THEN 0 WHEN 'pending' THEN 1 WHEN 'paused' THEN 2 "
|
||||
"ELSE 3 END, updated_at DESC LIMIT ?",
|
||||
(limit,),
|
||||
)
|
||||
return [self._to_dict(row) for row in rows]
|
||||
|
||||
async def get(self, download_id: int) -> dict | None:
|
||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
||||
return self._to_dict(row) if row else None
|
||||
|
||||
# ------------------------------------------------------------ événements SSE
|
||||
|
||||
async def subscribe(self) -> AsyncIterator[dict]:
|
||||
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
|
||||
self._listeners.append(queue)
|
||||
try:
|
||||
while True:
|
||||
yield await queue.get()
|
||||
finally:
|
||||
self._listeners.remove(queue)
|
||||
|
||||
async def _emit(self, download_id: int) -> None:
|
||||
data = await self.get(download_id)
|
||||
if data is None:
|
||||
return
|
||||
for queue in self._listeners:
|
||||
with contextlib.suppress(asyncio.QueueFull):
|
||||
queue.put_nowait(data)
|
||||
|
||||
# ------------------------------------------------------------ worker interne
|
||||
|
||||
async def _worker(self) -> None:
|
||||
while True:
|
||||
download_id = await self._queue.get()
|
||||
row = await db.fetchone("SELECT status FROM downloads WHERE id = ?", (download_id,))
|
||||
if row is None or row["status"] != "pending":
|
||||
continue # annulé/pausé entre-temps
|
||||
# Tâche dédiée : annuler un téléchargement ne doit pas tuer le worker
|
||||
task = asyncio.create_task(self._download(download_id))
|
||||
self._tasks[download_id] = task
|
||||
self._pause_events[download_id] = asyncio.Event()
|
||||
self._pause_events[download_id].set()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
if asyncio.current_task().cancelling() > 0:
|
||||
raise # le worker lui-même s'arrête (stop())
|
||||
finally:
|
||||
self._tasks.pop(download_id, None)
|
||||
self._pause_events.pop(download_id, None)
|
||||
self._progress.pop(download_id, None)
|
||||
|
||||
async def _download(self, download_id: int) -> None:
|
||||
"""Dispatche HTTP/HLS ; gestion d'erreurs centralisée ici."""
|
||||
row = await self._get_row(download_id)
|
||||
try:
|
||||
if ".m3u8" in row["video_url"]:
|
||||
await self._download_hls(download_id, row)
|
||||
else:
|
||||
await self._download_http(download_id, row)
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Téléchargement annulé : %s", row["file_path"])
|
||||
raise
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
# Échec réseau/disque : journalisé, statut 'failed' visible dans l'UI
|
||||
logger.error("Échec du téléchargement de %s : %s", row["file_path"], exc)
|
||||
part = self._part_path(row["file_path"])
|
||||
downloaded = part.stat().st_size if part.exists() else 0
|
||||
await self._fail(download_id, exc, downloaded)
|
||||
|
||||
async def _download_http(self, download_id: int, row: Any) -> None:
|
||||
"""Téléchargement HTTP direct avec reprise via Range."""
|
||||
video_url, file_path = row["video_url"], row["file_path"]
|
||||
target = get_settings().download_dir / file_path
|
||||
part = self._part_path(file_path)
|
||||
downloaded = part.stat().st_size if part.exists() else 0
|
||||
|
||||
headers: dict[str, str] = {}
|
||||
if row["page_url"]:
|
||||
headers["Referer"] = row["page_url"]
|
||||
if downloaded:
|
||||
headers["Range"] = f"bytes={downloaded}-"
|
||||
logger.info("Reprise de %s à %d octets", file_path, downloaded)
|
||||
|
||||
await self._set_status(download_id, "downloading")
|
||||
await self._emit(download_id)
|
||||
started = time.monotonic()
|
||||
last_emit = 0.0
|
||||
|
||||
async with self._client.stream("GET", video_url, headers=headers) as response:
|
||||
if response.status_code == 416: # plage invalide → déjà complet
|
||||
part.rename(target)
|
||||
await self._finish(download_id, downloaded)
|
||||
return
|
||||
response.raise_for_status()
|
||||
if downloaded and response.status_code != 206:
|
||||
downloaded = 0 # serveur sans support Range → on repart de zéro
|
||||
logger.warning("Pas de reprise possible pour %s", file_path)
|
||||
total = int(response.headers.get("content-length") or 0) + downloaded or None
|
||||
await db.execute(
|
||||
"UPDATE downloads SET total_bytes = ? WHERE id = ?", (total, download_id)
|
||||
)
|
||||
|
||||
mode = "ab" if downloaded else "wb"
|
||||
with part.open(mode) as fh:
|
||||
async for chunk in response.aiter_bytes(1 << 16):
|
||||
event = self._pause_events.get(download_id)
|
||||
if event is not None:
|
||||
await event.wait() # pause coopérative
|
||||
fh.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
now = time.monotonic()
|
||||
if now - last_emit >= 1.0:
|
||||
last_emit = now
|
||||
await self._report_progress(download_id, downloaded, total, started)
|
||||
|
||||
part.rename(target)
|
||||
await self._finish(download_id, downloaded)
|
||||
|
||||
async def _download_hls(self, download_id: int, row: Any) -> None:
|
||||
"""Télécharge un flux HLS (.m3u8) via ffmpeg (remux en mp4).
|
||||
|
||||
Pause via SIGSTOP/SIGCONT du processus, annulation via kill.
|
||||
La progression est estimée depuis la durée totale de la playlist.
|
||||
"""
|
||||
video_url, file_path = row["video_url"], row["file_path"]
|
||||
target = get_settings().download_dir / file_path
|
||||
part = self._part_path(file_path)
|
||||
part.unlink(missing_ok=True) # pas de reprise partielle en HLS
|
||||
|
||||
ffmpeg_headers = f"User-Agent: {get_settings().user_agent}\r\n"
|
||||
if row["page_url"]:
|
||||
ffmpeg_headers += f"Referer: {row['page_url']}\r\n"
|
||||
ffmpeg_headers += "Accept-Language: fr-FR,fr;q=0.9,en;q=0.8\r\n"
|
||||
|
||||
total_seconds = await self._hls_duration(video_url, row["page_url"])
|
||||
|
||||
await self._set_status(download_id, "downloading")
|
||||
await self._emit(download_id)
|
||||
started = time.monotonic()
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
"ffmpeg", "-y", "-nostdin", "-v", "error", "-nostats", "-progress", "pipe:2",
|
||||
"-headers", ffmpeg_headers,
|
||||
"-i", video_url,
|
||||
"-c", "copy", "-bsf:a", "aac_adtstoasc", "-f", "mp4",
|
||||
str(part),
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
self._hls_processes[download_id] = process
|
||||
try:
|
||||
assert process.stderr is not None
|
||||
async for raw_line in process.stderr:
|
||||
event = self._pause_events.get(download_id)
|
||||
if event is not None and not event.is_set():
|
||||
process.send_signal(signal.SIGSTOP)
|
||||
await event.wait()
|
||||
process.send_signal(signal.SIGCONT)
|
||||
line = raw_line.decode(errors="replace")
|
||||
if match := _FFMPEG_TIME_RE.search(line):
|
||||
elapsed_video = _parse_ffmpeg_time(match.group(1))
|
||||
size = part.stat().st_size if part.exists() else 0
|
||||
total_est = None
|
||||
if total_seconds and elapsed_video > 0:
|
||||
total_est = int(size / elapsed_video * total_seconds)
|
||||
await self._report_progress(download_id, size, total_est, started)
|
||||
return_code = await process.wait()
|
||||
finally:
|
||||
self._hls_processes.pop(download_id, None)
|
||||
|
||||
if return_code != 0:
|
||||
part.unlink(missing_ok=True)
|
||||
raise OSError(f"ffmpeg a échoué (code {return_code}) sur le flux HLS")
|
||||
size = part.stat().st_size
|
||||
part.rename(target)
|
||||
await self._finish(download_id, size)
|
||||
|
||||
async def _hls_duration(self, playlist_url: str, referer: str | None) -> float | None:
|
||||
"""Durée totale d'une playlist HLS (somme des EXTINF de la variante max)."""
|
||||
headers = {"Referer": referer} if referer else {}
|
||||
try:
|
||||
response = await self._client.get(playlist_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
body = response.text
|
||||
# Playlist maître → on suit la variante de plus haut débit
|
||||
variant = _best_variant(body, playlist_url)
|
||||
if variant and variant != playlist_url:
|
||||
response = await self._client.get(variant, headers=headers)
|
||||
response.raise_for_status()
|
||||
body = response.text
|
||||
durations = [float(m) for m in _EXTINF_RE.findall(body)]
|
||||
return sum(durations) if durations else None
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
logger.warning("Durée HLS indéterminée pour %s : %s", playlist_url, exc)
|
||||
return None
|
||||
|
||||
async def _report_progress(
|
||||
self, download_id: int, downloaded: int, total: int | None, started: float
|
||||
) -> None:
|
||||
elapsed = time.monotonic() - started
|
||||
self._progress[download_id] = {
|
||||
"downloaded_bytes": downloaded,
|
||||
"total_bytes": total,
|
||||
"speed_bps": int(downloaded / elapsed) if elapsed > 0 else 0,
|
||||
}
|
||||
await self._emit(download_id)
|
||||
|
||||
async def _fail(self, download_id: int, exc: Exception, downloaded: int = 0) -> None:
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = 'failed', error = ?, "
|
||||
"downloaded_bytes = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
(str(exc)[:500], downloaded, download_id),
|
||||
)
|
||||
await self._emit(download_id)
|
||||
|
||||
async def _finish(self, download_id: int, size: int) -> None:
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = 'done', total_bytes = ?, downloaded_bytes = ?, "
|
||||
"updated_at = datetime('now') WHERE id = ?",
|
||||
(size, size, download_id),
|
||||
)
|
||||
await self._emit(download_id)
|
||||
logger.info("Téléchargement terminé (id=%s, %d octets)", download_id, size)
|
||||
|
||||
# ------------------------------------------------------------ helpers
|
||||
|
||||
async def _get_row(self, download_id: int) -> Any:
|
||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
||||
if row is None:
|
||||
raise KeyError(f"Téléchargement introuvable : {download_id}")
|
||||
return row
|
||||
|
||||
async def _set_status(self, download_id: int, status: str) -> None:
|
||||
await db.execute(
|
||||
"UPDATE downloads SET status = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
(status, download_id),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _part_path(file_path: str | None) -> Path:
|
||||
name = file_path or "video"
|
||||
return get_settings().download_dir / (name + ".part")
|
||||
|
||||
@staticmethod
|
||||
def _guess_extension(url: str) -> str:
|
||||
match = re.search(r"\.(mp4|mkv|webm|avi|m3u8)(?:\?|$)", url)
|
||||
ext = match.group(1) if match else "mp4"
|
||||
return ".mp4" if ext == "m3u8" else f".{ext}"
|
||||
|
||||
def _to_dict(self, row: Any, duplicate: bool = False) -> dict:
|
||||
data = dict(row)
|
||||
live = self._progress.get(data["id"], {})
|
||||
downloaded = live.get("downloaded_bytes", data["downloaded_bytes"])
|
||||
total = live.get("total_bytes", data["total_bytes"])
|
||||
speed = live.get("speed_bps", 0)
|
||||
percent = round(downloaded / total * 100, 1) if total else None
|
||||
eta = int((total - downloaded) / speed) if total and speed else None
|
||||
data.update(
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=total,
|
||||
percent=percent,
|
||||
speed_bps=speed,
|
||||
eta_seconds=eta,
|
||||
status_label=_STATUS_LABELS.get(data["status"], data["status"]),
|
||||
duplicate=duplicate,
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
download_manager = DownloadManager()
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Enrichissement de métadonnées via l'API Kitsu (https://kitsu.io/api/edge).
|
||||
|
||||
Fusion intelligente : seuls les champs manquants d'un TitleDetails sont complétés.
|
||||
Les échecs (réseau, cache indisponible) sont loggés en warning et jamais remontés
|
||||
au caller — la fiche d'origine est retournée inchangée (dégradation gracieuse).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import db
|
||||
from app.scrapers.base import TitleDetails
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_NOISE_WORDS_RE = re.compile(
|
||||
r"\b(?:VOSTFR\d*|VOST|VF[IV]?|TRUEFRENCH|FRENCH|MULTI|SUBFR?)\b", re.IGNORECASE
|
||||
)
|
||||
_TRAILING_SEASON_RE = re.compile(r"[\s\-–—:.]*\s*(?:saison|season)\s*\d+\s*$", re.IGNORECASE)
|
||||
_TRAILING_CODE_RE = re.compile(r"[\s\-–—:.]*\s*S\d+(?:E\d+)?\s*$", re.IGNORECASE)
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
"""Nettoie un titre de scraping avant recherche Kitsu (bruit, saison, tirets)."""
|
||||
cleaned = _NOISE_WORDS_RE.sub(" ", title)
|
||||
cleaned = _TRAILING_SEASON_RE.sub("", cleaned)
|
||||
cleaned = _TRAILING_CODE_RE.sub("", cleaned)
|
||||
cleaned = re.sub(r"\s*[-–—_]+\s*", " ", cleaned)
|
||||
cleaned = re.sub(r"\s+", " ", cleaned)
|
||||
return cleaned.strip(" -–—:.")
|
||||
|
||||
|
||||
class KitsuService:
|
||||
"""Recherche et cache des métadonnées anime depuis l'API Kitsu."""
|
||||
|
||||
async def search_anime(self, title: str) -> dict | None:
|
||||
"""Recherche le meilleur match Kitsu pour un titre (None si échec/rien trouvé)."""
|
||||
settings = get_settings()
|
||||
query = normalize_title(title)
|
||||
if not query:
|
||||
return None
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=settings.http_timeout,
|
||||
headers={
|
||||
"User-Agent": settings.user_agent,
|
||||
"Accept": "application/vnd.api+json",
|
||||
},
|
||||
) as client:
|
||||
response = await client.get(
|
||||
f"{settings.kitsu_base_url}/anime",
|
||||
params={"filter[text]": query, "page[limit]": 5, "include": "genres"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
logger.warning("Recherche Kitsu échouée pour %r : %s", title, exc)
|
||||
return None
|
||||
return self._pick_best_match(payload, query)
|
||||
|
||||
async def enrich(self, details: TitleDetails) -> TitleDetails:
|
||||
"""Complète les champs manquants de details via Kitsu (jamais episodes/url/source)."""
|
||||
query = normalize_title(details.title)
|
||||
cache_key = f"kitsu:anime:{query.casefold()}"
|
||||
|
||||
result = await self.get_cached(cache_key)
|
||||
from_cache = result is not None
|
||||
if result is None:
|
||||
result = await self.search_anime(details.title)
|
||||
if result is not None:
|
||||
await self.set_cached(cache_key, result)
|
||||
|
||||
if result is None:
|
||||
logger.info("Aucune métadonnée Kitsu pour %r — fiche inchangée", details.title)
|
||||
return details
|
||||
|
||||
attrs = result.get("attributes", {})
|
||||
if not details.synopsis:
|
||||
details.synopsis = attrs.get("synopsis")
|
||||
if not details.image_url:
|
||||
poster = attrs.get("posterImage") or {}
|
||||
details.image_url = poster.get("large") or poster.get("medium")
|
||||
if not details.banner_url:
|
||||
cover = attrs.get("coverImage") or {}
|
||||
details.banner_url = cover.get("large") or cover.get("original")
|
||||
if not details.genres:
|
||||
details.genres = list(result.get("genres", []))
|
||||
if details.rating is None:
|
||||
details.rating = self._to_rating_10(attrs.get("averageRating"))
|
||||
if details.year is None:
|
||||
details.year = self._extract_year(attrs.get("startDate"))
|
||||
if details.episode_count is None:
|
||||
details.episode_count = attrs.get("episodeCount")
|
||||
|
||||
logger.debug(
|
||||
"Fiche %r enrichie via Kitsu (cache=%s, id=%s)",
|
||||
details.title,
|
||||
from_cache,
|
||||
result.get("id"),
|
||||
)
|
||||
return details
|
||||
|
||||
async def get_cached(self, cache_key: str) -> dict | None:
|
||||
"""Retourne le payload en cache s'il existe et est frais (TTL), sinon None."""
|
||||
ttl_hours = get_settings().metadata_cache_ttl_hours
|
||||
try:
|
||||
row = await db.fetchone(
|
||||
"SELECT payload FROM metadata_cache "
|
||||
f"WHERE cache_key = ? AND fetched_at > datetime('now', '-{ttl_hours} hours')",
|
||||
(cache_key,),
|
||||
)
|
||||
except (RuntimeError, sqlite3.Error) as exc:
|
||||
logger.warning(
|
||||
"Cache métadonnées illisible (%s), continuation sans cache : %s", cache_key, exc
|
||||
)
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
try:
|
||||
return json.loads(row["payload"])
|
||||
except (TypeError, ValueError) as exc:
|
||||
logger.warning("Cache métadonnées corrompu pour %s, re-fetch : %s", cache_key, exc)
|
||||
return None
|
||||
|
||||
async def set_cached(self, cache_key: str, payload: dict) -> None:
|
||||
"""Écrit/upsert une entrée de cache (échec non bloquant, warning seulement)."""
|
||||
try:
|
||||
await db.execute(
|
||||
"INSERT INTO metadata_cache (cache_key, payload, fetched_at) "
|
||||
"VALUES (?, ?, datetime('now')) "
|
||||
"ON CONFLICT(cache_key) DO UPDATE SET "
|
||||
"payload = excluded.payload, fetched_at = excluded.fetched_at",
|
||||
(cache_key, json.dumps(payload, ensure_ascii=False)),
|
||||
)
|
||||
except (RuntimeError, sqlite3.Error) as exc:
|
||||
logger.warning(
|
||||
"Cache métadonnées non écrit (%s), continuation sans cache : %s", cache_key, exc
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _pick_best_match(payload: dict, query: str) -> dict | None:
|
||||
items = [item for item in payload.get("data", []) if item.get("type") == "anime"]
|
||||
if not items:
|
||||
return None
|
||||
|
||||
genre_names = {
|
||||
obj.get("id"): obj.get("attributes", {}).get("name")
|
||||
for obj in payload.get("included", [])
|
||||
if obj.get("type") == "genres"
|
||||
}
|
||||
|
||||
def titles_of(item: dict) -> set[str]:
|
||||
attrs = item.get("attributes", {})
|
||||
titles = attrs.get("titles") or {}
|
||||
return {
|
||||
t.strip().casefold()
|
||||
for t in (attrs.get("canonicalTitle"), titles.get("en"), titles.get("en_jp"))
|
||||
if t
|
||||
}
|
||||
|
||||
def popularity(item: dict) -> tuple[int, float]:
|
||||
attrs = item.get("attributes", {})
|
||||
return (
|
||||
attrs.get("userCount") or 0,
|
||||
float(attrs.get("averageRating") or 0),
|
||||
)
|
||||
|
||||
best = next((item for item in items if query.casefold() in titles_of(item)), None)
|
||||
if best is None:
|
||||
best = max(items, key=popularity)
|
||||
|
||||
genre_ids = best.get("relationships", {}).get("genres", {}).get("data") or []
|
||||
genres = [genre_names[g["id"]] for g in genre_ids if g.get("id") in genre_names]
|
||||
return {"id": best.get("id"), "attributes": best.get("attributes", {}), "genres": genres}
|
||||
|
||||
@staticmethod
|
||||
def _to_rating_10(average_rating: str | None) -> float | None:
|
||||
"""Kitsu note sur 100 (chaîne) → note sur 10 arrondie à 1 décimale."""
|
||||
if average_rating is None:
|
||||
return None
|
||||
try:
|
||||
return round(float(average_rating) / 10, 1)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_year(start_date: str | None) -> int | None:
|
||||
if not start_date:
|
||||
return None
|
||||
try:
|
||||
return int(str(start_date)[:4])
|
||||
except ValueError:
|
||||
return None
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Paramètres persistés en DB (activation des sources, réglages UI…)."""
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
from app.db import db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def get_setting(key: str, default: object = None) -> object:
|
||||
row = await db.fetchone("SELECT value FROM settings WHERE key = ?", (key,))
|
||||
if row is None:
|
||||
return default
|
||||
try:
|
||||
return json.loads(row["value"])
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Paramètre %r corrompu, valeur par défaut utilisée", key)
|
||||
return default
|
||||
|
||||
|
||||
async def set_setting(key: str, value: object) -> None:
|
||||
await db.execute(
|
||||
"INSERT INTO settings (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(key, json.dumps(value, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
|
||||
async def is_source_enabled(name: str) -> bool:
|
||||
return bool(await get_setting(f"source:{name}:enabled", True))
|
||||
|
||||
|
||||
async def set_source_enabled(name: str, enabled: bool) -> None:
|
||||
await set_setting(f"source:{name}:enabled", enabled)
|
||||
logger.info("Source %s %s", name, "activée" if enabled else "désactivée")
|
||||
|
||||
# ---------------------------------------------------------------- intégrations *arr
|
||||
|
||||
TORZNAB_APIKEY_KEY = "torznab:apikey"
|
||||
SONARR_URL_KEY = "sonarr:url"
|
||||
SONARR_APIKEY_KEY = "sonarr:apikey"
|
||||
|
||||
|
||||
async def get_torznab_apikey() -> str:
|
||||
"""Clé API Torznab (générée au premier appel, persistée en DB)."""
|
||||
key = await get_setting(TORZNAB_APIKEY_KEY)
|
||||
if not isinstance(key, str) or not key:
|
||||
key = secrets.token_hex(16)
|
||||
await set_setting(TORZNAB_APIKEY_KEY, key)
|
||||
logger.info("Clé API Torznab générée")
|
||||
return key
|
||||
|
||||
|
||||
async def reset_torznab_apikey() -> str:
|
||||
key = secrets.token_hex(16)
|
||||
await set_setting(TORZNAB_APIKEY_KEY, key)
|
||||
logger.info("Clé API Torznab régénérée")
|
||||
return key
|
||||
|
||||
|
||||
async def get_sonarr_config() -> dict[str, str]:
|
||||
return {
|
||||
"url": await get_setting(SONARR_URL_KEY, ""),
|
||||
"apikey": await get_setting(SONARR_APIKEY_KEY, ""),
|
||||
}
|
||||
|
||||
|
||||
async def set_sonarr_config(url: str, apikey: str) -> None:
|
||||
await set_setting(SONARR_URL_KEY, url.rstrip("/"))
|
||||
await set_setting(SONARR_APIKEY_KEY, apikey.strip())
|
||||
logger.info("Configuration Sonarr enregistrée (%s)", url)
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Client Sonarr (API v3) — personnalisation de « Pour toi ».
|
||||
|
||||
OhmStreaming lit ce qui est téléchargé/grabé sur Sonarr :
|
||||
- `/api/v3/history` (événements grab/import) → titres récemment obtenus ;
|
||||
- `/api/v3/series` → genres de chaque série (fournis par Sonarr lui-même).
|
||||
|
||||
Ces données alimentent le profil de genres des recommandations et la liste
|
||||
d'exclusion (le déjà-possédé sur Sonarr n'est pas re-recommandé).
|
||||
|
||||
Toujours tolérant aux pannes : Sonarr absent/non configuré → profil vide,
|
||||
la découverte continue de fonctionner avec l'historique local.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.kitsu import normalize_title
|
||||
from app.services.settings import get_sonarr_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HISTORY_PAGE_SIZE = 100
|
||||
_MAX_TITLES = 30 # titres Sonarr récents analysés
|
||||
_PROFILE_TTL = 900.0 # profil Sonarr re-quantifié au bout de 15 min
|
||||
_REQUEST_TIMEOUT = 15.0
|
||||
|
||||
|
||||
class SonarrService:
|
||||
"""Profil de consommation Sonarr : titres téléchargés + genres associés."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._cache: tuple[float, tuple[set[str], dict[str, int]]] | None = None
|
||||
|
||||
async def _config(self) -> tuple[str, str]:
|
||||
"""(url, apikey) — lu en DB à chaque usage (lecture SQLite négligeable)."""
|
||||
config = await get_sonarr_config()
|
||||
return config["url"], config["apikey"]
|
||||
|
||||
def invalidate(self) -> None:
|
||||
"""Force le recalcul du profil au prochain appel (réglages modifiés)."""
|
||||
self._cache = None
|
||||
|
||||
async def _client(self) -> httpx.AsyncClient | None:
|
||||
url, apikey = await self._config()
|
||||
if not url or not apikey:
|
||||
return None
|
||||
return httpx.AsyncClient(
|
||||
base_url=url,
|
||||
timeout=_REQUEST_TIMEOUT,
|
||||
headers={"X-Api-Key": apikey},
|
||||
)
|
||||
|
||||
async def _get(self, path: str, params: dict | None = None) -> dict | list | None:
|
||||
client = await self._client()
|
||||
if client is None:
|
||||
return None
|
||||
try:
|
||||
async with client:
|
||||
response = await client.get(path, params=params)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
logger.warning("Sonarr %s KO : %s", path, exc)
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------ profil
|
||||
|
||||
async def profile(self) -> tuple[set[str], dict[str, int]]:
|
||||
"""(titres possédés normalisés, comptage de genres) — vide si non configuré."""
|
||||
if self._cache is not None and self._cache[0] > time.monotonic():
|
||||
return self._cache[1]
|
||||
|
||||
titles = await self.downloaded_titles()
|
||||
genres_by_title = await self.series_genres()
|
||||
|
||||
owned: set[str] = set()
|
||||
genre_counts: dict[str, int] = {}
|
||||
for title in titles:
|
||||
normalized = normalize_title(title).casefold()
|
||||
if normalized:
|
||||
owned.add(normalized)
|
||||
for genre in genres_by_title.get(normalized, []):
|
||||
genre_counts[genre] = genre_counts.get(genre, 0) + 1
|
||||
|
||||
result = (owned, genre_counts)
|
||||
self._cache = (time.monotonic() + _PROFILE_TTL, result)
|
||||
return result
|
||||
|
||||
async def downloaded_titles(self) -> list[str]:
|
||||
"""Titres récemment grabés/importés sur Sonarr (les plus récents d'abord)."""
|
||||
payload = await self._get(
|
||||
"/api/v3/history",
|
||||
{
|
||||
"page": 1,
|
||||
"pageSize": _HISTORY_PAGE_SIZE,
|
||||
"eventType": 1, # grab ; les imports (3) suivent le même titre
|
||||
"sortKey": "date",
|
||||
"sortDir": "desc",
|
||||
},
|
||||
)
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
titles: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for record in payload.get("records", []):
|
||||
title = record.get("series", {}).get("title")
|
||||
if not title or title in seen:
|
||||
continue
|
||||
seen.add(title)
|
||||
titles.append(title)
|
||||
if len(titles) >= _MAX_TITLES:
|
||||
break
|
||||
return titles
|
||||
|
||||
async def series_genres(self) -> dict[str, list[str]]:
|
||||
"""Genres par série, clé = titre normalisé (minuscule)."""
|
||||
payload = await self._get("/api/v3/series")
|
||||
if not isinstance(payload, list):
|
||||
return {}
|
||||
return {
|
||||
normalize_title(s.get("title", "")).casefold(): [
|
||||
g for g in s.get("genres", []) if isinstance(g, str)
|
||||
]
|
||||
for s in payload
|
||||
if s.get("title")
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------ admin
|
||||
|
||||
async def test_connection(self) -> dict:
|
||||
"""Diagnostic admin : version Sonarr, séries, grabs récents."""
|
||||
url, apikey = await self._config()
|
||||
if not url or not apikey:
|
||||
return {"ok": False, "detail": "URL ou clé API manquante"}
|
||||
payload = await self._get("/api/v3/system/status")
|
||||
if payload is None:
|
||||
return {"ok": False, "detail": "Connexion impossible — vérifiez URL et clé"}
|
||||
series = await self._get("/api/v3/series")
|
||||
titles = await self.downloaded_titles()
|
||||
return {
|
||||
"ok": True,
|
||||
"detail": (
|
||||
f"Sonarr {payload.get('version', '?')} — "
|
||||
f"{len(series) if isinstance(series, list) else 0} séries, "
|
||||
f"{len(titles)} saisies récentes"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
sonarr = SonarrService()
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Indexeur Torznab/Newznab : expose le catalogue OhmStreaming à Sonarr/Prowlarr.
|
||||
|
||||
OhmStreaming devient une source d'indexeur à part entière : Sonarr (ou Prowlarr,
|
||||
qui relaiera vers Radarr/Lidarr…) interroge `/torznab/api` comme n'importe quel
|
||||
indexer Jackett. Chaque « release » correspond à un épisode résolu depuis les
|
||||
sources de scraping ; le grab (`/torznab/download`) déclenche l'extraction puis
|
||||
l'ajout dans la file de téléchargements OhmStreaming — le fichier arrive donc
|
||||
dans la bibliothèque locale, comme un téléchargement manuel.
|
||||
|
||||
Formats :
|
||||
- `t=caps` → capacités du serveur (catégories TV/Anime, paramètres supportés)
|
||||
- `t=tvsearch` → recherche par série + saison + épisode (Sonarr)
|
||||
- `t=search` → recherche libre (Prowlarr, recherche manuelle)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import format_datetime
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from app.scrapers.base import (
|
||||
Episode,
|
||||
ScrapeError,
|
||||
SourceScraper,
|
||||
VideoLink,
|
||||
all_sources,
|
||||
import_all_scrapers,
|
||||
resolve_hoster,
|
||||
)
|
||||
from app.services.downloads import download_manager
|
||||
from app.services.settings import is_source_enabled
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
import_all_scrapers()
|
||||
|
||||
TORZNAB_NS = "http://torznab.com/schemas/2015/feed"
|
||||
CAT_TV = "5000"
|
||||
CAT_ANIME = "5070"
|
||||
_SIZE_ESTIMATE = 400 * 1024 * 1024 # estimation affichée (~400 Mo/épisode)
|
||||
_MAX_SERIES_PER_SOURCE = 2 # fiches explorées par source lors d'une recherche
|
||||
_MAX_RELEASES = 100 # borne du flux retourné
|
||||
_SEARCH_TIMEOUT = 40.0 # scraping lent : garde-fou par source
|
||||
_EPISODE_CACHE_TTL = 600.0 # listes d'épisodes re-scrapées au bout de 10 min
|
||||
|
||||
|
||||
@dataclass
|
||||
class Release:
|
||||
"""Un épisode vu comme une release par Sonarr/Prowlarr."""
|
||||
|
||||
series: str
|
||||
season: int
|
||||
ep: int
|
||||
source: str
|
||||
source_id: str
|
||||
episode_url: str
|
||||
|
||||
@property
|
||||
def sonarr_title(self) -> str:
|
||||
return f"{self.series} S{self.season:02d}E{self.ep:02d} VOSTFR WEB-DL"
|
||||
|
||||
|
||||
|
||||
def _bencode(value) -> bytes:
|
||||
if isinstance(value, int):
|
||||
return f"i{value}e".encode()
|
||||
if isinstance(value, str):
|
||||
raw = value.encode()
|
||||
return f"{len(raw)}:".encode() + raw
|
||||
if isinstance(value, bytes):
|
||||
return f"{len(value)}:".encode() + value
|
||||
if isinstance(value, dict):
|
||||
return b"d" + b"".join(
|
||||
_bencode(k) + _bencode(v) for k, v in sorted(value.items())
|
||||
) + b"e"
|
||||
if isinstance(value, list):
|
||||
return b"l" + b"".join(_bencode(v) for v in value) + b"e"
|
||||
raise TypeError(f"type non encodable en bencode : {type(value)!r}")
|
||||
|
||||
|
||||
def torrent_stub(announce_url: str, name: str) -> bytes:
|
||||
"""Fichier .torrent minimal (le vrai téléchargement est fait par OhmStreaming)."""
|
||||
return _bencode(
|
||||
{
|
||||
"announce": announce_url,
|
||||
"created by": "OhmStreaming",
|
||||
"comment": name,
|
||||
"info": {
|
||||
"name": name + ".mp4",
|
||||
"length": 0,
|
||||
"piece length": 32768,
|
||||
"pieces": b"\x00" * 20,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TorznabService:
|
||||
"""Recherche multi-sources mappée en releases Torznab + grab → file interne."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._episodes_cache: dict[tuple[str, str], tuple[float, list[Episode]]] = {}
|
||||
|
||||
# ------------------------------------------------------------ recherche
|
||||
|
||||
async def tvsearch(self, q: str, season: int | None = None, ep: int | None = None) -> list[Release]:
|
||||
"""Recherche type Sonarr : série (+ saison/épisode optionnels)."""
|
||||
releases: list[Release] = []
|
||||
for source in await self._enabled_sources():
|
||||
outcomes = await self._search_source(source, q)
|
||||
for result in outcomes[:_MAX_SERIES_PER_SOURCE]:
|
||||
try:
|
||||
episodes = await self._episodes_of(source, result.source_id)
|
||||
except (ScrapeError, TimeoutError):
|
||||
continue
|
||||
releases.extend(
|
||||
self._releases_for(result.title, source, result.source_id, episodes, season, ep)
|
||||
)
|
||||
if len(releases) >= _MAX_RELEASES:
|
||||
return releases[:_MAX_RELEASES]
|
||||
return releases
|
||||
|
||||
async def search(self, q: str) -> list[Release]:
|
||||
"""Recherche libre : tous les épisodes des fiches trouvées."""
|
||||
return await self.tvsearch(q)
|
||||
|
||||
async def _search_source(self, source: SourceScraper, q: str):
|
||||
try:
|
||||
return await asyncio.wait_for(source.search(q), timeout=_SEARCH_TIMEOUT)
|
||||
except (ScrapeError, TimeoutError) as exc:
|
||||
logger.warning("Torznab : recherche %s KO pour %r : %s", source.name, q, exc)
|
||||
return []
|
||||
|
||||
async def _enabled_sources(self) -> list[SourceScraper]:
|
||||
return [s for s in all_sources() if await is_source_enabled(s.name)]
|
||||
|
||||
async def _episodes_of(self, source: SourceScraper, source_id: str) -> list[Episode]:
|
||||
"""Liste d'épisodes d'une fiche, avec cache mémoire TTL."""
|
||||
key = (source.name, source_id)
|
||||
cached = self._episodes_cache.get(key)
|
||||
if cached is not None and cached[0] > time.monotonic():
|
||||
return cached[1]
|
||||
episodes = await asyncio.wait_for(source.list_episodes(source_id), timeout=_SEARCH_TIMEOUT)
|
||||
self._episodes_cache[key] = (time.monotonic() + _EPISODE_CACHE_TTL, episodes)
|
||||
return episodes
|
||||
|
||||
def _releases_for(
|
||||
self,
|
||||
series: str,
|
||||
source: SourceScraper,
|
||||
source_id: str,
|
||||
episodes: list[Episode],
|
||||
season: int | None,
|
||||
ep: int | None,
|
||||
) -> list[Release]:
|
||||
"""Filtre les épisodes selon saison/épisode demandés (entiers uniquement)."""
|
||||
out = []
|
||||
for item in episodes:
|
||||
if item.number != int(item.number): # OAV 2.5 → ignorée (inparseable Sonarr)
|
||||
continue
|
||||
if season is not None and item.season != season:
|
||||
continue
|
||||
if ep is not None and int(item.number) != ep:
|
||||
continue
|
||||
out.append(
|
||||
Release(
|
||||
series=series,
|
||||
season=item.season,
|
||||
ep=int(item.number),
|
||||
source=source.name,
|
||||
source_id=source_id,
|
||||
episode_url=item.url,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
# ------------------------------------------------------------ grab
|
||||
|
||||
async def grab(self, source: str, source_id: str, season: int, ep: int, series: str) -> dict:
|
||||
"""Résout l'épisode (embed → vidéo directe) puis l'ajoute à la file interne.
|
||||
|
||||
Retourne le dict du téléchargement (existant si doublon actif).
|
||||
Lève ScrapeError si introuvable ou qu'aucun hébergeur n'a répondu.
|
||||
"""
|
||||
from app.scrapers.base import get_source
|
||||
|
||||
scraper = get_source(source)
|
||||
episodes = await self._episodes_of(scraper, source_id)
|
||||
match = next(
|
||||
(
|
||||
e
|
||||
for e in episodes
|
||||
if e.season == season and e.number == int(ep)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if match is None:
|
||||
raise ScrapeError(f"Épisode S{season:02d}E{ep:02d} introuvable sur {source}")
|
||||
|
||||
link = await self._resolve_video(scraper, match.url)
|
||||
title = f"{series} S{season:02d}E{ep:02d}"
|
||||
result = await download_manager.enqueue(link.url, match.url, title)
|
||||
if link.is_hls or link.headers.get("Referer"):
|
||||
result["note"] = "HLS/proxy : OhmStreaming gère le téléchargement via ffmpeg"
|
||||
logger.info("Torznab grab %s → download id=%s", title, result.get("id"))
|
||||
return result
|
||||
|
||||
async def _resolve_video(self, scraper: SourceScraper, episode_url: str) -> VideoLink:
|
||||
"""Chaîne complète : page épisode → embeds → première URL directe valide."""
|
||||
embeds = await asyncio.wait_for(
|
||||
scraper.extract_embed_links(episode_url), timeout=_SEARCH_TIMEOUT
|
||||
)
|
||||
errors: list[str] = []
|
||||
for embed in embeds:
|
||||
extractor = resolve_hoster(embed)
|
||||
if extractor is None:
|
||||
errors.append(f"hébergeur non supporté : {embed}")
|
||||
continue
|
||||
try:
|
||||
link = await asyncio.wait_for(extractor.extract(embed), timeout=_SEARCH_TIMEOUT)
|
||||
if link and link.url:
|
||||
return link
|
||||
except (ScrapeError, TimeoutError) as exc:
|
||||
errors.append(str(exc))
|
||||
raise ScrapeError(f"Aucun hébergeur résolu pour {episode_url} ({'; '.join(errors[:3])})")
|
||||
|
||||
# ------------------------------------------------------------ XML
|
||||
|
||||
def download_url(self, base_url: str, apikey: str, release: Release) -> str:
|
||||
query = urlencode(
|
||||
{
|
||||
"apikey": apikey,
|
||||
"source": release.source,
|
||||
"sid": release.source_id,
|
||||
"season": release.season,
|
||||
"ep": release.ep,
|
||||
"series": release.series,
|
||||
}
|
||||
)
|
||||
return f"{base_url}/torznab/download?{query}"
|
||||
|
||||
def caps_xml(self, base_url: str) -> str:
|
||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<caps>
|
||||
<server version="1.0" title="OhmStreaming" url="{base_url}"
|
||||
email="ohm@localhost" image="{base_url}/static/img/logo.png"/>
|
||||
<searching>
|
||||
<search available="yes" supportedParams="q"/>
|
||||
<tv-search available="yes" supportedParams="q,season,ep"/>
|
||||
<movie-search available="no" supportedParams=""/>
|
||||
<audio-search available="no" supportedParams=""/>
|
||||
</searching>
|
||||
<categories>
|
||||
<category id="{CAT_TV}" name="TV">
|
||||
<subcat id="{CAT_ANIME}" name="Anime"/>
|
||||
</category>
|
||||
</categories>
|
||||
</caps>"""
|
||||
|
||||
def results_xml(self, base_url: str, apikey: str, releases: list[Release]) -> str:
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
items = []
|
||||
for release in releases:
|
||||
url = escape(self.download_url(base_url, apikey, release))
|
||||
pubdate = escape(format_datetime(datetime.now(UTC)))
|
||||
items.append(f""" <item>
|
||||
<title>{escape(release.sonarr_title)}</title>
|
||||
<guid isPermaLink="true">{url}</guid>
|
||||
<link>{url}</link>
|
||||
<comments>{escape(release.episode_url)}</comments>
|
||||
<pubDate>{pubdate}</pubDate>
|
||||
<category>{CAT_ANIME}</category>
|
||||
<enclosure url="{url}" length="{_SIZE_ESTIMATE}" type="application/x-bittorrent"/>
|
||||
<torznab:attr name="seeders" value="1"/>
|
||||
<torznab:attr name="peers" value="1"/>
|
||||
<torznab:attr name="downloadvolumefactor" value="0"/>
|
||||
<torznab:attr name="uploadvolumefactor" value="0"/>
|
||||
</item>""")
|
||||
body = "\n".join(items)
|
||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:torznab="{TORZNAB_NS}">
|
||||
<channel>
|
||||
<title>OhmStreaming</title>
|
||||
<link>{escape(base_url)}</link>
|
||||
<description>Indexeur OhmStreaming — animes VOSTFR scrapés en direct</description>
|
||||
<language>fr-FR</language>
|
||||
{body}
|
||||
</channel>
|
||||
</rss>"""
|
||||
|
||||
def error_xml(self, code: int, description: str) -> str:
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
return (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||
f'<error code="{code}" description="{escape(description)}"/>'
|
||||
)
|
||||
|
||||
|
||||
torznab = TorznabService()
|
||||
Reference in New Issue
Block a user