v0.1.0 — Réécriture complète de OhmStreaming

Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree,
French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement
HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration,
découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
This commit is contained in:
Roman
2026-09-22 10:05:47 +00:00
commit 41566ab5fb
66 changed files with 8895 additions and 0 deletions
View File
+352
View File
@@ -0,0 +1,352 @@
"""Découverte : nouveautés des sources, incontournables et recommandations.
Trois sections :
- **latest** — « récemment ajoutés » scrapés sur chaque source activée (cliquables
directement vers la fiche) ;
- **must_watch** — titres les plus populaires du catalogue Kitsu (tous temps) ;
- **for_you** — recommandations par genres : les genres des titres téléchargés
(serveur), des favoris (par utilisateur) et des séries téléchargées sur Sonarr
sont agrégés, puis Kitsu est interrogé sur ces catégories en excluant le
déjà-possédé.
Toutes les sources externes sont optionnelles : un échec (réseau, scraping, API)
laisse la section vide et n'est jamais remonté au caller (dégradation gracieuse).
Un cache mémoire TTL évite de re-scaper à chaque chargement de page.
"""
import asyncio
import dataclasses
import json
import logging
import re
import time
import unicodedata
import httpx
from app.config import get_settings
from app.db import db
from app.scrapers.base import (
ScrapeError,
SourceScraper,
all_sources,
import_all_scrapers,
)
from app.services.kitsu import KitsuService, normalize_title
from app.services.settings import is_source_enabled
from app.services.sonarr import sonarr
logger = logging.getLogger(__name__)
import_all_scrapers()
# Bornes de l'algorithme
_MAX_HISTORY_TITLES = 12 # titres récents analysés (téléchargements + favoris)
_MAX_GENRES = 4 # genres retenus pour la requête Kitsu
_KITSU_PAGE_MAX = 20 # limite dure de l'API Kitsu (page[limit] > 20 → 400)
_ENRICH_CONCURRENCY = 6 # enrichissements Kitsu parallèles max (nouveautés)
_LATEST_TTL_SECONDS = 600 # nouveautés : re-scrape au bout de 10 min
_MUST_WATCH_TTL_SECONDS = 21600 # incontournables : quasi statique, 6 h
_FOR_YOU_TTL_SECONDS = 3600 # recommandations : 1 h (l'historique évolue lentement)
class _TTLCache:
"""Cache mémoire minimal avec expiration (mono-processus, suffisant ici)."""
def __init__(self) -> None:
self._data: dict[str, tuple[float, object]] = {}
def get(self, key: str) -> object | None:
entry = self._data.get(key)
if entry is None:
return None
expires_at, value = entry
if expires_at <= time.monotonic():
del self._data[key]
return None
return value
def set(self, key: str, value: object, ttl_seconds: float) -> None:
self._data[key] = (time.monotonic() + ttl_seconds, value)
def clear(self, prefix: str = "") -> None:
"""Invalide les clés commençant par prefix (vide = tout le cache)."""
for key in [k for k in self._data if k.startswith(prefix)]:
del self._data[key]
def category_slug(name: str) -> str:
"""Nom de genre → slug de catégorie Kitsu (« Slice of Life » → « slice-of-life »)."""
decomposed = unicodedata.normalize("NFKD", name)
ascii_only = "".join(char for char in decomposed if not unicodedata.combining(char))
return re.sub(r"[^a-z0-9]+", "-", ascii_only.casefold()).strip("-")
class DiscoverService:
"""Agrégation des trois sections de découverte, avec cache mémoire."""
def __init__(self) -> None:
self._cache = _TTLCache()
self._kitsu = KitsuService()
# ------------------------------------------------------------ nouveautés
async def latest(self, limit: int = 24) -> list[dict]:
"""Nouveautés toutes sources confondues, triées par date de sortie réelle.
Les « récemment ajoutés » de chaque source sont fusionnés (doublons retirés),
enrichis via Kitsu (date de début, statut de diffusion) puis triés du plus
récent au plus ancien — ce qui sort / vient de sortir en premier.
"""
cached = self._cache.get(f"latest:{limit}")
if cached is not None:
return cached # type: ignore[return-value]
sources = [s for s in all_sources() if await is_source_enabled(s.name)]
outcomes = await asyncio.gather(*(self._latest_of(source) for source in sources))
items = [item for outcome in outcomes for item in (outcome or [])]
merged: dict[str, dict] = {}
for item in items:
key = normalize_title(item["title"]).casefold()
existing = merged.get(key)
if existing is None or (not existing.get("image_url") and item.get("image_url")):
merged[key] = item
semaphore = asyncio.Semaphore(_ENRICH_CONCURRENCY)
async def bounded(item: dict) -> dict:
async with semaphore:
return await self._with_release_info(item)
enriched = await asyncio.gather(*(bounded(item) for item in merged.values()))
result = sorted(enriched, key=lambda it: it.get("start_date") or "", reverse=True)
result = result[:limit]
self._cache.set(f"latest:{limit}", result, _LATEST_TTL_SECONDS)
return result
async def _latest_of(self, source: SourceScraper) -> list[dict] | None:
"""Items latest() d'une source, aplatis avec les infos de source ([] si KO)."""
try:
results = await source.latest()
except ScrapeError as exc:
logger.warning("Nouveautés indisponibles pour %s : %s", source.name, exc)
return None
return [
{**dataclasses.asdict(r), "source": source.name, "label": source.label}
for r in results
]
async def _with_release_info(self, item: dict) -> dict:
"""Complète un item de nouveauté avec sa date de sortie Kitsu (None si absent)."""
item.setdefault("start_date", None)
item.setdefault("status", None)
item.setdefault("rating", None)
match = await self._kitsu_match_for_title(item["title"])
if match is None:
return item
attrs = match.get("attributes", {})
item["start_date"] = attrs.get("startDate")
item["status"] = attrs.get("status")
item["rating"] = KitsuService._to_rating_10(attrs.get("averageRating"))
return item
# --------------------------------------------------------- incontournables
async def must_watch(self, limit: int = _KITSU_PAGE_MAX) -> list[dict]:
"""Titres les plus populaires du catalogue Kitsu (tous temps)."""
limit = min(limit, _KITSU_PAGE_MAX)
key = f"must_watch:{limit}"
cached = self._cache.get(key)
if cached is not None:
return cached # type: ignore[return-value]
items = await self._kitsu_anime({"sort": "-userCount", "page[limit]": limit})
self._cache.set(key, items, _MUST_WATCH_TTL_SECONDS)
return items
# ------------------------------------------------------------ pour toi
async def for_you(self, user_id: int, limit: int = _KITSU_PAGE_MAX) -> dict:
"""Recommandations par genres, à partir de l'historique de l'utilisateur.
Genres = téléchargements du serveur (titres → Kitsu) + favoris de l'utilisateur
(genres du payload) + séries téléchargées sur Sonarr (genres fournis par
Sonarr). On exclut les titres déjà possédés (local et Sonarr).
"""
limit = min(limit, _KITSU_PAGE_MAX)
cache_key = f"for_you:{user_id}:{limit}"
cached = self._cache.get(cache_key)
if cached is not None:
return cached # type: ignore[return-value]
owned, favorite_genres = await self._owned(user_id)
genre_counts = await self._genres_from_downloads(owned)
for genre, count in favorite_genres.items():
genre_counts[genre] = genre_counts.get(genre, 0) + count
sonarr_owned, sonarr_genres = await sonarr.profile()
for genre, count in sonarr_genres.items():
genre_counts[genre] = genre_counts.get(genre, 0) + count
owned |= sonarr_owned
if not genre_counts:
result: dict = {"based_on": [], "items": []}
self._cache.set(cache_key, result, _FOR_YOU_TTL_SECONDS)
return result
top_genres = sorted(genre_counts, key=genre_counts.get, reverse=True)[:_MAX_GENRES]
slugs = [category_slug(genre) for genre in top_genres]
items = await self._kitsu_anime(
{
"filter[categories]": ",".join(slugs),
"sort": "-userCount",
"page[limit]": limit, # le déjà-possédé est filtré après
}
)
kept = [item for item in items if item["title"] and item["title"].casefold() not in owned]
result = {"based_on": top_genres, "items": kept}
self._cache.set(cache_key, result, _FOR_YOU_TTL_SECONDS)
return result
def invalidate_for_you(self) -> None:
"""Recommandations recalculées au prochain appel (réglages Sonarr modifiés)."""
self._cache.clear("for_you:")
async def _owned(self, user_id: int) -> tuple[set[str], dict[str, int]]:
"""Titres possédés (normalisés) + genres directement connus via les favoris."""
rows = await db.fetchall(
"SELECT DISTINCT title FROM downloads ORDER BY created_at DESC LIMIT ?",
(_MAX_HISTORY_TITLES,),
)
fav_rows = await db.fetchall(
"SELECT payload FROM favorites WHERE user_id = ? ORDER BY created_at DESC LIMIT ?",
(user_id, _MAX_HISTORY_TITLES),
)
owned = {normalize_title(row["title"]).casefold() for row in rows}
owned.discard("")
genre_counts: dict[str, int] = {}
for row in fav_rows:
try:
payload = json.loads(row["payload"]) if row["payload"] else {}
except (TypeError, ValueError):
continue
for genre in payload.get("genres") or []:
if isinstance(genre, str) and genre.strip():
genre_counts[genre.strip()] = genre_counts.get(genre.strip(), 0) + 1
return owned, genre_counts
async def _genres_from_downloads(self, titles: set[str]) -> dict[str, int]:
"""Genres Kitsu des titres téléchargés (cache DB puis recherche)."""
semaphore = asyncio.Semaphore(_MAX_HISTORY_TITLES)
async def genres_of(title: str) -> list[str]:
async with semaphore:
return await self._kitsu_genres_for_title(title)
outcomes = await asyncio.gather(*(genres_of(t) for t in list(titles)[:_MAX_HISTORY_TITLES]))
counts: dict[str, int] = {}
for genres in outcomes:
for genre in genres:
counts[genre] = counts.get(genre, 0) + 1
return counts
async def _kitsu_match_for_title(self, title: str) -> dict | None:
"""Match Kitsu d'un titre scrapé (cache DB 72 h via metadata_cache)."""
query = normalize_title(title)
if not query:
return None
cache_key = f"kitsu:anime:{query.casefold()}"
match = await self._kitsu.get_cached(cache_key)
if match is None:
match = await self._kitsu.search_anime(title)
if match is not None:
await self._kitsu.set_cached(cache_key, match)
return match
async def _kitsu_genres_for_title(self, title: str) -> list[str]:
"""Genres Kitsu d'un titre scrapé (cache DB 72 h via metadata_cache).
La recherche Kitsu ne renvoie plus les genres (`include=genres` vide) :
on complète avec l'endpoint /anime/<id>/categories.
"""
match = await self._kitsu_match_for_title(title)
if match is None:
return []
genres = [g for g in match.get("genres", []) if isinstance(g, str)]
if not genres:
genres = await self._kitsu_categories(match.get("id"))
match["genres"] = genres
query = normalize_title(title)
await self._kitsu.set_cached( # refresh avec les genres
f"kitsu:anime:{query.casefold()}", match
)
return genres
# -------------------------------------------------------------- Kitsu
async def _kitsu_anime(self, params: dict) -> list[dict]:
"""Requête générique liste Kitsu → items normalisés ([] si échec)."""
settings = get_settings()
try:
async with httpx.AsyncClient(
timeout=settings.http_timeout,
headers={
"User-Agent": settings.user_agent,
"Accept": "application/vnd.api+json",
},
) as client:
response = await client.get(f"{settings.kitsu_base_url}/anime", params=params)
response.raise_for_status()
payload = response.json()
except (httpx.HTTPError, ValueError) as exc:
logger.warning("Liste Kitsu échouée (%s) : %s", params, exc)
return []
return [
self._normalize_anime(item)
for item in payload.get("data", [])
if item.get("type") == "anime"
]
async def _kitsu_categories(self, anime_id: object) -> list[str]:
"""Titres des catégories Kitsu d'un anime ([] si échec)."""
if not anime_id:
return []
settings = get_settings()
try:
async with httpx.AsyncClient(
timeout=settings.http_timeout,
headers={
"User-Agent": settings.user_agent,
"Accept": "application/vnd.api+json",
},
) as client:
response = await client.get(
f"{settings.kitsu_base_url}/anime/{anime_id}/categories",
params={"page[limit]": _KITSU_PAGE_MAX},
)
response.raise_for_status()
payload = response.json()
except (httpx.HTTPError, ValueError) as exc:
logger.warning("Catégories Kitsu échouées (anime %s) : %s", anime_id, exc)
return []
return [
attrs["title"]
for item in payload.get("data", [])
if isinstance(attrs := item.get("attributes", {}), dict) and attrs.get("title")
]
@staticmethod
def _normalize_anime(item: dict) -> dict:
attrs = item.get("attributes", {})
titles = attrs.get("titles") or {}
poster = attrs.get("posterImage") or {}
return {
"kitsu_id": item.get("id"),
"title": attrs.get("canonicalTitle") or titles.get("en_jp"),
"image_url": poster.get("large") or poster.get("medium") or poster.get("tiny"),
"rating": KitsuService._to_rating_10(attrs.get("averageRating")),
"year": KitsuService._extract_year(attrs.get("startDate")),
"subtype": attrs.get("subtype"),
"user_count": attrs.get("userCount"),
}
discover = DiscoverService()
+499
View File
@@ -0,0 +1,499 @@
"""Gestionnaire de téléchargements : file asyncio, parallélisme limité,
pause/reprise (Range HTTP), anti-doublons, persistance, progression temps réel.
Les statuts : pending → downloading → done | failed | cancelled
↕ paused
"""
import asyncio
import contextlib
import logging
import re
import signal
import time
import unicodedata
from collections.abc import AsyncIterator
from pathlib import Path
from typing import Any
from urllib.parse import urljoin
import httpx
from app.config import get_settings
from app.db import db
logger = logging.getLogger(__name__)
ACTIVE_STATUSES = ("pending", "downloading", "paused")
_FFMPEG_TIME_RE = re.compile(r"time=(\d+:\d+:\d+(?:\.\d+)?)")
_EXTINF_RE = re.compile(r"#EXTINF:([\d.]+)")
_BANDWIDTH_RE = re.compile(r"#EXT-X-STREAM-INF:[^\n]*BANDWIDTH=(\d+)[^\n]*\n(\S+)")
def _parse_ffmpeg_time(value: str) -> float:
parts = value.split(":")
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
def _best_variant(master_body: str, base_url: str) -> str | None:
"""URL de la variante au plus haut débit d'une playlist maître HLS."""
variants = [
(int(bw), urljoin(base_url, uri)) for bw, uri in _BANDWIDTH_RE.findall(master_body)
]
return max(variants)[1] if variants else None
_STATUS_LABELS = {
"pending": "en attente",
"downloading": "en cours",
"paused": "en pause",
"done": "terminé",
"failed": "échec",
"cancelled": "annulé",
}
def sanitize_filename(name: str) -> str:
"""Nettoie un nom de fichier : caractères interdits retirés, anti-traversée."""
name = unicodedata.normalize("NFKC", name)
name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', " ", name)
name = re.sub(r"\s+", " ", name).strip(" .")
if not name or name in (".", ".."):
name = "video"
return name[:150]
class DownloadManager:
"""File d'attente de téléchargements, injectée dans les routes via app.state."""
def __init__(self) -> None:
self._queue: asyncio.Queue[int] # créée dans start() (affinité avec la boucle)
self._tasks: dict[int, asyncio.Task] = {} # download_id → tâche asyncio
self._pause_events: dict[int, asyncio.Event] = {} # set = peut tourner
self._progress: dict[int, dict[str, Any]] = {} # progression temps réel en mémoire
self._listeners: list[asyncio.Queue] = []
self._workers: list[asyncio.Task] = []
self._client: httpx.AsyncClient | None = None
self._hls_processes: dict[int, asyncio.subprocess.Process] = {}
# ------------------------------------------------------------ cycle de vie
async def start(self) -> None:
self._queue = asyncio.Queue()
settings = get_settings()
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(30.0, read=300.0),
follow_redirects=True,
headers={
"User-Agent": settings.user_agent,
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
},
)
# Restaure les téléchargements interrompus (crash/arrêt) en 'pending'
await db.execute(
"UPDATE downloads SET status = 'pending', updated_at = datetime('now') "
"WHERE status = 'downloading'"
)
await self._scan_download_dir()
for _ in range(settings.max_parallel_downloads):
self._workers.append(asyncio.create_task(self._worker()))
logger.info("DownloadManager démarré (%d workers)", settings.max_parallel_downloads)
async def stop(self) -> None:
for worker in self._workers:
worker.cancel()
for task in self._tasks.values():
task.cancel()
for proc in self._hls_processes.values():
if proc.returncode is None:
proc.kill()
if self._client:
await self._client.aclose()
self._workers.clear()
self._tasks.clear()
async def _scan_download_dir(self) -> None:
"""Restaure en 'done' les fichiers présents sur disque mais inconnus de la DB."""
download_dir = get_settings().download_dir
rows = await db.fetchall("SELECT file_path FROM downloads WHERE file_path IS NOT NULL")
known = {row["file_path"] for row in rows}
for path in download_dir.iterdir():
if path.is_file() and path.suffix != ".part" and path.name not in known:
size = path.stat().st_size
await db.execute(
"INSERT INTO downloads "
"(source_key, video_url, page_url, title, file_path, status, "
" total_bytes, downloaded_bytes) "
"VALUES (?, ?, ?, ?, ?, 'done', ?, ?)",
(
f"file:{path.name}",
"",
"",
path.stem,
path.name,
size,
size,
),
)
logger.info("Fichier restauré depuis le disque : %s", path.name)
# ------------------------------------------------------------ API publique
async def enqueue(self, video_url: str, page_url: str, title: str) -> dict:
"""Ajoute un téléchargement ; retourne la tâche existante si doublon actif."""
source_key = video_url
existing = await db.fetchone(
f"SELECT * FROM downloads WHERE source_key = ? AND status IN "
f"({','.join('?' * len(ACTIVE_STATUSES))})",
(source_key, *ACTIVE_STATUSES),
)
if existing:
logger.info("Anti-doublon : %s déjà en file (id=%s)", title, existing["id"])
return self._to_dict(existing, duplicate=True)
filename = sanitize_filename(title) + self._guess_extension(video_url)
cursor = await db.execute(
"INSERT INTO downloads (source_key, video_url, page_url, title, file_path) "
"VALUES (?, ?, ?, ?, ?)",
(source_key, video_url, page_url, title, filename),
)
download_id = cursor.lastrowid
await self._queue.put(download_id)
await self._emit(download_id)
logger.info("Téléchargement ajouté : %s (id=%s)", title, download_id)
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
return self._to_dict(row)
async def pause(self, download_id: int) -> bool:
event = self._pause_events.get(download_id)
if event:
event.clear()
await self._set_status(download_id, "paused")
return True
async def resume(self, download_id: int) -> bool:
row = await self._get_row(download_id)
if row["status"] != "paused":
return False
await self._set_status(download_id, "pending")
await self._queue.put(download_id)
return True
async def retry(self, download_id: int) -> bool:
row = await self._get_row(download_id)
if row["status"] not in ("failed", "cancelled"):
return False
await db.execute(
"UPDATE downloads SET status = 'pending', error = NULL, downloaded_bytes = 0, "
"updated_at = datetime('now') WHERE id = ?",
(download_id,),
)
part = self._part_path(row["file_path"])
part.unlink(missing_ok=True)
await self._queue.put(download_id)
await self._emit(download_id)
return True
async def cancel(self, download_id: int) -> bool:
task = self._tasks.get(download_id)
if task:
task.cancel()
proc = self._hls_processes.get(download_id)
if proc and proc.returncode is None:
proc.kill()
await self._set_status(download_id, "cancelled")
row = await self._get_row(download_id)
self._part_path(row["file_path"]).unlink(missing_ok=True)
await self._emit(download_id)
return True
async def cancel_all(self) -> int:
rows = await db.fetchall(
f"SELECT id FROM downloads WHERE status IN ({','.join('?' * len(ACTIVE_STATUSES))})",
ACTIVE_STATUSES,
)
for row in rows:
await self.cancel(row["id"])
return len(rows)
async def clear_finished(self) -> int:
"""Supprime de la file les tâches terminées/échouées/annulées (fichiers gardés)."""
cursor = await db.execute(
"DELETE FROM downloads WHERE status IN ('done', 'failed', 'cancelled')"
)
return cursor.rowcount or 0
async def list_all(self, limit: int = 200) -> list[dict]:
rows = await db.fetchall(
"SELECT * FROM downloads ORDER BY "
"CASE status WHEN 'downloading' THEN 0 WHEN 'pending' THEN 1 WHEN 'paused' THEN 2 "
"ELSE 3 END, updated_at DESC LIMIT ?",
(limit,),
)
return [self._to_dict(row) for row in rows]
async def get(self, download_id: int) -> dict | None:
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
return self._to_dict(row) if row else None
# ------------------------------------------------------------ événements SSE
async def subscribe(self) -> AsyncIterator[dict]:
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
self._listeners.append(queue)
try:
while True:
yield await queue.get()
finally:
self._listeners.remove(queue)
async def _emit(self, download_id: int) -> None:
data = await self.get(download_id)
if data is None:
return
for queue in self._listeners:
with contextlib.suppress(asyncio.QueueFull):
queue.put_nowait(data)
# ------------------------------------------------------------ worker interne
async def _worker(self) -> None:
while True:
download_id = await self._queue.get()
row = await db.fetchone("SELECT status FROM downloads WHERE id = ?", (download_id,))
if row is None or row["status"] != "pending":
continue # annulé/pausé entre-temps
# Tâche dédiée : annuler un téléchargement ne doit pas tuer le worker
task = asyncio.create_task(self._download(download_id))
self._tasks[download_id] = task
self._pause_events[download_id] = asyncio.Event()
self._pause_events[download_id].set()
try:
await task
except asyncio.CancelledError:
if asyncio.current_task().cancelling() > 0:
raise # le worker lui-même s'arrête (stop())
finally:
self._tasks.pop(download_id, None)
self._pause_events.pop(download_id, None)
self._progress.pop(download_id, None)
async def _download(self, download_id: int) -> None:
"""Dispatche HTTP/HLS ; gestion d'erreurs centralisée ici."""
row = await self._get_row(download_id)
try:
if ".m3u8" in row["video_url"]:
await self._download_hls(download_id, row)
else:
await self._download_http(download_id, row)
except asyncio.CancelledError:
logger.info("Téléchargement annulé : %s", row["file_path"])
raise
except (httpx.HTTPError, OSError) as exc:
# Échec réseau/disque : journalisé, statut 'failed' visible dans l'UI
logger.error("Échec du téléchargement de %s : %s", row["file_path"], exc)
part = self._part_path(row["file_path"])
downloaded = part.stat().st_size if part.exists() else 0
await self._fail(download_id, exc, downloaded)
async def _download_http(self, download_id: int, row: Any) -> None:
"""Téléchargement HTTP direct avec reprise via Range."""
video_url, file_path = row["video_url"], row["file_path"]
target = get_settings().download_dir / file_path
part = self._part_path(file_path)
downloaded = part.stat().st_size if part.exists() else 0
headers: dict[str, str] = {}
if row["page_url"]:
headers["Referer"] = row["page_url"]
if downloaded:
headers["Range"] = f"bytes={downloaded}-"
logger.info("Reprise de %s à %d octets", file_path, downloaded)
await self._set_status(download_id, "downloading")
await self._emit(download_id)
started = time.monotonic()
last_emit = 0.0
async with self._client.stream("GET", video_url, headers=headers) as response:
if response.status_code == 416: # plage invalide → déjà complet
part.rename(target)
await self._finish(download_id, downloaded)
return
response.raise_for_status()
if downloaded and response.status_code != 206:
downloaded = 0 # serveur sans support Range → on repart de zéro
logger.warning("Pas de reprise possible pour %s", file_path)
total = int(response.headers.get("content-length") or 0) + downloaded or None
await db.execute(
"UPDATE downloads SET total_bytes = ? WHERE id = ?", (total, download_id)
)
mode = "ab" if downloaded else "wb"
with part.open(mode) as fh:
async for chunk in response.aiter_bytes(1 << 16):
event = self._pause_events.get(download_id)
if event is not None:
await event.wait() # pause coopérative
fh.write(chunk)
downloaded += len(chunk)
now = time.monotonic()
if now - last_emit >= 1.0:
last_emit = now
await self._report_progress(download_id, downloaded, total, started)
part.rename(target)
await self._finish(download_id, downloaded)
async def _download_hls(self, download_id: int, row: Any) -> None:
"""Télécharge un flux HLS (.m3u8) via ffmpeg (remux en mp4).
Pause via SIGSTOP/SIGCONT du processus, annulation via kill.
La progression est estimée depuis la durée totale de la playlist.
"""
video_url, file_path = row["video_url"], row["file_path"]
target = get_settings().download_dir / file_path
part = self._part_path(file_path)
part.unlink(missing_ok=True) # pas de reprise partielle en HLS
ffmpeg_headers = f"User-Agent: {get_settings().user_agent}\r\n"
if row["page_url"]:
ffmpeg_headers += f"Referer: {row['page_url']}\r\n"
ffmpeg_headers += "Accept-Language: fr-FR,fr;q=0.9,en;q=0.8\r\n"
total_seconds = await self._hls_duration(video_url, row["page_url"])
await self._set_status(download_id, "downloading")
await self._emit(download_id)
started = time.monotonic()
process = await asyncio.create_subprocess_exec(
"ffmpeg", "-y", "-nostdin", "-v", "error", "-nostats", "-progress", "pipe:2",
"-headers", ffmpeg_headers,
"-i", video_url,
"-c", "copy", "-bsf:a", "aac_adtstoasc", "-f", "mp4",
str(part),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE,
)
self._hls_processes[download_id] = process
try:
assert process.stderr is not None
async for raw_line in process.stderr:
event = self._pause_events.get(download_id)
if event is not None and not event.is_set():
process.send_signal(signal.SIGSTOP)
await event.wait()
process.send_signal(signal.SIGCONT)
line = raw_line.decode(errors="replace")
if match := _FFMPEG_TIME_RE.search(line):
elapsed_video = _parse_ffmpeg_time(match.group(1))
size = part.stat().st_size if part.exists() else 0
total_est = None
if total_seconds and elapsed_video > 0:
total_est = int(size / elapsed_video * total_seconds)
await self._report_progress(download_id, size, total_est, started)
return_code = await process.wait()
finally:
self._hls_processes.pop(download_id, None)
if return_code != 0:
part.unlink(missing_ok=True)
raise OSError(f"ffmpeg a échoué (code {return_code}) sur le flux HLS")
size = part.stat().st_size
part.rename(target)
await self._finish(download_id, size)
async def _hls_duration(self, playlist_url: str, referer: str | None) -> float | None:
"""Durée totale d'une playlist HLS (somme des EXTINF de la variante max)."""
headers = {"Referer": referer} if referer else {}
try:
response = await self._client.get(playlist_url, headers=headers)
response.raise_for_status()
body = response.text
# Playlist maître → on suit la variante de plus haut débit
variant = _best_variant(body, playlist_url)
if variant and variant != playlist_url:
response = await self._client.get(variant, headers=headers)
response.raise_for_status()
body = response.text
durations = [float(m) for m in _EXTINF_RE.findall(body)]
return sum(durations) if durations else None
except (httpx.HTTPError, ValueError) as exc:
logger.warning("Durée HLS indéterminée pour %s : %s", playlist_url, exc)
return None
async def _report_progress(
self, download_id: int, downloaded: int, total: int | None, started: float
) -> None:
elapsed = time.monotonic() - started
self._progress[download_id] = {
"downloaded_bytes": downloaded,
"total_bytes": total,
"speed_bps": int(downloaded / elapsed) if elapsed > 0 else 0,
}
await self._emit(download_id)
async def _fail(self, download_id: int, exc: Exception, downloaded: int = 0) -> None:
await db.execute(
"UPDATE downloads SET status = 'failed', error = ?, "
"downloaded_bytes = ?, updated_at = datetime('now') WHERE id = ?",
(str(exc)[:500], downloaded, download_id),
)
await self._emit(download_id)
async def _finish(self, download_id: int, size: int) -> None:
await db.execute(
"UPDATE downloads SET status = 'done', total_bytes = ?, downloaded_bytes = ?, "
"updated_at = datetime('now') WHERE id = ?",
(size, size, download_id),
)
await self._emit(download_id)
logger.info("Téléchargement terminé (id=%s, %d octets)", download_id, size)
# ------------------------------------------------------------ helpers
async def _get_row(self, download_id: int) -> Any:
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
if row is None:
raise KeyError(f"Téléchargement introuvable : {download_id}")
return row
async def _set_status(self, download_id: int, status: str) -> None:
await db.execute(
"UPDATE downloads SET status = ?, updated_at = datetime('now') WHERE id = ?",
(status, download_id),
)
@staticmethod
def _part_path(file_path: str | None) -> Path:
name = file_path or "video"
return get_settings().download_dir / (name + ".part")
@staticmethod
def _guess_extension(url: str) -> str:
match = re.search(r"\.(mp4|mkv|webm|avi|m3u8)(?:\?|$)", url)
ext = match.group(1) if match else "mp4"
return ".mp4" if ext == "m3u8" else f".{ext}"
def _to_dict(self, row: Any, duplicate: bool = False) -> dict:
data = dict(row)
live = self._progress.get(data["id"], {})
downloaded = live.get("downloaded_bytes", data["downloaded_bytes"])
total = live.get("total_bytes", data["total_bytes"])
speed = live.get("speed_bps", 0)
percent = round(downloaded / total * 100, 1) if total else None
eta = int((total - downloaded) / speed) if total and speed else None
data.update(
downloaded_bytes=downloaded,
total_bytes=total,
percent=percent,
speed_bps=speed,
eta_seconds=eta,
status_label=_STATUS_LABELS.get(data["status"], data["status"]),
duplicate=duplicate,
)
return data
download_manager = DownloadManager()
+198
View File
@@ -0,0 +1,198 @@
"""Enrichissement de métadonnées via l'API Kitsu (https://kitsu.io/api/edge).
Fusion intelligente : seuls les champs manquants d'un TitleDetails sont complétés.
Les échecs (réseau, cache indisponible) sont loggés en warning et jamais remontés
au caller — la fiche d'origine est retournée inchangée (dégradation gracieuse).
"""
import json
import logging
import re
import sqlite3
import httpx
from app.config import get_settings
from app.db import db
from app.scrapers.base import TitleDetails
logger = logging.getLogger(__name__)
_NOISE_WORDS_RE = re.compile(
r"\b(?:VOSTFR\d*|VOST|VF[IV]?|TRUEFRENCH|FRENCH|MULTI|SUBFR?)\b", re.IGNORECASE
)
_TRAILING_SEASON_RE = re.compile(r"[\s\-–—:.]*\s*(?:saison|season)\s*\d+\s*$", re.IGNORECASE)
_TRAILING_CODE_RE = re.compile(r"[\s\-–—:.]*\s*S\d+(?:E\d+)?\s*$", re.IGNORECASE)
def normalize_title(title: str) -> str:
"""Nettoie un titre de scraping avant recherche Kitsu (bruit, saison, tirets)."""
cleaned = _NOISE_WORDS_RE.sub(" ", title)
cleaned = _TRAILING_SEASON_RE.sub("", cleaned)
cleaned = _TRAILING_CODE_RE.sub("", cleaned)
cleaned = re.sub(r"\s*[-–—_]+\s*", " ", cleaned)
cleaned = re.sub(r"\s+", " ", cleaned)
return cleaned.strip(" -–—:.")
class KitsuService:
"""Recherche et cache des métadonnées anime depuis l'API Kitsu."""
async def search_anime(self, title: str) -> dict | None:
"""Recherche le meilleur match Kitsu pour un titre (None si échec/rien trouvé)."""
settings = get_settings()
query = normalize_title(title)
if not query:
return None
try:
async with httpx.AsyncClient(
timeout=settings.http_timeout,
headers={
"User-Agent": settings.user_agent,
"Accept": "application/vnd.api+json",
},
) as client:
response = await client.get(
f"{settings.kitsu_base_url}/anime",
params={"filter[text]": query, "page[limit]": 5, "include": "genres"},
)
response.raise_for_status()
payload = response.json()
except (httpx.HTTPError, ValueError) as exc:
logger.warning("Recherche Kitsu échouée pour %r : %s", title, exc)
return None
return self._pick_best_match(payload, query)
async def enrich(self, details: TitleDetails) -> TitleDetails:
"""Complète les champs manquants de details via Kitsu (jamais episodes/url/source)."""
query = normalize_title(details.title)
cache_key = f"kitsu:anime:{query.casefold()}"
result = await self.get_cached(cache_key)
from_cache = result is not None
if result is None:
result = await self.search_anime(details.title)
if result is not None:
await self.set_cached(cache_key, result)
if result is None:
logger.info("Aucune métadonnée Kitsu pour %r — fiche inchangée", details.title)
return details
attrs = result.get("attributes", {})
if not details.synopsis:
details.synopsis = attrs.get("synopsis")
if not details.image_url:
poster = attrs.get("posterImage") or {}
details.image_url = poster.get("large") or poster.get("medium")
if not details.banner_url:
cover = attrs.get("coverImage") or {}
details.banner_url = cover.get("large") or cover.get("original")
if not details.genres:
details.genres = list(result.get("genres", []))
if details.rating is None:
details.rating = self._to_rating_10(attrs.get("averageRating"))
if details.year is None:
details.year = self._extract_year(attrs.get("startDate"))
if details.episode_count is None:
details.episode_count = attrs.get("episodeCount")
logger.debug(
"Fiche %r enrichie via Kitsu (cache=%s, id=%s)",
details.title,
from_cache,
result.get("id"),
)
return details
async def get_cached(self, cache_key: str) -> dict | None:
"""Retourne le payload en cache s'il existe et est frais (TTL), sinon None."""
ttl_hours = get_settings().metadata_cache_ttl_hours
try:
row = await db.fetchone(
"SELECT payload FROM metadata_cache "
f"WHERE cache_key = ? AND fetched_at > datetime('now', '-{ttl_hours} hours')",
(cache_key,),
)
except (RuntimeError, sqlite3.Error) as exc:
logger.warning(
"Cache métadonnées illisible (%s), continuation sans cache : %s", cache_key, exc
)
return None
if row is None:
return None
try:
return json.loads(row["payload"])
except (TypeError, ValueError) as exc:
logger.warning("Cache métadonnées corrompu pour %s, re-fetch : %s", cache_key, exc)
return None
async def set_cached(self, cache_key: str, payload: dict) -> None:
"""Écrit/upsert une entrée de cache (échec non bloquant, warning seulement)."""
try:
await db.execute(
"INSERT INTO metadata_cache (cache_key, payload, fetched_at) "
"VALUES (?, ?, datetime('now')) "
"ON CONFLICT(cache_key) DO UPDATE SET "
"payload = excluded.payload, fetched_at = excluded.fetched_at",
(cache_key, json.dumps(payload, ensure_ascii=False)),
)
except (RuntimeError, sqlite3.Error) as exc:
logger.warning(
"Cache métadonnées non écrit (%s), continuation sans cache : %s", cache_key, exc
)
@staticmethod
def _pick_best_match(payload: dict, query: str) -> dict | None:
items = [item for item in payload.get("data", []) if item.get("type") == "anime"]
if not items:
return None
genre_names = {
obj.get("id"): obj.get("attributes", {}).get("name")
for obj in payload.get("included", [])
if obj.get("type") == "genres"
}
def titles_of(item: dict) -> set[str]:
attrs = item.get("attributes", {})
titles = attrs.get("titles") or {}
return {
t.strip().casefold()
for t in (attrs.get("canonicalTitle"), titles.get("en"), titles.get("en_jp"))
if t
}
def popularity(item: dict) -> tuple[int, float]:
attrs = item.get("attributes", {})
return (
attrs.get("userCount") or 0,
float(attrs.get("averageRating") or 0),
)
best = next((item for item in items if query.casefold() in titles_of(item)), None)
if best is None:
best = max(items, key=popularity)
genre_ids = best.get("relationships", {}).get("genres", {}).get("data") or []
genres = [genre_names[g["id"]] for g in genre_ids if g.get("id") in genre_names]
return {"id": best.get("id"), "attributes": best.get("attributes", {}), "genres": genres}
@staticmethod
def _to_rating_10(average_rating: str | None) -> float | None:
"""Kitsu note sur 100 (chaîne) → note sur 10 arrondie à 1 décimale."""
if average_rating is None:
return None
try:
return round(float(average_rating) / 10, 1)
except (TypeError, ValueError):
return None
@staticmethod
def _extract_year(start_date: str | None) -> int | None:
if not start_date:
return None
try:
return int(str(start_date)[:4])
except ValueError:
return None
+72
View File
@@ -0,0 +1,72 @@
"""Paramètres persistés en DB (activation des sources, réglages UI…)."""
import json
import logging
import secrets
from app.db import db
logger = logging.getLogger(__name__)
async def get_setting(key: str, default: object = None) -> object:
row = await db.fetchone("SELECT value FROM settings WHERE key = ?", (key,))
if row is None:
return default
try:
return json.loads(row["value"])
except json.JSONDecodeError:
logger.warning("Paramètre %r corrompu, valeur par défaut utilisée", key)
return default
async def set_setting(key: str, value: object) -> None:
await db.execute(
"INSERT INTO settings (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, json.dumps(value, ensure_ascii=False)),
)
async def is_source_enabled(name: str) -> bool:
return bool(await get_setting(f"source:{name}:enabled", True))
async def set_source_enabled(name: str, enabled: bool) -> None:
await set_setting(f"source:{name}:enabled", enabled)
logger.info("Source %s %s", name, "activée" if enabled else "désactivée")
# ---------------------------------------------------------------- intégrations *arr
TORZNAB_APIKEY_KEY = "torznab:apikey"
SONARR_URL_KEY = "sonarr:url"
SONARR_APIKEY_KEY = "sonarr:apikey"
async def get_torznab_apikey() -> str:
"""Clé API Torznab (générée au premier appel, persistée en DB)."""
key = await get_setting(TORZNAB_APIKEY_KEY)
if not isinstance(key, str) or not key:
key = secrets.token_hex(16)
await set_setting(TORZNAB_APIKEY_KEY, key)
logger.info("Clé API Torznab générée")
return key
async def reset_torznab_apikey() -> str:
key = secrets.token_hex(16)
await set_setting(TORZNAB_APIKEY_KEY, key)
logger.info("Clé API Torznab régénérée")
return key
async def get_sonarr_config() -> dict[str, str]:
return {
"url": await get_setting(SONARR_URL_KEY, ""),
"apikey": await get_setting(SONARR_APIKEY_KEY, ""),
}
async def set_sonarr_config(url: str, apikey: str) -> None:
await set_setting(SONARR_URL_KEY, url.rstrip("/"))
await set_setting(SONARR_APIKEY_KEY, apikey.strip())
logger.info("Configuration Sonarr enregistrée (%s)", url)
+152
View File
@@ -0,0 +1,152 @@
"""Client Sonarr (API v3) — personnalisation de « Pour toi ».
OhmStreaming lit ce qui est téléchargé/grabé sur Sonarr :
- `/api/v3/history` (événements grab/import) → titres récemment obtenus ;
- `/api/v3/series` → genres de chaque série (fournis par Sonarr lui-même).
Ces données alimentent le profil de genres des recommandations et la liste
d'exclusion (le déjà-possédé sur Sonarr n'est pas re-recommandé).
Toujours tolérant aux pannes : Sonarr absent/non configuré → profil vide,
la découverte continue de fonctionner avec l'historique local.
"""
import logging
import time
import httpx
from app.services.kitsu import normalize_title
from app.services.settings import get_sonarr_config
logger = logging.getLogger(__name__)
_HISTORY_PAGE_SIZE = 100
_MAX_TITLES = 30 # titres Sonarr récents analysés
_PROFILE_TTL = 900.0 # profil Sonarr re-quantifié au bout de 15 min
_REQUEST_TIMEOUT = 15.0
class SonarrService:
"""Profil de consommation Sonarr : titres téléchargés + genres associés."""
def __init__(self) -> None:
self._cache: tuple[float, tuple[set[str], dict[str, int]]] | None = None
async def _config(self) -> tuple[str, str]:
"""(url, apikey) — lu en DB à chaque usage (lecture SQLite négligeable)."""
config = await get_sonarr_config()
return config["url"], config["apikey"]
def invalidate(self) -> None:
"""Force le recalcul du profil au prochain appel (réglages modifiés)."""
self._cache = None
async def _client(self) -> httpx.AsyncClient | None:
url, apikey = await self._config()
if not url or not apikey:
return None
return httpx.AsyncClient(
base_url=url,
timeout=_REQUEST_TIMEOUT,
headers={"X-Api-Key": apikey},
)
async def _get(self, path: str, params: dict | None = None) -> dict | list | None:
client = await self._client()
if client is None:
return None
try:
async with client:
response = await client.get(path, params=params)
response.raise_for_status()
return response.json()
except (httpx.HTTPError, ValueError) as exc:
logger.warning("Sonarr %s KO : %s", path, exc)
return None
# ------------------------------------------------------------ profil
async def profile(self) -> tuple[set[str], dict[str, int]]:
"""(titres possédés normalisés, comptage de genres) — vide si non configuré."""
if self._cache is not None and self._cache[0] > time.monotonic():
return self._cache[1]
titles = await self.downloaded_titles()
genres_by_title = await self.series_genres()
owned: set[str] = set()
genre_counts: dict[str, int] = {}
for title in titles:
normalized = normalize_title(title).casefold()
if normalized:
owned.add(normalized)
for genre in genres_by_title.get(normalized, []):
genre_counts[genre] = genre_counts.get(genre, 0) + 1
result = (owned, genre_counts)
self._cache = (time.monotonic() + _PROFILE_TTL, result)
return result
async def downloaded_titles(self) -> list[str]:
"""Titres récemment grabés/importés sur Sonarr (les plus récents d'abord)."""
payload = await self._get(
"/api/v3/history",
{
"page": 1,
"pageSize": _HISTORY_PAGE_SIZE,
"eventType": 1, # grab ; les imports (3) suivent le même titre
"sortKey": "date",
"sortDir": "desc",
},
)
if not isinstance(payload, dict):
return []
titles: list[str] = []
seen: set[str] = set()
for record in payload.get("records", []):
title = record.get("series", {}).get("title")
if not title or title in seen:
continue
seen.add(title)
titles.append(title)
if len(titles) >= _MAX_TITLES:
break
return titles
async def series_genres(self) -> dict[str, list[str]]:
"""Genres par série, clé = titre normalisé (minuscule)."""
payload = await self._get("/api/v3/series")
if not isinstance(payload, list):
return {}
return {
normalize_title(s.get("title", "")).casefold(): [
g for g in s.get("genres", []) if isinstance(g, str)
]
for s in payload
if s.get("title")
}
# ------------------------------------------------------------ admin
async def test_connection(self) -> dict:
"""Diagnostic admin : version Sonarr, séries, grabs récents."""
url, apikey = await self._config()
if not url or not apikey:
return {"ok": False, "detail": "URL ou clé API manquante"}
payload = await self._get("/api/v3/system/status")
if payload is None:
return {"ok": False, "detail": "Connexion impossible — vérifiez URL et clé"}
series = await self._get("/api/v3/series")
titles = await self.downloaded_titles()
return {
"ok": True,
"detail": (
f"Sonarr {payload.get('version', '?')} — "
f"{len(series) if isinstance(series, list) else 0} séries, "
f"{len(titles)} saisies récentes"
),
}
sonarr = SonarrService()
+304
View File
@@ -0,0 +1,304 @@
"""Indexeur Torznab/Newznab : expose le catalogue OhmStreaming à Sonarr/Prowlarr.
OhmStreaming devient une source d'indexeur à part entière : Sonarr (ou Prowlarr,
qui relaiera vers Radarr/Lidarr…) interroge `/torznab/api` comme n'importe quel
indexer Jackett. Chaque « release » correspond à un épisode résolu depuis les
sources de scraping ; le grab (`/torznab/download`) déclenche l'extraction puis
l'ajout dans la file de téléchargements OhmStreaming — le fichier arrive donc
dans la bibliothèque locale, comme un téléchargement manuel.
Formats :
- `t=caps` → capacités du serveur (catégories TV/Anime, paramètres supportés)
- `t=tvsearch` → recherche par série + saison + épisode (Sonarr)
- `t=search` → recherche libre (Prowlarr, recherche manuelle)
"""
import asyncio
import logging
import time
from dataclasses import dataclass
from datetime import UTC, datetime
from email.utils import format_datetime
from urllib.parse import urlencode
from app.scrapers.base import (
Episode,
ScrapeError,
SourceScraper,
VideoLink,
all_sources,
import_all_scrapers,
resolve_hoster,
)
from app.services.downloads import download_manager
from app.services.settings import is_source_enabled
logger = logging.getLogger(__name__)
import_all_scrapers()
TORZNAB_NS = "http://torznab.com/schemas/2015/feed"
CAT_TV = "5000"
CAT_ANIME = "5070"
_SIZE_ESTIMATE = 400 * 1024 * 1024 # estimation affichée (~400 Mo/épisode)
_MAX_SERIES_PER_SOURCE = 2 # fiches explorées par source lors d'une recherche
_MAX_RELEASES = 100 # borne du flux retourné
_SEARCH_TIMEOUT = 40.0 # scraping lent : garde-fou par source
_EPISODE_CACHE_TTL = 600.0 # listes d'épisodes re-scrapées au bout de 10 min
@dataclass
class Release:
"""Un épisode vu comme une release par Sonarr/Prowlarr."""
series: str
season: int
ep: int
source: str
source_id: str
episode_url: str
@property
def sonarr_title(self) -> str:
return f"{self.series} S{self.season:02d}E{self.ep:02d} VOSTFR WEB-DL"
def _bencode(value) -> bytes:
if isinstance(value, int):
return f"i{value}e".encode()
if isinstance(value, str):
raw = value.encode()
return f"{len(raw)}:".encode() + raw
if isinstance(value, bytes):
return f"{len(value)}:".encode() + value
if isinstance(value, dict):
return b"d" + b"".join(
_bencode(k) + _bencode(v) for k, v in sorted(value.items())
) + b"e"
if isinstance(value, list):
return b"l" + b"".join(_bencode(v) for v in value) + b"e"
raise TypeError(f"type non encodable en bencode : {type(value)!r}")
def torrent_stub(announce_url: str, name: str) -> bytes:
"""Fichier .torrent minimal (le vrai téléchargement est fait par OhmStreaming)."""
return _bencode(
{
"announce": announce_url,
"created by": "OhmStreaming",
"comment": name,
"info": {
"name": name + ".mp4",
"length": 0,
"piece length": 32768,
"pieces": b"\x00" * 20,
},
}
)
class TorznabService:
"""Recherche multi-sources mappée en releases Torznab + grab → file interne."""
def __init__(self) -> None:
self._episodes_cache: dict[tuple[str, str], tuple[float, list[Episode]]] = {}
# ------------------------------------------------------------ recherche
async def tvsearch(self, q: str, season: int | None = None, ep: int | None = None) -> list[Release]:
"""Recherche type Sonarr : série (+ saison/épisode optionnels)."""
releases: list[Release] = []
for source in await self._enabled_sources():
outcomes = await self._search_source(source, q)
for result in outcomes[:_MAX_SERIES_PER_SOURCE]:
try:
episodes = await self._episodes_of(source, result.source_id)
except (ScrapeError, TimeoutError):
continue
releases.extend(
self._releases_for(result.title, source, result.source_id, episodes, season, ep)
)
if len(releases) >= _MAX_RELEASES:
return releases[:_MAX_RELEASES]
return releases
async def search(self, q: str) -> list[Release]:
"""Recherche libre : tous les épisodes des fiches trouvées."""
return await self.tvsearch(q)
async def _search_source(self, source: SourceScraper, q: str):
try:
return await asyncio.wait_for(source.search(q), timeout=_SEARCH_TIMEOUT)
except (ScrapeError, TimeoutError) as exc:
logger.warning("Torznab : recherche %s KO pour %r : %s", source.name, q, exc)
return []
async def _enabled_sources(self) -> list[SourceScraper]:
return [s for s in all_sources() if await is_source_enabled(s.name)]
async def _episodes_of(self, source: SourceScraper, source_id: str) -> list[Episode]:
"""Liste d'épisodes d'une fiche, avec cache mémoire TTL."""
key = (source.name, source_id)
cached = self._episodes_cache.get(key)
if cached is not None and cached[0] > time.monotonic():
return cached[1]
episodes = await asyncio.wait_for(source.list_episodes(source_id), timeout=_SEARCH_TIMEOUT)
self._episodes_cache[key] = (time.monotonic() + _EPISODE_CACHE_TTL, episodes)
return episodes
def _releases_for(
self,
series: str,
source: SourceScraper,
source_id: str,
episodes: list[Episode],
season: int | None,
ep: int | None,
) -> list[Release]:
"""Filtre les épisodes selon saison/épisode demandés (entiers uniquement)."""
out = []
for item in episodes:
if item.number != int(item.number): # OAV 2.5 → ignorée (inparseable Sonarr)
continue
if season is not None and item.season != season:
continue
if ep is not None and int(item.number) != ep:
continue
out.append(
Release(
series=series,
season=item.season,
ep=int(item.number),
source=source.name,
source_id=source_id,
episode_url=item.url,
)
)
return out
# ------------------------------------------------------------ grab
async def grab(self, source: str, source_id: str, season: int, ep: int, series: str) -> dict:
"""Résout l'épisode (embed → vidéo directe) puis l'ajoute à la file interne.
Retourne le dict du téléchargement (existant si doublon actif).
Lève ScrapeError si introuvable ou qu'aucun hébergeur n'a répondu.
"""
from app.scrapers.base import get_source
scraper = get_source(source)
episodes = await self._episodes_of(scraper, source_id)
match = next(
(
e
for e in episodes
if e.season == season and e.number == int(ep)
),
None,
)
if match is None:
raise ScrapeError(f"Épisode S{season:02d}E{ep:02d} introuvable sur {source}")
link = await self._resolve_video(scraper, match.url)
title = f"{series} S{season:02d}E{ep:02d}"
result = await download_manager.enqueue(link.url, match.url, title)
if link.is_hls or link.headers.get("Referer"):
result["note"] = "HLS/proxy : OhmStreaming gère le téléchargement via ffmpeg"
logger.info("Torznab grab %s → download id=%s", title, result.get("id"))
return result
async def _resolve_video(self, scraper: SourceScraper, episode_url: str) -> VideoLink:
"""Chaîne complète : page épisode → embeds → première URL directe valide."""
embeds = await asyncio.wait_for(
scraper.extract_embed_links(episode_url), timeout=_SEARCH_TIMEOUT
)
errors: list[str] = []
for embed in embeds:
extractor = resolve_hoster(embed)
if extractor is None:
errors.append(f"hébergeur non supporté : {embed}")
continue
try:
link = await asyncio.wait_for(extractor.extract(embed), timeout=_SEARCH_TIMEOUT)
if link and link.url:
return link
except (ScrapeError, TimeoutError) as exc:
errors.append(str(exc))
raise ScrapeError(f"Aucun hébergeur résolu pour {episode_url} ({'; '.join(errors[:3])})")
# ------------------------------------------------------------ XML
def download_url(self, base_url: str, apikey: str, release: Release) -> str:
query = urlencode(
{
"apikey": apikey,
"source": release.source,
"sid": release.source_id,
"season": release.season,
"ep": release.ep,
"series": release.series,
}
)
return f"{base_url}/torznab/download?{query}"
def caps_xml(self, base_url: str) -> str:
return f"""<?xml version="1.0" encoding="UTF-8"?>
<caps>
<server version="1.0" title="OhmStreaming" url="{base_url}"
email="ohm@localhost" image="{base_url}/static/img/logo.png"/>
<searching>
<search available="yes" supportedParams="q"/>
<tv-search available="yes" supportedParams="q,season,ep"/>
<movie-search available="no" supportedParams=""/>
<audio-search available="no" supportedParams=""/>
</searching>
<categories>
<category id="{CAT_TV}" name="TV">
<subcat id="{CAT_ANIME}" name="Anime"/>
</category>
</categories>
</caps>"""
def results_xml(self, base_url: str, apikey: str, releases: list[Release]) -> str:
from xml.sax.saxutils import escape
items = []
for release in releases:
url = escape(self.download_url(base_url, apikey, release))
pubdate = escape(format_datetime(datetime.now(UTC)))
items.append(f""" <item>
<title>{escape(release.sonarr_title)}</title>
<guid isPermaLink="true">{url}</guid>
<link>{url}</link>
<comments>{escape(release.episode_url)}</comments>
<pubDate>{pubdate}</pubDate>
<category>{CAT_ANIME}</category>
<enclosure url="{url}" length="{_SIZE_ESTIMATE}" type="application/x-bittorrent"/>
<torznab:attr name="seeders" value="1"/>
<torznab:attr name="peers" value="1"/>
<torznab:attr name="downloadvolumefactor" value="0"/>
<torznab:attr name="uploadvolumefactor" value="0"/>
</item>""")
body = "\n".join(items)
return f"""<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:torznab="{TORZNAB_NS}">
<channel>
<title>OhmStreaming</title>
<link>{escape(base_url)}</link>
<description>Indexeur OhmStreaming — animes VOSTFR scrapés en direct</description>
<language>fr-FR</language>
{body}
</channel>
</rss>"""
def error_xml(self, code: int, description: str) -> str:
from xml.sax.saxutils import escape
return (
'<?xml version="1.0" encoding="UTF-8"?>\n'
f'<error code="{code}" description="{escape(description)}"/>'
)
torznab = TorznabService()