v0.1.0 — Réécriture complète de OhmStreaming

Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree,
French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement
HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration,
découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
This commit is contained in:
Roman
2026-09-22 10:05:47 +00:00
commit 41566ab5fb
66 changed files with 8895 additions and 0 deletions
View File
+192
View File
@@ -0,0 +1,192 @@
"""Contrats et registres des scrapers.
Architecture à 2 registres :
- sources (sites de catalogues animes/séries) : recherche, détails, épisodes, liens embed
- hébergeurs vidéo : résolution d'une URL embed → URL directe du fichier
Ajouter une source = écrire une classe qui implémente le contrat et la décorer
@register_source / @register_hoster.
"""
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
logger = logging.getLogger(__name__)
class ScrapeError(Exception):
"""Échec de scraping/extraction — jamais silencieux, toujours journalisé et remonté."""
# ---------------------------------------------------------------- modèles
@dataclass
class SearchResult:
source: str
source_id: str # identifiant stable chez la source (slug, id…)
title: str
url: str
image_url: str | None = None
media_type: str = "anime" # anime | serie | film
@dataclass
class Episode:
number: float # 1, 2, 2.5 pour les OAV intermédiaires
title: str | None
url: str # page de l'épisode chez la source
season: int = 1
@dataclass
class TitleDetails:
source: str
source_id: str
title: str
url: str
synopsis: str | None = None
image_url: str | None = None
banner_url: str | None = None
genres: list[str] = field(default_factory=list)
rating: float | None = None
year: int | None = None
episode_count: int | None = None
episodes: list[Episode] = field(default_factory=list)
media_type: str = "anime"
@dataclass
class VideoLink:
"""Lien vidéo résolu (URL directe du fichier, lisible par un lecteur)."""
url: str
hoster: str
quality: str | None = None
headers: dict[str, str] = field(default_factory=dict) # ex. Referer obligatoire
is_hls: bool = False
# ---------------------------------------------------------------- contrats
class SourceScraper(ABC):
"""Contrat d'un site catalogue (animes, séries…)."""
name: str # identifiant unique, ex. "vostfree"
label: str # nom affiché
base_url: str
media_types: tuple[str, ...] = ("anime",)
@abstractmethod
async def search(self, query: str) -> list[SearchResult]: ...
@abstractmethod
async def get_details(self, source_id: str) -> TitleDetails: ...
@abstractmethod
async def list_episodes(self, source_id: str) -> list[Episode]: ...
@abstractmethod
async def extract_embed_links(self, episode_url: str) -> list[str]:
"""URLs des lecteurs embarqués trouvés sur la page d'un épisode."""
...
async def latest(self) -> list[SearchResult]:
"""Ajouts récents du catalogue (découverte). Vide si la source ne le supporte pas."""
return []
class HosterExtractor(ABC):
"""Contrat d'un hébergeur vidéo : embed URL → URL directe."""
name: str
domains: tuple[str, ...] = ()
def can_handle(self, url: str) -> bool:
return any(d in url for d in self.domains)
@abstractmethod
async def extract(self, embed_url: str) -> VideoLink: ...
# ---------------------------------------------------------------- registres
_source_registry: dict[str, type[SourceScraper]] = {}
_hoster_registry: list[type[HosterExtractor]] = []
def register_source(cls: type[SourceScraper]) -> type[SourceScraper]:
if cls.name in _source_registry:
raise ValueError(f"Source déjà enregistrée : {cls.name}")
_source_registry[cls.name] = cls
logger.debug("Source enregistrée : %s", cls.name)
return cls
def register_hoster(cls: type[HosterExtractor]) -> type[HosterExtractor]:
_hoster_registry.append(cls)
logger.debug("Hébergeur enregistré : %s", cls.name)
return cls
_source_instances: dict[str, SourceScraper] = {}
_hoster_instances: list[HosterExtractor] | None = None
def get_source(name: str) -> SourceScraper:
if name not in _source_instances:
cls = _source_registry.get(name)
if cls is None:
raise ScrapeError(f"Source inconnue : {name}")
_source_instances[name] = cls()
return _source_instances[name]
def all_sources() -> list[SourceScraper]:
return [get_source(name) for name in _source_registry]
def resolve_hoster(url: str) -> HosterExtractor | None:
"""Trouve l'extracteur capable de traiter une URL embed (None → générique)."""
global _hoster_instances
if _hoster_instances is None:
_hoster_instances = [cls() for cls in _hoster_registry]
for extractor in _hoster_instances:
if extractor.can_handle(url):
return extractor
return None
def import_all_scrapers() -> None:
"""Importe tous les modules pour déclencher les décorateurs d'enregistrement."""
import importlib
import pkgutil
import app.scrapers.hosters as hosters_pkg
import app.scrapers.sources as sources_pkg
for pkg in (sources_pkg, hosters_pkg):
for mod in pkgutil.iter_modules(pkg.__path__):
importlib.import_module(f"{pkg.__name__}.{mod.name}")
# ---------------------------------------------------------------- format interne
INTERNAL_SEP = "|"
def encode_internal_url(video_url: str, page_url: str, title: str) -> str:
"""Format interne `video_url|page_url|titre` pour transporter le contexte."""
for part in (video_url, page_url, title):
if INTERNAL_SEP in part:
raise ValueError(f"Caractère interdit '{INTERNAL_SEP}' dans : {part!r}")
return INTERNAL_SEP.join([video_url, page_url, title])
def decode_internal_url(value: str) -> tuple[str, str, str]:
parts = value.split(INTERNAL_SEP)
if len(parts) != 3 or not parts[0]:
raise ValueError(f"URL interne invalide : {value!r}")
return parts[0], parts[1], parts[2]
+31
View File
@@ -0,0 +1,31 @@
"""Chargement des configurations de scraping externalisées (sélecteurs YAML).
Permet de réparer un site cassé en modifiant un fichier de config sans toucher
au code. Chaque source peut avoir un fichier `configs/<nom_source>.yaml`.
"""
import logging
from functools import lru_cache
from typing import Any
import yaml
from app.config import get_settings
logger = logging.getLogger(__name__)
@lru_cache
def load_scraper_config(source_name: str) -> dict[str, Any]:
path = get_settings().scrapers_config_dir / f"{source_name}.yaml"
if not path.exists():
logger.debug("Pas de config externe pour %s (%s)", source_name, path)
return {}
try:
with path.open(encoding="utf-8") as fh:
config = yaml.safe_load(fh) or {}
logger.info("Config de scraping chargée pour %s", source_name)
return config
except yaml.YAMLError as exc:
logger.error("Config %s invalide : %s", path, exc)
return {}
+29
View File
@@ -0,0 +1,29 @@
# Sélecteurs de scraping French-Manga (DataLife Engine + API AJAX maison)
# Structure vérifiée en live sur https://w16.french-manga.net
search:
endpoint: "/engine/ajax/search.php" # POST query=<q>&page=1 (la recherche DLE native est cassée)
result: ".search-item" # lien dans l'attribut onclick
title: ".search-title"
image: ".search-poster img"
details:
config_el: "#serie-config" # data-title / data-news-id
poster: ".fposter img"
synopsis: ".flist .fdesc p"
release: ".facts .release" # année
genres: ".facts .genres a"
episode_badge: ".short-meta.short-label" # "Ep 32 sur 32"
episodes:
api: "/engine/ajax/manga_episodes_api.php" # ?id=<newsid> → JSON {vf, vostfr, info}
versions: ["vostfr", "vf"] # priorité des versions
fragment_prefix: "ep" # URL épisode : <fiche>#ep=<version>-<numéro>
# Nouveautés — section « Récemment mises à jour » de la page d'accueil épinglée
latest:
path: "/manga-streaming-1/"
item: ".short"
link: "a.short-poster" # href = index.php?newsid=<id>, alt = titre
title: ".short-title"
image: "a.short-poster img"
+47
View File
@@ -0,0 +1,47 @@
# Sélecteurs de scraping Vostfree (DataLife Engine) — surchargeables sans toucher au code.
# Structure vérifiée en live sur https://ipv4.vostfree.ws
search:
result: "div.search-result" # bloc d'un résultat de recherche
link: "div.title a" # lien + titre
image: "span.image img" # poster
genres: "ul.additional li" # "Genre:...", "Anneé:..." (détection film)
details:
title: "h1"
poster: ".slide-poster img"
synopsis: ".slide-desc" # les .cast internes sont retirés
genres: '.slide-top li.right a[href*="/genre/"]'
episode_badge: ".slide-poster .year" # ex. "Episode 293"
season_li: "ul.slide-top li" # ex. "Saison: 01"
episodes:
option: "select.new_player_selector option" # value="buttons_N" → Episode NN
button: "div.button_box" # repli si pas de sélecteur
content_prefix: "content_" # #player_M → #content_player_M
# Nouveautés — page « Animes VOSTFR récemment ajoutés »
latest:
path: "/animes-vostfr-recement-ajoutees.html"
item: "div.movie-poster"
link: ".play a" # href = fiche, alt = titre
image: "span.image img"
# class CSS du lecteur → template d'URL ({} = valeur de #content_player_M)
# chaîne vide = la valeur est déjà une URL complète
players:
new_player_vip: ""
new_player_moevideo: ""
new_player_sibnet: "https://video.sibnet.ru/shell.php?videoid={}"
new_player_netu: "https://video.sibnet.ru/shell.php?videoid={}"
new_player_uqload: "https://uqload.com/embed-{}.html"
new_player_mp4: "https://www.mp4upload.com/embed-{}.html"
new_player_fembed: "https://www.fembed.com/v/{}"
new_player_mytv: "https://www.myvi.top/embed/{}"
new_player_myvi: "https://myvi.ru/player/embed/html/{}"
new_player_rutube: "https://rutube.ru/play/embed/{}"
new_player_ok: "https://ok.ru/video/{}"
new_player_mail2: "https://my.mail.ru/video/embed/{}"
new_player_rapids: "https://rapidstream.co/embed-{}.html"
new_player_gtv: "https://iframedream.com/embed/{}.html"
new_player_cloudvideo: "https://cloudvideo.tv/embed-{}.html"
new_player_uptostream: "https://uptostream.com/iframe/{}"
View File
+40
View File
@@ -0,0 +1,40 @@
"""Décompresseur pour le JavaScript packé « p,a,c,k,e,d » (Dean Edwards).
Utilisé par plusieurs hébergeurs vidéo (Uqload, VidMoly…) pour masquer
l'URL du lecteur : le HTML contient `eval(function(p,a,c,k,e,d){...}(...))`.
"""
import re
_DIGITS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
_PACKED_RE = re.compile(
r"\}\('(?P<payload>.*?)',(?P<a>\d+),(?P<c>\d+),'(?P<keys>.*?)'\.split\('\|'\)",
re.DOTALL,
)
def _base_n(number: int, base: int) -> str:
if number == 0:
return "0"
chars = []
while number:
number, rest = divmod(number, base)
chars.append(_DIGITS[rest])
return "".join(reversed(chars))
def unpack_packed_js(text: str) -> str | None:
"""Retourne le JS dépacké si `text` contient un bloc packé, sinon None."""
match = _PACKED_RE.search(text)
if not match:
return None
payload = match.group("payload").replace("\\\\", "\\").replace("\\'", "'")
base, count = int(match.group("a")), int(match.group("c"))
keys = match.group("keys").split("|")
for index in range(count - 1, -1, -1):
token = _base_n(index, base)
if keys[index]:
replacement = keys[index].replace("\\", "\\\\")
payload = re.sub(rf"\b{re.escape(token)}\b", replacement, payload)
return payload
+50
View File
@@ -0,0 +1,50 @@
"""Extracteur Luluvid/Luluvdo (luluvdo.com, luluvid.com, lulustream.com) — embed → m3u8.
Famille StreamSB : la page embed configure jwplayer dans du JS packé
(p,a,c,k,e,d) avec `sources:[{file:"https://.../master.m3u8?t=<token>"}]`.
Le token CDN (tnmr.org…) est lié à l'IP **et à l'User-Agent** de la requête
d'embed : la VideoLink doit donc renvoyer le User-Agent du socle pour que le
téléchargement passe — sans lui, le CDN répond 403.
"""
import logging
import re
from urllib.parse import urlparse
from app.config import get_settings
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
from app.scrapers.hosters._packer import unpack_packed_js
from app.scrapers.http import fetch
logger = logging.getLogger(__name__)
_PATTERNS = (
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?file\s*:\s*["\'](?P<url>[^"\']+\.m3u8[^"\']*)["\']'),
re.compile(r'file\s*:\s*["\'](?P<url>https?://[^"\']+?\.m3u8[^"\']*)["\']'),
re.compile(r'["\'](?P<url>https?://[^"\']*?\.m3u8(?:\?[^"\']*)?)["\']'),
)
@register_hoster
class LuluvdoExtractor(HosterExtractor):
name = "luluvdo"
domains = ("luluvdo.com", "luluvid.com", "lulustream.com")
async def extract(self, embed_url: str) -> VideoLink:
html = await fetch(embed_url)
for content in (html, unpack_packed_js(html) or ""):
for pattern in _PATTERNS:
for match in pattern.finditer(content):
url = match.group("url")
if url.startswith("http"):
return VideoLink(
url=url,
hoster=self.name,
headers={
"Referer": f"https://{urlparse(embed_url).hostname}/",
"User-Agent": get_settings().user_agent,
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
},
is_hls=".m3u8" in url,
)
raise ScrapeError(f"luluvdo : URL vidéo introuvable dans {embed_url}")
+51
View File
@@ -0,0 +1,51 @@
"""Extracteur SendVid (sendvid.com) — page embed → mp4 direct.
La page embed expose soit une balise `<source src="...">` ( lecteur HTML5),
soit une variable JS `video_source`. Extraction par regex, sans dépendance JS.
"""
import logging
import re
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
from app.scrapers.hosters._packer import unpack_packed_js
from app.scrapers.http import fetch
logger = logging.getLogger(__name__)
_PATTERNS = (
re.compile(r'<source[^>]+src=["\'](?P<url>[^"\']+)["\']'),
re.compile(r'video_source\s*[:=]\s*["\'](?P<url>[^"\']+)["\']'),
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:mp4|m3u8)(?:\?[^"\']*)?)["\']'),
)
@register_hoster
class SendvidExtractor(HosterExtractor):
name = "sendvid"
domains = ("sendvid.com",)
async def extract(self, embed_url: str) -> VideoLink:
html = await fetch(embed_url)
candidates: list[str] = []
for content in (html, unpack_packed_js(html) or ""):
for pattern in _PATTERNS:
for match in pattern.finditer(content):
candidates.append(match.group("url"))
url = self._pick(candidates)
if not url:
raise ScrapeError(f"sendvid : URL vidéo introuvable dans {embed_url}")
return VideoLink(
url=url,
hoster=self.name,
headers={"Referer": embed_url},
is_hls=".m3u8" in url,
)
@staticmethod
def _pick(candidates: list[str]) -> str | None:
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
for url in video:
if ".mp4" in url:
return url
return video[0] if video else None
+69
View File
@@ -0,0 +1,69 @@
"""Extracteur Sibnet (video.sibnet.ru) — page shell/watch → mp4 direct.
La page embed `shell.php?videoid=N` (ou la page publique `/videoN`) contient
une config jwplayer du type `player.src([{src: "/v/<hash>/<id>.mp4"}])`.
`shell.php` répond 403 depuis certains réseaux : on retombe alors sur la page
publique de la vidéo, qui expose la même config.
"""
import logging
import re
from urllib.parse import urljoin
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
from app.scrapers.http import fetch
logger = logging.getLogger(__name__)
_BASE = "https://video.sibnet.ru"
_VIDEO_ID_RE = re.compile(r"(?:videoid=|/video)(\d+)")
_MP4_PATTERNS = (
re.compile(r'player\.src\(\[\{src:\s*"(?P<url>/v/[^"]+?\.mp4)"'),
re.compile(r'["\'](?P<url>/v/[0-9a-f]{32}/\d+\.mp4)["\']'),
re.compile(r'["\'](?P<url>https?://[^"\']*?/v/[^"\']+?\.mp4)["\']'),
)
@register_hoster
class SibnetExtractor(HosterExtractor):
name = "sibnet"
domains = ("video.sibnet.ru",)
async def extract(self, embed_url: str) -> VideoLink:
match = _VIDEO_ID_RE.search(embed_url)
if not match:
raise ScrapeError(f"sibnet : identifiant vidéo introuvable dans {embed_url}")
video_id = match.group(1)
watch_url = f"{_BASE}/video{video_id}"
pages: list[tuple[str, str]] = []
try:
pages.append(("embed", await fetch(embed_url, retries=0)))
except ScrapeError as exc:
logger.info(
"sibnet : page embed %s inaccessible (%s), essai page publique", embed_url, exc
)
try:
pages.append(("watch", await fetch(watch_url, retries=1)))
except ScrapeError as exc:
logger.error("sibnet : page publique %s inaccessible : %s", watch_url, exc)
for source, html in pages:
url = self._find_mp4(html, source)
if url:
return VideoLink(
url=url,
hoster=self.name,
headers={"Referer": watch_url},
is_hls=False,
)
raise ScrapeError(f"sibnet : URL mp4 introuvable pour la vidéo {video_id}")
def _find_mp4(self, html: str, source: str) -> str | None:
for pattern in _MP4_PATTERNS:
match = pattern.search(html)
if match:
url = urljoin(_BASE + "/", match.group("url"))
logger.debug("sibnet : mp4 trouvé via %s → %s", source, url)
return url
return None
+45
View File
@@ -0,0 +1,45 @@
"""Extracteur Uqload (uqload.to/.co/.com/.io) — page embed → mp4/m3u8.
La page embed contient un jwplayer configuré dans du JS packé
(p,a,c,k,e,d) : `sources:[{file:"https://.../master.m3u8?..."}]`.
On dépacke puis on extrait par regex.
"""
import logging
import re
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
from app.scrapers.hosters._packer import unpack_packed_js
from app.scrapers.http import fetch
logger = logging.getLogger(__name__)
_PATTERNS = (
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?(?:file|src)\s*:\s*["\'](?P<url>[^"\']+)["\']'),
re.compile(r'(?:file|src)\s*:\s*["\'](?P<url>https?://[^"\']+?\.(?:m3u8|mp4)[^"\']*)["\']'),
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:m3u8|mp4)(?:\?[^"\']*)?)["\']'),
)
@register_hoster
class UqloadExtractor(HosterExtractor):
name = "uqload"
domains = ("uqload.to", "uqload.co", "uqload.com", "uqload.io")
async def extract(self, embed_url: str) -> VideoLink:
html = await fetch(embed_url)
candidates: list[str] = []
for content in (html, unpack_packed_js(html) or ""):
for pattern in _PATTERNS:
for match in pattern.finditer(content):
candidates.append(match.group("url"))
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
url = next((u for u in video if ".mp4" in u), None) or (video[0] if video else None)
if not url:
raise ScrapeError(f"uqload : URL vidéo introuvable dans {embed_url}")
return VideoLink(
url=url,
hoster=self.name,
headers={"Referer": embed_url},
is_hls=".m3u8" in url,
)
+44
View File
@@ -0,0 +1,44 @@
"""Extracteur VidMoly (vidmoly.to / vidmoly.me) — page embed → mp4/m3u8.
Le lecteur jwplayer est configuré dans du JS parfois packé (p,a,c,k,e,d) :
`sources: [{file: "...m3u8"}]`. On dépacke si besoin puis on extrait par regex.
"""
import logging
import re
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
from app.scrapers.hosters._packer import unpack_packed_js
from app.scrapers.http import fetch
logger = logging.getLogger(__name__)
_PATTERNS = (
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?file\s*:\s*["\'](?P<url>[^"\']+)["\']'),
re.compile(r'file\s*:\s*["\'](?P<url>https?://[^"\']+?\.(?:m3u8|mp4)[^"\']*)["\']'),
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:m3u8|mp4)(?:\?[^"\']*)?)["\']'),
)
@register_hoster
class VidmolyExtractor(HosterExtractor):
name = "vidmoly"
domains = ("vidmoly.to", "vidmoly.me", "vidmoly.biz", "vidmoly.com")
async def extract(self, embed_url: str) -> VideoLink:
html = await fetch(embed_url)
candidates: list[str] = []
for content in (html, unpack_packed_js(html) or ""):
for pattern in _PATTERNS:
for match in pattern.finditer(content):
candidates.append(match.group("url"))
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
url = next((u for u in video if ".mp4" in u), None) or (video[0] if video else None)
if not url:
raise ScrapeError(f"vidmoly : URL vidéo introuvable dans {embed_url}")
return VideoLink(
url=url,
hoster=self.name,
headers={"Referer": embed_url},
is_hls=".m3u8" in url,
)
+78
View File
@@ -0,0 +1,78 @@
"""Extracteur Vidzy (vidzy.org) — page embed videojs → m3u8.
La page embed ne contient pas l'URL vidéo en clair : le script videojs appelle
une fonction de décodage inline `atob(s)` + reverse + XOR, avec une graine
dérivée du hostname (`somme des codes des caractères & 0xFF`). Si le décodage
échoue, la page sert un leurre `https://s1.fsvid.lol/troll/master.m3u8`
(même valeur pour tous les épisodes) — on le rejette explicitement.
Algorithme (reproduit fidèlement depuis le JS de la page) :
b = base64decode(s) ; a = b[::-1]
r[i] = chr(a[i] ^ ((0x3D + i*89 + H) & 0xFF)) avec H = sum(ord(hostname)) & 0xFF
"""
import base64
import logging
import re
from urllib.parse import urlparse
from app.config import get_settings
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
from app.scrapers.http import fetch
logger = logging.getLogger(__name__)
_B64_RE = re.compile(
r"atob\(s\)(?s:.){0,800}?\}\)\(\"(?P<b64>[A-Za-z0-9+/=]{50,})\"",
)
_CLEAR_PATTERNS = (
re.compile(r'sources\s*:\s*\[\s*\{\s*src\s*:\s*["\'](?P<url>https?://[^"\']+?\.m3u8[^"\']*)["\']'),
re.compile(r'_fsvHls\s*=\s*"(?P<url>https?://[^"]+?\.m3u8[^"]*)"'),
)
_TROLL = "/troll/"
def _decode(b64: str, hostname: str) -> str | None:
try:
data = base64.b64decode(b64)
except (ValueError, TypeError):
return None
seed = sum(ord(c) for c in hostname) & 0xFF
decoded = "".join(chr(b ^ ((0x3D + i * 89 + seed) & 0xFF)) for i, b in enumerate(data[::-1]))
return decoded if decoded.startswith(("http://", "https://")) else None
@register_hoster
class VidzyExtractor(HosterExtractor):
name = "vidzy"
domains = ("vidzy.org",)
async def extract(self, embed_url: str) -> VideoLink:
html = await fetch(embed_url)
hostname = urlparse(embed_url).hostname or ""
for match in _B64_RE.finditer(html):
url = _decode(match.group("b64"), hostname)
if url and _TROLL not in url:
return self._video_link(url, embed_url)
logger.warning("vidzy : décodage atob+XOR sans résultat pour %s", embed_url)
for pattern in _CLEAR_PATTERNS:
for match in pattern.finditer(html):
url = match.group("url")
if _TROLL not in url:
return self._video_link(url, embed_url)
raise ScrapeError(
f"vidzy : URL vidéo introuvable dans {embed_url} (leurre anti-bot ou page modifiée)"
)
def _video_link(self, url: str, embed_url: str) -> VideoLink:
return VideoLink(
url=url,
hoster=self.name,
headers={
"Referer": f"https://{urlparse(embed_url).hostname}/",
"User-Agent": get_settings().user_agent,
},
is_hls=".m3u8" in url,
)
+63
View File
@@ -0,0 +1,63 @@
"""Client HTTP partagé pour le scraping (httpx async, headers navigateur, retries)."""
import asyncio
import logging
import httpx
from bs4 import BeautifulSoup
from app.config import get_settings
from app.scrapers.base import ScrapeError
logger = logging.getLogger(__name__)
_client: httpx.AsyncClient | None = None
def get_client() -> httpx.AsyncClient:
global _client
if _client is None:
settings = get_settings()
_client = httpx.AsyncClient(
timeout=settings.http_timeout,
follow_redirects=True,
headers={
"User-Agent": settings.user_agent,
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
},
)
return _client
async def close_client() -> None:
global _client
if _client is not None:
await _client.aclose()
_client = None
async def fetch(
url: str,
*,
referer: str | None = None,
retries: int = 2,
) -> str:
"""GET d'une page avec retries ; lève ScrapeError en cas d'échec définitif."""
headers = {"Referer": referer} if referer else {}
last_error: Exception | None = None
for attempt in range(retries + 1):
try:
response = await get_client().get(url, headers=headers)
response.raise_for_status()
return response.text
except (httpx.HTTPError, httpx.InvalidURL) as exc:
last_error = exc
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
if attempt < retries:
await asyncio.sleep(1.0 * (attempt + 1))
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
async def fetch_soup(url: str, *, referer: str | None = None) -> BeautifulSoup:
html = await fetch(url, referer=referer)
return BeautifulSoup(html, "lxml")
View File
+313
View File
@@ -0,0 +1,313 @@
"""Source French-Manga (w16.french-manga.net) — animes VF/VOSTFR, moteur DataLife Engine.
Faits structurels (vérifiés en live) :
- La recherche native DLE est cassée (erreur MySQL en GET, « moins de 4 caractères » en POST) ;
le site utilise un endpoint AJAX : POST /engine/ajax/search.php avec `query=<q>&page=1`
→ blocs `.search-item` (lien dans l'attribut onclick).
- Nouveautés : /manga-streaming-1/ (« Récemment mises à jour ») → blocs `.short`
(lien `a.short-poster` href = `index.php?newsid=<id>`, titre `.short-title`, poster img).
- Fiche : `/<id>-<slug>.html` (alias canonique `index.php?newsid=<id>`), métadonnées dans
`.facts`, `.fdesc`, `.fposter` et `#serie-config` (data-title, data-news-id).
- Épisodes et lecteurs : GET /engine/ajax/manga_episodes_api.php?id=<newsid> → JSON
{"vf": {ep: {hoster: url}}, "vostfr": {...}, "info": {ep: {title, poster}}}.
- Pas de page par épisode : l'URL d'épisode est `<fiche>#ep=<version>-<numéro>`.
"""
import asyncio
import json
import logging
import re
from copy import deepcopy
from urllib.parse import urljoin
import httpx
from bs4 import BeautifulSoup, Tag
from app.scrapers.base import (
Episode,
ScrapeError,
SearchResult,
SourceScraper,
TitleDetails,
register_source,
)
from app.scrapers.config_loader import load_scraper_config
from app.scrapers.http import fetch, fetch_soup, get_client
logger = logging.getLogger(__name__)
DEFAULT_CONFIG: dict = {
"search": {
"endpoint": "/engine/ajax/search.php",
"result": ".search-item",
"title": ".search-title",
"image": ".search-poster img",
},
"details": {
"config_el": "#serie-config",
"poster": ".fposter img",
"synopsis": ".flist .fdesc p",
"release": ".facts .release",
"genres": ".facts .genres a",
"episode_badge": ".short-meta.short-label",
"episodes": {
"api": "/engine/ajax/manga_episodes_api.php",
"versions": ["vostfr", "vf"],
"fragment_prefix": "ep",
},
"latest": {
"path": "/manga-streaming-1/", # « Récemment mises à jour »
"item": ".short",
"link": "a.short-poster", # href = index.php?newsid=<id>, alt = titre
"title": ".short-title",
"image": "a.short-poster img",
},
},
}
_ID_RE = re.compile(r"newsid=(\d+)|/(\d+)-[^/]+\.html?")
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
_SEASON_RE = re.compile(r"Saison\s+(\d+)", re.IGNORECASE)
_YEAR_RE = re.compile(r"\((\d{4})\)\s*$")
def _merged_config() -> dict:
merged = deepcopy(DEFAULT_CONFIG)
for key, values in load_scraper_config("french_manga").items():
if isinstance(values, dict) and isinstance(merged.get(key), dict):
merged[key].update(values)
else:
merged[key] = values
return merged
@register_source
class FrenchMangaScraper(SourceScraper):
name = "french_manga"
label = "French-Manga"
base_url = "https://w16.french-manga.net"
media_types = ("anime",)
# ------------------------------------------------------------- helpers
@staticmethod
def _id_from_url(url: str) -> str | None:
match = _ID_RE.search(url)
if not match:
return None
return match.group(1) or match.group(2)
def _title_url(self, source_id: str) -> str:
return f"{self.base_url}/index.php?newsid={source_id}"
async def _post(self, url: str, data: dict[str, str]) -> str:
"""POST avec retries (fetch du socle est GET uniquement)."""
last_error: Exception | None = None
for attempt in range(3):
try:
response = await get_client().post(url, data=data)
response.raise_for_status()
return response.text
except (httpx.HTTPError, httpx.InvalidURL) as exc:
last_error = exc
logger.warning("french_manga POST %s — tentative %d/3 : %s", url, attempt + 1, exc)
await asyncio.sleep(1.0 * (attempt + 1))
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
async def _fetch_episodes_api(self, source_id: str) -> dict:
config = _merged_config()
api_url = urljoin(self.base_url + "/", config["episodes"]["api"])
html = await fetch(f"{api_url}?id={source_id}", referer=self._title_url(source_id))
try:
data = json.loads(html)
except json.JSONDecodeError as exc:
raise ScrapeError(
f"french_manga : réponse JSON invalide de l'API épisodes ({source_id}) : {exc}"
) from exc
if not isinstance(data, dict):
raise ScrapeError(f"french_manga : réponse inattendue de l'API épisodes ({source_id})")
return data
# ------------------------------------------------------------- search
async def search(self, query: str) -> list[SearchResult]:
config = _merged_config()
search_cfg = config["search"]
endpoint = urljoin(self.base_url + "/", search_cfg["endpoint"])
html = await self._post(endpoint, {"query": query, "page": "1"})
soup = BeautifulSoup(html, "lxml")
results: list[SearchResult] = []
for item in soup.select(search_cfg["result"]):
onclick = item.get("onclick", "")
match = re.search(r"location\.href='([^']+)'", onclick)
if not match:
logger.warning("french_manga : search-item sans lien (onclick=%r)", onclick[:80])
continue
url = urljoin(self.base_url + "/", match.group(1))
source_id = self._id_from_url(url)
if not source_id:
logger.warning("french_manga : identifiant introuvable dans %s", url)
continue
title_el = item.select_one(search_cfg["title"])
title = title_el.get_text(strip=True) if title_el else url
image = item.select_one(search_cfg["image"])
results.append(
SearchResult(
source=self.name,
source_id=source_id,
title=_YEAR_RE.sub("", title).strip() or title,
url=url,
image_url=image.get("src") if image else None,
media_type="film" if re.search(r"\bfilm\b", title, re.IGNORECASE) else "anime",
)
)
logger.info("french_manga : %d résultats pour %r", len(results), query)
return results
# ------------------------------------------------------------- latest
async def latest(self) -> list[SearchResult]:
"""Ajouts récents — section « Récemment mises à jour »."""
config = _merged_config()
latest_cfg = config["latest"]
url = urljoin(self.base_url + "/", latest_cfg["path"])
soup = await fetch_soup(url)
results: list[SearchResult] = []
for item in soup.select(latest_cfg["item"]):
link = item.select_one(latest_cfg["link"])
href = link.get("href") if link else None
if not href:
logger.warning("french_manga : bloc nouveauté sans lien, ignoré")
continue
absolute = urljoin(self.base_url + "/", href)
source_id = self._id_from_url(absolute)
if not source_id:
logger.warning("french_manga : identifiant introuvable dans %s", absolute)
continue
image = item.select_one(latest_cfg["image"])
title_el = item.select_one(latest_cfg["title"])
title = title_el.get_text(strip=True) if title_el else (link.get("alt") or absolute)
title = _YEAR_RE.sub("", title).strip() or title
results.append(
SearchResult(
source=self.name,
source_id=source_id,
title=title,
url=self._title_url(source_id),
image_url=image.get("src") if image else None,
)
)
logger.info("french_manga : %d nouveautés récupérées", len(results))
return results
# ------------------------------------------------------------- details
async def get_details(self, source_id: str) -> TitleDetails:
config = _merged_config()
url = self._title_url(source_id)
soup = await fetch_soup(url)
details_cfg = config["details"]
config_el = soup.select_one(details_cfg["config_el"])
news_id = config_el.get("data-news-id") if config_el else None
if news_id != source_id:
raise ScrapeError(f"french_manga : fiche introuvable pour {source_id} ({url})")
title = config_el.get("data-title") or (
soup.title.get_text(strip=True) if soup.title else source_id
)
release = self._text(soup.select_one(details_cfg["release"]))
release_match = re.search(r"(\d{4})", release)
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
badge_match = _NUMBER_RE.search(badge)
episodes = await self._episodes_from_api(source_id, title, url)
return TitleDetails(
source=self.name,
source_id=source_id,
title=title,
url=url,
synopsis=self._text(soup.select_one(details_cfg["synopsis"])) or None,
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
year=int(release_match.group(1)) if release_match else None,
episode_count=int(float(badge_match.group(1).replace(",", ".")))
if badge_match
else None,
episodes=episodes,
)
@staticmethod
def _text(element: Tag | None) -> str:
return element.get_text(" ", strip=True) if element else ""
# ------------------------------------------------------------ episodes
async def list_episodes(self, source_id: str) -> list[Episode]:
return await self._episodes_from_api(source_id, None, self._title_url(source_id))
async def _episodes_from_api(
self, source_id: str, title: str | None, page_url: str
) -> list[Episode]:
config = _merged_config()
data = await self._fetch_episodes_api(source_id)
versions: list[str] = config["episodes"]["versions"]
season = 1
if title:
season_match = _SEASON_RE.search(title)
if season_match:
season = int(season_match.group(1))
numbers: set[float] = set()
for version in versions:
for number_text in data.get(version) or {}:
try:
numbers.add(float(number_text))
except ValueError:
logger.warning("french_manga : numéro d'épisode invalide %r", number_text)
info = data.get("info") or {}
episodes: list[Episode] = []
for number in sorted(numbers):
number_text = str(int(number)) if number.is_integer() else str(number)
episode_info = info.get(number_text) or info.get(str(int(number)))
label = episode_info.get("title") if isinstance(episode_info, dict) else None
version = next((v for v in versions if number_text in (data.get(v) or {})), versions[0])
episodes.append(
Episode(
number=number,
title=label,
url=f"{page_url}#{config['episodes']['fragment_prefix']}={version}-{number_text}",
season=season,
)
)
if not episodes:
raise ScrapeError(f"french_manga : aucun épisode trouvé pour {source_id}")
return episodes
# -------------------------------------------------------------- embeds
async def extract_embed_links(self, episode_url: str) -> list[str]:
config = _merged_config()
page_url, _, fragment = episode_url.partition("#")
prefix = config["episodes"]["fragment_prefix"] + "="
match = re.match(rf"{re.escape(prefix)}([A-Za-z0-9]+)-([\d.]+)$", fragment)
if not match:
raise ScrapeError(f"french_manga : fragment d'épisode invalide dans {episode_url}")
version, number_text = match.group(1), match.group(2)
if number_text.isdigit():
number_text = str(int(number_text))
source_id = self._id_from_url(page_url)
if not source_id:
raise ScrapeError(f"french_manga : identifiant introuvable dans {episode_url}")
data = await self._fetch_episodes_api(source_id)
hosters = (data.get(version) or {}).get(number_text)
if not hosters:
raise ScrapeError(
f"french_manga : aucun lecteur pour {version} épisode {number_text} ({episode_url})"
)
links = list(hosters.values())
logger.info("french_manga : %d liens embed pour %s", len(links), episode_url)
return links
+327
View File
@@ -0,0 +1,327 @@
"""Source Vostfree (ipv4.vostfree.ws) — moteur DataLife Engine, animes/films VF & VOSTFR.
Faits structurels (vérifiés en live) :
- Recherche : GET /index.php?do=search&subaction=search&story=<q> → blocs `div.search-result`.
- Nouveautés : /animes-vostfr-recement-ajoutees.html → blocs `div.movie-poster`
(lien `.play a`, alt = titre, poster `span.image img`).
- Fiche : `/444-telecharger-....html` — métadonnées dans `.slide-*`, épisodes dans un
`select.new_player_selector` (une `option value="buttons_N"` par épisode).
- Chaque `div#buttons_N` contient un ou plusieurs `div.new_player_<hoster>#player_M` ;
l'URL embed est le texte de `div#content_player_M` (URL complète ou simple ID selon
l'hébergeur — les templates de reconstruction viennent du JS `anime.js` du site).
- Pas de page par épisode : l'URL d'épisode est `<fiche>#buttons_N`.
"""
import logging
import re
from copy import deepcopy
from urllib.parse import quote
from bs4 import BeautifulSoup, Tag
from app.scrapers.base import (
Episode,
ScrapeError,
SearchResult,
SourceScraper,
TitleDetails,
register_source,
)
from app.scrapers.config_loader import load_scraper_config
from app.scrapers.http import fetch_soup
logger = logging.getLogger(__name__)
DEFAULT_CONFIG: dict = {
"search": {
"result": "div.search-result",
"link": "div.title a",
"image": "span.image img",
"genres": "ul.additional li",
},
"details": {
"title": "h1",
"poster": ".slide-poster img",
"synopsis": ".slide-desc",
"genres": '.slide-top li.right a[href*="/genre/"]',
"episode_badge": ".slide-poster .year",
"season_li": "ul.slide-top li",
},
"episodes": {
"option": "select.new_player_selector option",
"button": "div.button_box",
"content_prefix": "content_",
},
"latest": {
"path": "/animes-vostfr-recement-ajoutees.html",
"item": "div.movie-poster",
"link": ".play a", # href = fiche, alt = titre
"image": "span.image img",
},
# class CSS du lecteur → template d'URL ({} = valeur de #content_player_M ;
# chaîne vide = la valeur est déjà une URL complète)
"players": {
"new_player_vip": "",
"new_player_moevideo": "",
"new_player_sibnet": "https://video.sibnet.ru/shell.php?videoid={}",
"new_player_netu": "https://video.sibnet.ru/shell.php?videoid={}",
"new_player_uqload": "https://uqload.com/embed-{}.html",
"new_player_mp4": "https://www.mp4upload.com/embed-{}.html",
"new_player_fembed": "https://www.fembed.com/v/{}",
"new_player_mytv": "https://www.myvi.top/embed/{}",
"new_player_myvi": "https://myvi.ru/player/embed/html/{}",
"new_player_moevideo_mail": "",
"new_player_rutube": "https://rutube.ru/play/embed/{}",
"new_player_ok": "https://ok.ru/video/{}",
"new_player_mail2": "https://my.mail.ru/video/embed/{}",
"new_player_rapids": "https://rapidstream.co/embed-{}.html",
"new_player_gtv": "https://iframedream.com/embed/{}.html",
"new_player_cloudvideo": "https://cloudvideo.tv/embed-{}.html",
"new_player_uptostream": "https://uptostream.com/iframe/{}",
},
}
_SLUG_RE = re.compile(r"([^/]+)\.html?$")
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
_SEASON_RE = re.compile(r"Saison\s*:?\s*(\d+)", re.IGNORECASE)
_SAFE_FRAGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+$")
def _merged_config() -> dict:
merged = deepcopy(DEFAULT_CONFIG)
for key, values in load_scraper_config("vostfree").items():
if isinstance(values, dict) and isinstance(merged.get(key), dict):
merged[key].update(values)
else:
merged[key] = values
return merged
@register_source
class VostfreeScraper(SourceScraper):
name = "vostfree"
label = "Vostfree"
base_url = "https://ipv4.vostfree.ws"
media_types = ("anime",)
# ------------------------------------------------------------- helpers
def _title_url(self, source_id: str) -> str:
return f"{self.base_url}/{source_id}.html"
@staticmethod
def _slug_from_url(url: str) -> str | None:
match = _SLUG_RE.search(url)
return match.group(1) if match else None
@staticmethod
def _text(element: Tag | None) -> str:
return element.get_text(" ", strip=True) if element else ""
# ------------------------------------------------------------- search
async def search(self, query: str) -> list[SearchResult]:
config = _merged_config()
url = f"{self.base_url}/index.php?do=search&subaction=search&story={quote(query)}"
soup = await fetch_soup(url)
results: list[SearchResult] = []
for block in soup.select(config["search"]["result"]):
link = block.select_one(config["search"]["link"])
href = link.get("href") if link else None
if not href:
logger.warning("vostfree : bloc de résultat sans lien, ignoré")
continue
source_id = self._slug_from_url(href)
if not source_id:
logger.warning("vostfree : slug introuvable dans %s", href)
continue
image = block.select_one(config["search"]["image"])
genres_text = " ".join(
li.get_text(" ", strip=True) for li in block.select(config["search"]["genres"])
)
results.append(
SearchResult(
source=self.name,
source_id=source_id,
title=self._text(link),
url=href,
image_url=image.get("src") if image else None,
media_type="film"
if re.search(r"\bfilm", genres_text, re.IGNORECASE)
else "anime",
)
)
logger.info("vostfree : %d résultats pour %r", len(results), query)
return results
# ------------------------------------------------------------- latest
async def latest(self) -> list[SearchResult]:
"""Ajouts récents — page « Animes VOSTFR récemment ajoutés »."""
config = _merged_config()
latest_cfg = config["latest"]
soup = await fetch_soup(f"{self.base_url}{latest_cfg['path']}")
results: list[SearchResult] = []
for block in soup.select(latest_cfg["item"]):
link = block.select_one(latest_cfg["link"])
href = link.get("href") if link else None
if not href:
logger.warning("vostfree : bloc nouveauté sans lien, ignoré")
continue
source_id = self._slug_from_url(href)
if not source_id:
logger.warning("vostfree : slug introuvable dans %s", href)
continue
image = block.select_one(latest_cfg["image"])
title = link.get("alt") or self._text(link)
results.append(
SearchResult(
source=self.name,
source_id=source_id,
title=title,
url=href,
image_url=image.get("src") if image else None,
)
)
logger.info("vostfree : %d nouveautés récupérées", len(results))
return results
# ------------------------------------------------------------- details
async def get_details(self, source_id: str) -> TitleDetails:
config = _merged_config()
url = self._title_url(source_id)
soup = await fetch_soup(url)
details_cfg = config["details"]
title_el = soup.select_one(details_cfg["title"])
if title_el is None:
raise ScrapeError(f"vostfree : fiche introuvable pour {source_id} ({url})")
synopsis_el = soup.select_one(details_cfg["synopsis"])
if synopsis_el is not None:
for cast in synopsis_el.select(".cast"):
cast.decompose()
synopsis = self._text(synopsis_el) or None
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
badge_match = _NUMBER_RE.search(badge)
season = 1
for li in soup.select(details_cfg["season_li"]):
season_match = _SEASON_RE.search(self._text(li))
if season_match:
season = int(season_match.group(1))
break
episodes = self._parse_episodes(soup, url, season)
return TitleDetails(
source=self.name,
source_id=source_id,
title=self._text(title_el),
url=url,
synopsis=synopsis,
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
episode_count=int(badge_match.group(1).replace(",", ".")) if badge_match else None,
episodes=episodes,
)
# ------------------------------------------------------------ episodes
async def list_episodes(self, source_id: str) -> list[Episode]:
url = self._title_url(source_id)
soup = await fetch_soup(url)
season = self._find_season(soup)
episodes = self._parse_episodes(soup, url, season)
if not episodes:
raise ScrapeError(f"vostfree : aucun épisode trouvé pour {source_id} ({url})")
return episodes
def _find_season(self, soup: BeautifulSoup) -> int:
config = _merged_config()
for li in soup.select(config["details"]["season_li"]):
season_match = _SEASON_RE.search(self._text(li))
if season_match:
return int(season_match.group(1))
return 1
def _parse_episodes(self, soup: BeautifulSoup, page_url: str, season: int) -> list[Episode]:
config = _merged_config()
options = soup.select(config["episodes"]["option"])
entries: list[tuple[str, str]] = []
seen_ids: set[str] = set()
for opt in options:
value = opt.get("value", "")
if value in seen_ids:
continue # le site duplique parfois une option (ex. buttons_268)
seen_ids.add(value)
entries.append((value, opt.get_text(strip=True)))
if not entries: # fiche sans sélecteur : on retombe sur les blocs de boutons
entries = [
(box.get("id", ""), f"Episode {index}")
for index, box in enumerate(soup.select(config["episodes"]["button"]), start=1)
]
episodes: list[Episode] = []
for index, (button_id, label) in enumerate(entries, start=1):
number_match = _NUMBER_RE.search(label)
number = (
float(number_match.group(1).replace(",", ".")) if number_match else float(index)
)
episodes.append(
Episode(
number=number,
title=label or f"Episode {index}",
url=f"{page_url}#{button_id}" if button_id else page_url,
season=season,
)
)
return episodes
# -------------------------------------------------------------- embeds
async def extract_embed_links(self, episode_url: str) -> list[str]:
config = _merged_config()
page_url, _, fragment = episode_url.partition("#")
soup = await fetch_soup(page_url)
button_cfg = config["episodes"]
button: Tag | None = None
if fragment and _SAFE_FRAGMENT_RE.match(fragment):
button = soup.select_one(f"#{fragment}")
if button is None:
logger.warning(
"vostfree : fragment #%s sans bloc correspondant dans %s", fragment, page_url
)
if button is None:
button = soup.select_one(button_cfg["button"])
if button is None:
raise ScrapeError(f"vostfree : aucun lecteur trouvé sur {episode_url}")
prefix = button_cfg["content_prefix"]
links: list[str] = []
for player in button.find_all("div", recursive=False):
player_id = player.get("id")
content = soup.select_one(f"#{prefix}{player_id}") if player_id else None
if content is None:
logger.warning(
"vostfree : contenu absent pour le lecteur #%s (%s)", player_id, page_url
)
continue
value = content.get_text(strip=True)
if not value:
continue
player_class = (player.get("class") or [""])[0]
template = config["players"].get(player_class)
if value.startswith("http"):
links.append(value)
elif template is not None:
links.append(template.format(value))
else:
logger.warning(
"vostfree : pas de template pour le lecteur %r (valeur %r)", player_class, value
)
if not links:
raise ScrapeError(f"vostfree : aucun lien embed extrait de {episode_url}")
logger.info("vostfree : %d liens embed pour %s", len(links), episode_url)
return links