v0.1.0 — Réécriture complète de OhmStreaming
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
This commit is contained in:
@@ -0,0 +1,192 @@
|
||||
"""Contrats et registres des scrapers.
|
||||
|
||||
Architecture à 2 registres :
|
||||
- sources (sites de catalogues animes/séries) : recherche, détails, épisodes, liens embed
|
||||
- hébergeurs vidéo : résolution d'une URL embed → URL directe du fichier
|
||||
|
||||
Ajouter une source = écrire une classe qui implémente le contrat et la décorer
|
||||
@register_source / @register_hoster.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScrapeError(Exception):
|
||||
"""Échec de scraping/extraction — jamais silencieux, toujours journalisé et remonté."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- modèles
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
source: str
|
||||
source_id: str # identifiant stable chez la source (slug, id…)
|
||||
title: str
|
||||
url: str
|
||||
image_url: str | None = None
|
||||
media_type: str = "anime" # anime | serie | film
|
||||
|
||||
|
||||
@dataclass
|
||||
class Episode:
|
||||
number: float # 1, 2, 2.5 pour les OAV intermédiaires
|
||||
title: str | None
|
||||
url: str # page de l'épisode chez la source
|
||||
season: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class TitleDetails:
|
||||
source: str
|
||||
source_id: str
|
||||
title: str
|
||||
url: str
|
||||
synopsis: str | None = None
|
||||
image_url: str | None = None
|
||||
banner_url: str | None = None
|
||||
genres: list[str] = field(default_factory=list)
|
||||
rating: float | None = None
|
||||
year: int | None = None
|
||||
episode_count: int | None = None
|
||||
episodes: list[Episode] = field(default_factory=list)
|
||||
media_type: str = "anime"
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoLink:
|
||||
"""Lien vidéo résolu (URL directe du fichier, lisible par un lecteur)."""
|
||||
|
||||
url: str
|
||||
hoster: str
|
||||
quality: str | None = None
|
||||
headers: dict[str, str] = field(default_factory=dict) # ex. Referer obligatoire
|
||||
is_hls: bool = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- contrats
|
||||
|
||||
|
||||
class SourceScraper(ABC):
|
||||
"""Contrat d'un site catalogue (animes, séries…)."""
|
||||
|
||||
name: str # identifiant unique, ex. "vostfree"
|
||||
label: str # nom affiché
|
||||
base_url: str
|
||||
media_types: tuple[str, ...] = ("anime",)
|
||||
|
||||
@abstractmethod
|
||||
async def search(self, query: str) -> list[SearchResult]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_details(self, source_id: str) -> TitleDetails: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_episodes(self, source_id: str) -> list[Episode]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
||||
"""URLs des lecteurs embarqués trouvés sur la page d'un épisode."""
|
||||
...
|
||||
|
||||
async def latest(self) -> list[SearchResult]:
|
||||
"""Ajouts récents du catalogue (découverte). Vide si la source ne le supporte pas."""
|
||||
return []
|
||||
|
||||
|
||||
class HosterExtractor(ABC):
|
||||
"""Contrat d'un hébergeur vidéo : embed URL → URL directe."""
|
||||
|
||||
name: str
|
||||
domains: tuple[str, ...] = ()
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
return any(d in url for d in self.domains)
|
||||
|
||||
@abstractmethod
|
||||
async def extract(self, embed_url: str) -> VideoLink: ...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- registres
|
||||
|
||||
_source_registry: dict[str, type[SourceScraper]] = {}
|
||||
_hoster_registry: list[type[HosterExtractor]] = []
|
||||
|
||||
|
||||
def register_source(cls: type[SourceScraper]) -> type[SourceScraper]:
|
||||
if cls.name in _source_registry:
|
||||
raise ValueError(f"Source déjà enregistrée : {cls.name}")
|
||||
_source_registry[cls.name] = cls
|
||||
logger.debug("Source enregistrée : %s", cls.name)
|
||||
return cls
|
||||
|
||||
|
||||
def register_hoster(cls: type[HosterExtractor]) -> type[HosterExtractor]:
|
||||
_hoster_registry.append(cls)
|
||||
logger.debug("Hébergeur enregistré : %s", cls.name)
|
||||
return cls
|
||||
|
||||
|
||||
_source_instances: dict[str, SourceScraper] = {}
|
||||
_hoster_instances: list[HosterExtractor] | None = None
|
||||
|
||||
|
||||
def get_source(name: str) -> SourceScraper:
|
||||
if name not in _source_instances:
|
||||
cls = _source_registry.get(name)
|
||||
if cls is None:
|
||||
raise ScrapeError(f"Source inconnue : {name}")
|
||||
_source_instances[name] = cls()
|
||||
return _source_instances[name]
|
||||
|
||||
|
||||
def all_sources() -> list[SourceScraper]:
|
||||
return [get_source(name) for name in _source_registry]
|
||||
|
||||
|
||||
def resolve_hoster(url: str) -> HosterExtractor | None:
|
||||
"""Trouve l'extracteur capable de traiter une URL embed (None → générique)."""
|
||||
global _hoster_instances
|
||||
if _hoster_instances is None:
|
||||
_hoster_instances = [cls() for cls in _hoster_registry]
|
||||
for extractor in _hoster_instances:
|
||||
if extractor.can_handle(url):
|
||||
return extractor
|
||||
return None
|
||||
|
||||
|
||||
def import_all_scrapers() -> None:
|
||||
"""Importe tous les modules pour déclencher les décorateurs d'enregistrement."""
|
||||
import importlib
|
||||
import pkgutil
|
||||
|
||||
import app.scrapers.hosters as hosters_pkg
|
||||
import app.scrapers.sources as sources_pkg
|
||||
|
||||
for pkg in (sources_pkg, hosters_pkg):
|
||||
for mod in pkgutil.iter_modules(pkg.__path__):
|
||||
importlib.import_module(f"{pkg.__name__}.{mod.name}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- format interne
|
||||
|
||||
INTERNAL_SEP = "|"
|
||||
|
||||
|
||||
def encode_internal_url(video_url: str, page_url: str, title: str) -> str:
|
||||
"""Format interne `video_url|page_url|titre` pour transporter le contexte."""
|
||||
for part in (video_url, page_url, title):
|
||||
if INTERNAL_SEP in part:
|
||||
raise ValueError(f"Caractère interdit '{INTERNAL_SEP}' dans : {part!r}")
|
||||
return INTERNAL_SEP.join([video_url, page_url, title])
|
||||
|
||||
|
||||
def decode_internal_url(value: str) -> tuple[str, str, str]:
|
||||
parts = value.split(INTERNAL_SEP)
|
||||
if len(parts) != 3 or not parts[0]:
|
||||
raise ValueError(f"URL interne invalide : {value!r}")
|
||||
return parts[0], parts[1], parts[2]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Chargement des configurations de scraping externalisées (sélecteurs YAML).
|
||||
|
||||
Permet de réparer un site cassé en modifiant un fichier de config sans toucher
|
||||
au code. Chaque source peut avoir un fichier `configs/<nom_source>.yaml`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def load_scraper_config(source_name: str) -> dict[str, Any]:
|
||||
path = get_settings().scrapers_config_dir / f"{source_name}.yaml"
|
||||
if not path.exists():
|
||||
logger.debug("Pas de config externe pour %s (%s)", source_name, path)
|
||||
return {}
|
||||
try:
|
||||
with path.open(encoding="utf-8") as fh:
|
||||
config = yaml.safe_load(fh) or {}
|
||||
logger.info("Config de scraping chargée pour %s", source_name)
|
||||
return config
|
||||
except yaml.YAMLError as exc:
|
||||
logger.error("Config %s invalide : %s", path, exc)
|
||||
return {}
|
||||
@@ -0,0 +1,29 @@
|
||||
# Sélecteurs de scraping French-Manga (DataLife Engine + API AJAX maison)
|
||||
# Structure vérifiée en live sur https://w16.french-manga.net
|
||||
|
||||
search:
|
||||
endpoint: "/engine/ajax/search.php" # POST query=<q>&page=1 (la recherche DLE native est cassée)
|
||||
result: ".search-item" # lien dans l'attribut onclick
|
||||
title: ".search-title"
|
||||
image: ".search-poster img"
|
||||
|
||||
details:
|
||||
config_el: "#serie-config" # data-title / data-news-id
|
||||
poster: ".fposter img"
|
||||
synopsis: ".flist .fdesc p"
|
||||
release: ".facts .release" # année
|
||||
genres: ".facts .genres a"
|
||||
episode_badge: ".short-meta.short-label" # "Ep 32 sur 32"
|
||||
|
||||
episodes:
|
||||
api: "/engine/ajax/manga_episodes_api.php" # ?id=<newsid> → JSON {vf, vostfr, info}
|
||||
versions: ["vostfr", "vf"] # priorité des versions
|
||||
fragment_prefix: "ep" # URL épisode : <fiche>#ep=<version>-<numéro>
|
||||
|
||||
# Nouveautés — section « Récemment mises à jour » de la page d'accueil épinglée
|
||||
latest:
|
||||
path: "/manga-streaming-1/"
|
||||
item: ".short"
|
||||
link: "a.short-poster" # href = index.php?newsid=<id>, alt = titre
|
||||
title: ".short-title"
|
||||
image: "a.short-poster img"
|
||||
@@ -0,0 +1,47 @@
|
||||
# Sélecteurs de scraping Vostfree (DataLife Engine) — surchargeables sans toucher au code.
|
||||
# Structure vérifiée en live sur https://ipv4.vostfree.ws
|
||||
|
||||
search:
|
||||
result: "div.search-result" # bloc d'un résultat de recherche
|
||||
link: "div.title a" # lien + titre
|
||||
image: "span.image img" # poster
|
||||
genres: "ul.additional li" # "Genre:...", "Anneé:..." (détection film)
|
||||
|
||||
details:
|
||||
title: "h1"
|
||||
poster: ".slide-poster img"
|
||||
synopsis: ".slide-desc" # les .cast internes sont retirés
|
||||
genres: '.slide-top li.right a[href*="/genre/"]'
|
||||
episode_badge: ".slide-poster .year" # ex. "Episode 293"
|
||||
season_li: "ul.slide-top li" # ex. "Saison: 01"
|
||||
|
||||
episodes:
|
||||
option: "select.new_player_selector option" # value="buttons_N" → Episode NN
|
||||
button: "div.button_box" # repli si pas de sélecteur
|
||||
content_prefix: "content_" # #player_M → #content_player_M
|
||||
|
||||
# Nouveautés — page « Animes VOSTFR récemment ajoutés »
|
||||
latest:
|
||||
path: "/animes-vostfr-recement-ajoutees.html"
|
||||
item: "div.movie-poster"
|
||||
link: ".play a" # href = fiche, alt = titre
|
||||
image: "span.image img"
|
||||
# class CSS du lecteur → template d'URL ({} = valeur de #content_player_M)
|
||||
# chaîne vide = la valeur est déjà une URL complète
|
||||
players:
|
||||
new_player_vip: ""
|
||||
new_player_moevideo: ""
|
||||
new_player_sibnet: "https://video.sibnet.ru/shell.php?videoid={}"
|
||||
new_player_netu: "https://video.sibnet.ru/shell.php?videoid={}"
|
||||
new_player_uqload: "https://uqload.com/embed-{}.html"
|
||||
new_player_mp4: "https://www.mp4upload.com/embed-{}.html"
|
||||
new_player_fembed: "https://www.fembed.com/v/{}"
|
||||
new_player_mytv: "https://www.myvi.top/embed/{}"
|
||||
new_player_myvi: "https://myvi.ru/player/embed/html/{}"
|
||||
new_player_rutube: "https://rutube.ru/play/embed/{}"
|
||||
new_player_ok: "https://ok.ru/video/{}"
|
||||
new_player_mail2: "https://my.mail.ru/video/embed/{}"
|
||||
new_player_rapids: "https://rapidstream.co/embed-{}.html"
|
||||
new_player_gtv: "https://iframedream.com/embed/{}.html"
|
||||
new_player_cloudvideo: "https://cloudvideo.tv/embed-{}.html"
|
||||
new_player_uptostream: "https://uptostream.com/iframe/{}"
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Décompresseur pour le JavaScript packé « p,a,c,k,e,d » (Dean Edwards).
|
||||
|
||||
Utilisé par plusieurs hébergeurs vidéo (Uqload, VidMoly…) pour masquer
|
||||
l'URL du lecteur : le HTML contient `eval(function(p,a,c,k,e,d){...}(...))`.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
_DIGITS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
|
||||
_PACKED_RE = re.compile(
|
||||
r"\}\('(?P<payload>.*?)',(?P<a>\d+),(?P<c>\d+),'(?P<keys>.*?)'\.split\('\|'\)",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def _base_n(number: int, base: int) -> str:
|
||||
if number == 0:
|
||||
return "0"
|
||||
chars = []
|
||||
while number:
|
||||
number, rest = divmod(number, base)
|
||||
chars.append(_DIGITS[rest])
|
||||
return "".join(reversed(chars))
|
||||
|
||||
|
||||
def unpack_packed_js(text: str) -> str | None:
|
||||
"""Retourne le JS dépacké si `text` contient un bloc packé, sinon None."""
|
||||
match = _PACKED_RE.search(text)
|
||||
if not match:
|
||||
return None
|
||||
payload = match.group("payload").replace("\\\\", "\\").replace("\\'", "'")
|
||||
base, count = int(match.group("a")), int(match.group("c"))
|
||||
keys = match.group("keys").split("|")
|
||||
for index in range(count - 1, -1, -1):
|
||||
token = _base_n(index, base)
|
||||
if keys[index]:
|
||||
replacement = keys[index].replace("\\", "\\\\")
|
||||
payload = re.sub(rf"\b{re.escape(token)}\b", replacement, payload)
|
||||
return payload
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Extracteur Luluvid/Luluvdo (luluvdo.com, luluvid.com, lulustream.com) — embed → m3u8.
|
||||
|
||||
Famille StreamSB : la page embed configure jwplayer dans du JS packé
|
||||
(p,a,c,k,e,d) avec `sources:[{file:"https://.../master.m3u8?t=<token>"}]`.
|
||||
Le token CDN (tnmr.org…) est lié à l'IP **et à l'User-Agent** de la requête
|
||||
d'embed : la VideoLink doit donc renvoyer le User-Agent du socle pour que le
|
||||
téléchargement passe — sans lui, le CDN répond 403.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.config import get_settings
|
||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
||||
from app.scrapers.hosters._packer import unpack_packed_js
|
||||
from app.scrapers.http import fetch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PATTERNS = (
|
||||
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?file\s*:\s*["\'](?P<url>[^"\']+\.m3u8[^"\']*)["\']'),
|
||||
re.compile(r'file\s*:\s*["\'](?P<url>https?://[^"\']+?\.m3u8[^"\']*)["\']'),
|
||||
re.compile(r'["\'](?P<url>https?://[^"\']*?\.m3u8(?:\?[^"\']*)?)["\']'),
|
||||
)
|
||||
|
||||
|
||||
@register_hoster
|
||||
class LuluvdoExtractor(HosterExtractor):
|
||||
name = "luluvdo"
|
||||
domains = ("luluvdo.com", "luluvid.com", "lulustream.com")
|
||||
|
||||
async def extract(self, embed_url: str) -> VideoLink:
|
||||
html = await fetch(embed_url)
|
||||
for content in (html, unpack_packed_js(html) or ""):
|
||||
for pattern in _PATTERNS:
|
||||
for match in pattern.finditer(content):
|
||||
url = match.group("url")
|
||||
if url.startswith("http"):
|
||||
return VideoLink(
|
||||
url=url,
|
||||
hoster=self.name,
|
||||
headers={
|
||||
"Referer": f"https://{urlparse(embed_url).hostname}/",
|
||||
"User-Agent": get_settings().user_agent,
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
||||
},
|
||||
is_hls=".m3u8" in url,
|
||||
)
|
||||
raise ScrapeError(f"luluvdo : URL vidéo introuvable dans {embed_url}")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Extracteur SendVid (sendvid.com) — page embed → mp4 direct.
|
||||
|
||||
La page embed expose soit une balise `<source src="...">` ( lecteur HTML5),
|
||||
soit une variable JS `video_source`. Extraction par regex, sans dépendance JS.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
||||
from app.scrapers.hosters._packer import unpack_packed_js
|
||||
from app.scrapers.http import fetch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PATTERNS = (
|
||||
re.compile(r'<source[^>]+src=["\'](?P<url>[^"\']+)["\']'),
|
||||
re.compile(r'video_source\s*[:=]\s*["\'](?P<url>[^"\']+)["\']'),
|
||||
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:mp4|m3u8)(?:\?[^"\']*)?)["\']'),
|
||||
)
|
||||
|
||||
|
||||
@register_hoster
|
||||
class SendvidExtractor(HosterExtractor):
|
||||
name = "sendvid"
|
||||
domains = ("sendvid.com",)
|
||||
|
||||
async def extract(self, embed_url: str) -> VideoLink:
|
||||
html = await fetch(embed_url)
|
||||
candidates: list[str] = []
|
||||
for content in (html, unpack_packed_js(html) or ""):
|
||||
for pattern in _PATTERNS:
|
||||
for match in pattern.finditer(content):
|
||||
candidates.append(match.group("url"))
|
||||
url = self._pick(candidates)
|
||||
if not url:
|
||||
raise ScrapeError(f"sendvid : URL vidéo introuvable dans {embed_url}")
|
||||
return VideoLink(
|
||||
url=url,
|
||||
hoster=self.name,
|
||||
headers={"Referer": embed_url},
|
||||
is_hls=".m3u8" in url,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _pick(candidates: list[str]) -> str | None:
|
||||
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
|
||||
for url in video:
|
||||
if ".mp4" in url:
|
||||
return url
|
||||
return video[0] if video else None
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Extracteur Sibnet (video.sibnet.ru) — page shell/watch → mp4 direct.
|
||||
|
||||
La page embed `shell.php?videoid=N` (ou la page publique `/videoN`) contient
|
||||
une config jwplayer du type `player.src([{src: "/v/<hash>/<id>.mp4"}])`.
|
||||
`shell.php` répond 403 depuis certains réseaux : on retombe alors sur la page
|
||||
publique de la vidéo, qui expose la même config.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
||||
from app.scrapers.http import fetch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://video.sibnet.ru"
|
||||
_VIDEO_ID_RE = re.compile(r"(?:videoid=|/video)(\d+)")
|
||||
_MP4_PATTERNS = (
|
||||
re.compile(r'player\.src\(\[\{src:\s*"(?P<url>/v/[^"]+?\.mp4)"'),
|
||||
re.compile(r'["\'](?P<url>/v/[0-9a-f]{32}/\d+\.mp4)["\']'),
|
||||
re.compile(r'["\'](?P<url>https?://[^"\']*?/v/[^"\']+?\.mp4)["\']'),
|
||||
)
|
||||
|
||||
|
||||
@register_hoster
|
||||
class SibnetExtractor(HosterExtractor):
|
||||
name = "sibnet"
|
||||
domains = ("video.sibnet.ru",)
|
||||
|
||||
async def extract(self, embed_url: str) -> VideoLink:
|
||||
match = _VIDEO_ID_RE.search(embed_url)
|
||||
if not match:
|
||||
raise ScrapeError(f"sibnet : identifiant vidéo introuvable dans {embed_url}")
|
||||
video_id = match.group(1)
|
||||
watch_url = f"{_BASE}/video{video_id}"
|
||||
|
||||
pages: list[tuple[str, str]] = []
|
||||
try:
|
||||
pages.append(("embed", await fetch(embed_url, retries=0)))
|
||||
except ScrapeError as exc:
|
||||
logger.info(
|
||||
"sibnet : page embed %s inaccessible (%s), essai page publique", embed_url, exc
|
||||
)
|
||||
try:
|
||||
pages.append(("watch", await fetch(watch_url, retries=1)))
|
||||
except ScrapeError as exc:
|
||||
logger.error("sibnet : page publique %s inaccessible : %s", watch_url, exc)
|
||||
|
||||
for source, html in pages:
|
||||
url = self._find_mp4(html, source)
|
||||
if url:
|
||||
return VideoLink(
|
||||
url=url,
|
||||
hoster=self.name,
|
||||
headers={"Referer": watch_url},
|
||||
is_hls=False,
|
||||
)
|
||||
raise ScrapeError(f"sibnet : URL mp4 introuvable pour la vidéo {video_id}")
|
||||
|
||||
def _find_mp4(self, html: str, source: str) -> str | None:
|
||||
for pattern in _MP4_PATTERNS:
|
||||
match = pattern.search(html)
|
||||
if match:
|
||||
url = urljoin(_BASE + "/", match.group("url"))
|
||||
logger.debug("sibnet : mp4 trouvé via %s → %s", source, url)
|
||||
return url
|
||||
return None
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Extracteur Uqload (uqload.to/.co/.com/.io) — page embed → mp4/m3u8.
|
||||
|
||||
La page embed contient un jwplayer configuré dans du JS packé
|
||||
(p,a,c,k,e,d) : `sources:[{file:"https://.../master.m3u8?..."}]`.
|
||||
On dépacke puis on extrait par regex.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
||||
from app.scrapers.hosters._packer import unpack_packed_js
|
||||
from app.scrapers.http import fetch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PATTERNS = (
|
||||
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?(?:file|src)\s*:\s*["\'](?P<url>[^"\']+)["\']'),
|
||||
re.compile(r'(?:file|src)\s*:\s*["\'](?P<url>https?://[^"\']+?\.(?:m3u8|mp4)[^"\']*)["\']'),
|
||||
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:m3u8|mp4)(?:\?[^"\']*)?)["\']'),
|
||||
)
|
||||
|
||||
|
||||
@register_hoster
|
||||
class UqloadExtractor(HosterExtractor):
|
||||
name = "uqload"
|
||||
domains = ("uqload.to", "uqload.co", "uqload.com", "uqload.io")
|
||||
|
||||
async def extract(self, embed_url: str) -> VideoLink:
|
||||
html = await fetch(embed_url)
|
||||
candidates: list[str] = []
|
||||
for content in (html, unpack_packed_js(html) or ""):
|
||||
for pattern in _PATTERNS:
|
||||
for match in pattern.finditer(content):
|
||||
candidates.append(match.group("url"))
|
||||
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
|
||||
url = next((u for u in video if ".mp4" in u), None) or (video[0] if video else None)
|
||||
if not url:
|
||||
raise ScrapeError(f"uqload : URL vidéo introuvable dans {embed_url}")
|
||||
return VideoLink(
|
||||
url=url,
|
||||
hoster=self.name,
|
||||
headers={"Referer": embed_url},
|
||||
is_hls=".m3u8" in url,
|
||||
)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Extracteur VidMoly (vidmoly.to / vidmoly.me) — page embed → mp4/m3u8.
|
||||
|
||||
Le lecteur jwplayer est configuré dans du JS parfois packé (p,a,c,k,e,d) :
|
||||
`sources: [{file: "...m3u8"}]`. On dépacke si besoin puis on extrait par regex.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
||||
from app.scrapers.hosters._packer import unpack_packed_js
|
||||
from app.scrapers.http import fetch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PATTERNS = (
|
||||
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?file\s*:\s*["\'](?P<url>[^"\']+)["\']'),
|
||||
re.compile(r'file\s*:\s*["\'](?P<url>https?://[^"\']+?\.(?:m3u8|mp4)[^"\']*)["\']'),
|
||||
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:m3u8|mp4)(?:\?[^"\']*)?)["\']'),
|
||||
)
|
||||
|
||||
|
||||
@register_hoster
|
||||
class VidmolyExtractor(HosterExtractor):
|
||||
name = "vidmoly"
|
||||
domains = ("vidmoly.to", "vidmoly.me", "vidmoly.biz", "vidmoly.com")
|
||||
|
||||
async def extract(self, embed_url: str) -> VideoLink:
|
||||
html = await fetch(embed_url)
|
||||
candidates: list[str] = []
|
||||
for content in (html, unpack_packed_js(html) or ""):
|
||||
for pattern in _PATTERNS:
|
||||
for match in pattern.finditer(content):
|
||||
candidates.append(match.group("url"))
|
||||
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
|
||||
url = next((u for u in video if ".mp4" in u), None) or (video[0] if video else None)
|
||||
if not url:
|
||||
raise ScrapeError(f"vidmoly : URL vidéo introuvable dans {embed_url}")
|
||||
return VideoLink(
|
||||
url=url,
|
||||
hoster=self.name,
|
||||
headers={"Referer": embed_url},
|
||||
is_hls=".m3u8" in url,
|
||||
)
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Extracteur Vidzy (vidzy.org) — page embed videojs → m3u8.
|
||||
|
||||
La page embed ne contient pas l'URL vidéo en clair : le script videojs appelle
|
||||
une fonction de décodage inline `atob(s)` + reverse + XOR, avec une graine
|
||||
dérivée du hostname (`somme des codes des caractères & 0xFF`). Si le décodage
|
||||
échoue, la page sert un leurre `https://s1.fsvid.lol/troll/master.m3u8`
|
||||
(même valeur pour tous les épisodes) — on le rejette explicitement.
|
||||
|
||||
Algorithme (reproduit fidèlement depuis le JS de la page) :
|
||||
b = base64decode(s) ; a = b[::-1]
|
||||
r[i] = chr(a[i] ^ ((0x3D + i*89 + H) & 0xFF)) avec H = sum(ord(hostname)) & 0xFF
|
||||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.config import get_settings
|
||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
||||
from app.scrapers.http import fetch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_B64_RE = re.compile(
|
||||
r"atob\(s\)(?s:.){0,800}?\}\)\(\"(?P<b64>[A-Za-z0-9+/=]{50,})\"",
|
||||
)
|
||||
_CLEAR_PATTERNS = (
|
||||
re.compile(r'sources\s*:\s*\[\s*\{\s*src\s*:\s*["\'](?P<url>https?://[^"\']+?\.m3u8[^"\']*)["\']'),
|
||||
re.compile(r'_fsvHls\s*=\s*"(?P<url>https?://[^"]+?\.m3u8[^"]*)"'),
|
||||
)
|
||||
_TROLL = "/troll/"
|
||||
|
||||
|
||||
def _decode(b64: str, hostname: str) -> str | None:
|
||||
try:
|
||||
data = base64.b64decode(b64)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
seed = sum(ord(c) for c in hostname) & 0xFF
|
||||
decoded = "".join(chr(b ^ ((0x3D + i * 89 + seed) & 0xFF)) for i, b in enumerate(data[::-1]))
|
||||
return decoded if decoded.startswith(("http://", "https://")) else None
|
||||
|
||||
|
||||
@register_hoster
|
||||
class VidzyExtractor(HosterExtractor):
|
||||
name = "vidzy"
|
||||
domains = ("vidzy.org",)
|
||||
|
||||
async def extract(self, embed_url: str) -> VideoLink:
|
||||
html = await fetch(embed_url)
|
||||
hostname = urlparse(embed_url).hostname or ""
|
||||
|
||||
for match in _B64_RE.finditer(html):
|
||||
url = _decode(match.group("b64"), hostname)
|
||||
if url and _TROLL not in url:
|
||||
return self._video_link(url, embed_url)
|
||||
logger.warning("vidzy : décodage atob+XOR sans résultat pour %s", embed_url)
|
||||
|
||||
for pattern in _CLEAR_PATTERNS:
|
||||
for match in pattern.finditer(html):
|
||||
url = match.group("url")
|
||||
if _TROLL not in url:
|
||||
return self._video_link(url, embed_url)
|
||||
raise ScrapeError(
|
||||
f"vidzy : URL vidéo introuvable dans {embed_url} (leurre anti-bot ou page modifiée)"
|
||||
)
|
||||
|
||||
def _video_link(self, url: str, embed_url: str) -> VideoLink:
|
||||
return VideoLink(
|
||||
url=url,
|
||||
hoster=self.name,
|
||||
headers={
|
||||
"Referer": f"https://{urlparse(embed_url).hostname}/",
|
||||
"User-Agent": get_settings().user_agent,
|
||||
},
|
||||
is_hls=".m3u8" in url,
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Client HTTP partagé pour le scraping (httpx async, headers navigateur, retries)."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.config import get_settings
|
||||
from app.scrapers.base import ScrapeError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_client: httpx.AsyncClient | None = None
|
||||
|
||||
|
||||
def get_client() -> httpx.AsyncClient:
|
||||
global _client
|
||||
if _client is None:
|
||||
settings = get_settings()
|
||||
_client = httpx.AsyncClient(
|
||||
timeout=settings.http_timeout,
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"User-Agent": settings.user_agent,
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
||||
},
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
async def close_client() -> None:
|
||||
global _client
|
||||
if _client is not None:
|
||||
await _client.aclose()
|
||||
_client = None
|
||||
|
||||
|
||||
async def fetch(
|
||||
url: str,
|
||||
*,
|
||||
referer: str | None = None,
|
||||
retries: int = 2,
|
||||
) -> str:
|
||||
"""GET d'une page avec retries ; lève ScrapeError en cas d'échec définitif."""
|
||||
headers = {"Referer": referer} if referer else {}
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(retries + 1):
|
||||
try:
|
||||
response = await get_client().get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
||||
last_error = exc
|
||||
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
|
||||
if attempt < retries:
|
||||
await asyncio.sleep(1.0 * (attempt + 1))
|
||||
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
||||
|
||||
|
||||
async def fetch_soup(url: str, *, referer: str | None = None) -> BeautifulSoup:
|
||||
html = await fetch(url, referer=referer)
|
||||
return BeautifulSoup(html, "lxml")
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Source French-Manga (w16.french-manga.net) — animes VF/VOSTFR, moteur DataLife Engine.
|
||||
|
||||
Faits structurels (vérifiés en live) :
|
||||
- La recherche native DLE est cassée (erreur MySQL en GET, « moins de 4 caractères » en POST) ;
|
||||
le site utilise un endpoint AJAX : POST /engine/ajax/search.php avec `query=<q>&page=1`
|
||||
→ blocs `.search-item` (lien dans l'attribut onclick).
|
||||
- Nouveautés : /manga-streaming-1/ (« Récemment mises à jour ») → blocs `.short`
|
||||
(lien `a.short-poster` href = `index.php?newsid=<id>`, titre `.short-title`, poster img).
|
||||
- Fiche : `/<id>-<slug>.html` (alias canonique `index.php?newsid=<id>`), métadonnées dans
|
||||
`.facts`, `.fdesc`, `.fposter` et `#serie-config` (data-title, data-news-id).
|
||||
- Épisodes et lecteurs : GET /engine/ajax/manga_episodes_api.php?id=<newsid> → JSON
|
||||
{"vf": {ep: {hoster: url}}, "vostfr": {...}, "info": {ep: {title, poster}}}.
|
||||
- Pas de page par épisode : l'URL d'épisode est `<fiche>#ep=<version>-<numéro>`.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from app.scrapers.base import (
|
||||
Episode,
|
||||
ScrapeError,
|
||||
SearchResult,
|
||||
SourceScraper,
|
||||
TitleDetails,
|
||||
register_source,
|
||||
)
|
||||
from app.scrapers.config_loader import load_scraper_config
|
||||
from app.scrapers.http import fetch, fetch_soup, get_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_CONFIG: dict = {
|
||||
"search": {
|
||||
"endpoint": "/engine/ajax/search.php",
|
||||
"result": ".search-item",
|
||||
"title": ".search-title",
|
||||
"image": ".search-poster img",
|
||||
},
|
||||
"details": {
|
||||
"config_el": "#serie-config",
|
||||
"poster": ".fposter img",
|
||||
"synopsis": ".flist .fdesc p",
|
||||
"release": ".facts .release",
|
||||
"genres": ".facts .genres a",
|
||||
"episode_badge": ".short-meta.short-label",
|
||||
"episodes": {
|
||||
"api": "/engine/ajax/manga_episodes_api.php",
|
||||
"versions": ["vostfr", "vf"],
|
||||
"fragment_prefix": "ep",
|
||||
},
|
||||
"latest": {
|
||||
"path": "/manga-streaming-1/", # « Récemment mises à jour »
|
||||
"item": ".short",
|
||||
"link": "a.short-poster", # href = index.php?newsid=<id>, alt = titre
|
||||
"title": ".short-title",
|
||||
"image": "a.short-poster img",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_ID_RE = re.compile(r"newsid=(\d+)|/(\d+)-[^/]+\.html?")
|
||||
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
|
||||
_SEASON_RE = re.compile(r"Saison\s+(\d+)", re.IGNORECASE)
|
||||
_YEAR_RE = re.compile(r"\((\d{4})\)\s*$")
|
||||
|
||||
|
||||
def _merged_config() -> dict:
|
||||
merged = deepcopy(DEFAULT_CONFIG)
|
||||
for key, values in load_scraper_config("french_manga").items():
|
||||
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
||||
merged[key].update(values)
|
||||
else:
|
||||
merged[key] = values
|
||||
return merged
|
||||
|
||||
|
||||
@register_source
|
||||
class FrenchMangaScraper(SourceScraper):
|
||||
name = "french_manga"
|
||||
label = "French-Manga"
|
||||
base_url = "https://w16.french-manga.net"
|
||||
media_types = ("anime",)
|
||||
|
||||
# ------------------------------------------------------------- helpers
|
||||
|
||||
@staticmethod
|
||||
def _id_from_url(url: str) -> str | None:
|
||||
match = _ID_RE.search(url)
|
||||
if not match:
|
||||
return None
|
||||
return match.group(1) or match.group(2)
|
||||
|
||||
def _title_url(self, source_id: str) -> str:
|
||||
return f"{self.base_url}/index.php?newsid={source_id}"
|
||||
|
||||
async def _post(self, url: str, data: dict[str, str]) -> str:
|
||||
"""POST avec retries (fetch du socle est GET uniquement)."""
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
response = await get_client().post(url, data=data)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
||||
last_error = exc
|
||||
logger.warning("french_manga POST %s — tentative %d/3 : %s", url, attempt + 1, exc)
|
||||
await asyncio.sleep(1.0 * (attempt + 1))
|
||||
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
||||
|
||||
async def _fetch_episodes_api(self, source_id: str) -> dict:
|
||||
config = _merged_config()
|
||||
api_url = urljoin(self.base_url + "/", config["episodes"]["api"])
|
||||
html = await fetch(f"{api_url}?id={source_id}", referer=self._title_url(source_id))
|
||||
try:
|
||||
data = json.loads(html)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ScrapeError(
|
||||
f"french_manga : réponse JSON invalide de l'API épisodes ({source_id}) : {exc}"
|
||||
) from exc
|
||||
if not isinstance(data, dict):
|
||||
raise ScrapeError(f"french_manga : réponse inattendue de l'API épisodes ({source_id})")
|
||||
return data
|
||||
|
||||
# ------------------------------------------------------------- search
|
||||
|
||||
async def search(self, query: str) -> list[SearchResult]:
|
||||
config = _merged_config()
|
||||
search_cfg = config["search"]
|
||||
endpoint = urljoin(self.base_url + "/", search_cfg["endpoint"])
|
||||
html = await self._post(endpoint, {"query": query, "page": "1"})
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results: list[SearchResult] = []
|
||||
for item in soup.select(search_cfg["result"]):
|
||||
onclick = item.get("onclick", "")
|
||||
match = re.search(r"location\.href='([^']+)'", onclick)
|
||||
if not match:
|
||||
logger.warning("french_manga : search-item sans lien (onclick=%r)", onclick[:80])
|
||||
continue
|
||||
url = urljoin(self.base_url + "/", match.group(1))
|
||||
source_id = self._id_from_url(url)
|
||||
if not source_id:
|
||||
logger.warning("french_manga : identifiant introuvable dans %s", url)
|
||||
continue
|
||||
title_el = item.select_one(search_cfg["title"])
|
||||
title = title_el.get_text(strip=True) if title_el else url
|
||||
image = item.select_one(search_cfg["image"])
|
||||
results.append(
|
||||
SearchResult(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=_YEAR_RE.sub("", title).strip() or title,
|
||||
url=url,
|
||||
image_url=image.get("src") if image else None,
|
||||
media_type="film" if re.search(r"\bfilm\b", title, re.IGNORECASE) else "anime",
|
||||
)
|
||||
)
|
||||
logger.info("french_manga : %d résultats pour %r", len(results), query)
|
||||
return results
|
||||
|
||||
# ------------------------------------------------------------- latest
|
||||
|
||||
async def latest(self) -> list[SearchResult]:
|
||||
"""Ajouts récents — section « Récemment mises à jour »."""
|
||||
config = _merged_config()
|
||||
latest_cfg = config["latest"]
|
||||
url = urljoin(self.base_url + "/", latest_cfg["path"])
|
||||
soup = await fetch_soup(url)
|
||||
results: list[SearchResult] = []
|
||||
for item in soup.select(latest_cfg["item"]):
|
||||
link = item.select_one(latest_cfg["link"])
|
||||
href = link.get("href") if link else None
|
||||
if not href:
|
||||
logger.warning("french_manga : bloc nouveauté sans lien, ignoré")
|
||||
continue
|
||||
absolute = urljoin(self.base_url + "/", href)
|
||||
source_id = self._id_from_url(absolute)
|
||||
if not source_id:
|
||||
logger.warning("french_manga : identifiant introuvable dans %s", absolute)
|
||||
continue
|
||||
image = item.select_one(latest_cfg["image"])
|
||||
title_el = item.select_one(latest_cfg["title"])
|
||||
title = title_el.get_text(strip=True) if title_el else (link.get("alt") or absolute)
|
||||
title = _YEAR_RE.sub("", title).strip() or title
|
||||
results.append(
|
||||
SearchResult(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=title,
|
||||
url=self._title_url(source_id),
|
||||
image_url=image.get("src") if image else None,
|
||||
)
|
||||
)
|
||||
logger.info("french_manga : %d nouveautés récupérées", len(results))
|
||||
return results
|
||||
|
||||
# ------------------------------------------------------------- details
|
||||
|
||||
async def get_details(self, source_id: str) -> TitleDetails:
|
||||
config = _merged_config()
|
||||
url = self._title_url(source_id)
|
||||
soup = await fetch_soup(url)
|
||||
details_cfg = config["details"]
|
||||
|
||||
config_el = soup.select_one(details_cfg["config_el"])
|
||||
news_id = config_el.get("data-news-id") if config_el else None
|
||||
if news_id != source_id:
|
||||
raise ScrapeError(f"french_manga : fiche introuvable pour {source_id} ({url})")
|
||||
title = config_el.get("data-title") or (
|
||||
soup.title.get_text(strip=True) if soup.title else source_id
|
||||
)
|
||||
|
||||
release = self._text(soup.select_one(details_cfg["release"]))
|
||||
release_match = re.search(r"(\d{4})", release)
|
||||
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
|
||||
badge_match = _NUMBER_RE.search(badge)
|
||||
|
||||
episodes = await self._episodes_from_api(source_id, title, url)
|
||||
|
||||
return TitleDetails(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=title,
|
||||
url=url,
|
||||
synopsis=self._text(soup.select_one(details_cfg["synopsis"])) or None,
|
||||
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
|
||||
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
|
||||
year=int(release_match.group(1)) if release_match else None,
|
||||
episode_count=int(float(badge_match.group(1).replace(",", ".")))
|
||||
if badge_match
|
||||
else None,
|
||||
episodes=episodes,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _text(element: Tag | None) -> str:
|
||||
return element.get_text(" ", strip=True) if element else ""
|
||||
|
||||
# ------------------------------------------------------------ episodes
|
||||
|
||||
async def list_episodes(self, source_id: str) -> list[Episode]:
|
||||
return await self._episodes_from_api(source_id, None, self._title_url(source_id))
|
||||
|
||||
async def _episodes_from_api(
|
||||
self, source_id: str, title: str | None, page_url: str
|
||||
) -> list[Episode]:
|
||||
config = _merged_config()
|
||||
data = await self._fetch_episodes_api(source_id)
|
||||
versions: list[str] = config["episodes"]["versions"]
|
||||
season = 1
|
||||
if title:
|
||||
season_match = _SEASON_RE.search(title)
|
||||
if season_match:
|
||||
season = int(season_match.group(1))
|
||||
|
||||
numbers: set[float] = set()
|
||||
for version in versions:
|
||||
for number_text in data.get(version) or {}:
|
||||
try:
|
||||
numbers.add(float(number_text))
|
||||
except ValueError:
|
||||
logger.warning("french_manga : numéro d'épisode invalide %r", number_text)
|
||||
|
||||
info = data.get("info") or {}
|
||||
episodes: list[Episode] = []
|
||||
for number in sorted(numbers):
|
||||
number_text = str(int(number)) if number.is_integer() else str(number)
|
||||
episode_info = info.get(number_text) or info.get(str(int(number)))
|
||||
label = episode_info.get("title") if isinstance(episode_info, dict) else None
|
||||
version = next((v for v in versions if number_text in (data.get(v) or {})), versions[0])
|
||||
episodes.append(
|
||||
Episode(
|
||||
number=number,
|
||||
title=label,
|
||||
url=f"{page_url}#{config['episodes']['fragment_prefix']}={version}-{number_text}",
|
||||
season=season,
|
||||
)
|
||||
)
|
||||
if not episodes:
|
||||
raise ScrapeError(f"french_manga : aucun épisode trouvé pour {source_id}")
|
||||
return episodes
|
||||
|
||||
# -------------------------------------------------------------- embeds
|
||||
|
||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
||||
config = _merged_config()
|
||||
page_url, _, fragment = episode_url.partition("#")
|
||||
prefix = config["episodes"]["fragment_prefix"] + "="
|
||||
match = re.match(rf"{re.escape(prefix)}([A-Za-z0-9]+)-([\d.]+)$", fragment)
|
||||
if not match:
|
||||
raise ScrapeError(f"french_manga : fragment d'épisode invalide dans {episode_url}")
|
||||
version, number_text = match.group(1), match.group(2)
|
||||
if number_text.isdigit():
|
||||
number_text = str(int(number_text))
|
||||
|
||||
source_id = self._id_from_url(page_url)
|
||||
if not source_id:
|
||||
raise ScrapeError(f"french_manga : identifiant introuvable dans {episode_url}")
|
||||
data = await self._fetch_episodes_api(source_id)
|
||||
hosters = (data.get(version) or {}).get(number_text)
|
||||
if not hosters:
|
||||
raise ScrapeError(
|
||||
f"french_manga : aucun lecteur pour {version} épisode {number_text} ({episode_url})"
|
||||
)
|
||||
links = list(hosters.values())
|
||||
logger.info("french_manga : %d liens embed pour %s", len(links), episode_url)
|
||||
return links
|
||||
@@ -0,0 +1,327 @@
|
||||
"""Source Vostfree (ipv4.vostfree.ws) — moteur DataLife Engine, animes/films VF & VOSTFR.
|
||||
|
||||
Faits structurels (vérifiés en live) :
|
||||
- Recherche : GET /index.php?do=search&subaction=search&story=<q> → blocs `div.search-result`.
|
||||
- Nouveautés : /animes-vostfr-recement-ajoutees.html → blocs `div.movie-poster`
|
||||
(lien `.play a`, alt = titre, poster `span.image img`).
|
||||
- Fiche : `/444-telecharger-....html` — métadonnées dans `.slide-*`, épisodes dans un
|
||||
`select.new_player_selector` (une `option value="buttons_N"` par épisode).
|
||||
- Chaque `div#buttons_N` contient un ou plusieurs `div.new_player_<hoster>#player_M` ;
|
||||
l'URL embed est le texte de `div#content_player_M` (URL complète ou simple ID selon
|
||||
l'hébergeur — les templates de reconstruction viennent du JS `anime.js` du site).
|
||||
- Pas de page par épisode : l'URL d'épisode est `<fiche>#buttons_N`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from urllib.parse import quote
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from app.scrapers.base import (
|
||||
Episode,
|
||||
ScrapeError,
|
||||
SearchResult,
|
||||
SourceScraper,
|
||||
TitleDetails,
|
||||
register_source,
|
||||
)
|
||||
from app.scrapers.config_loader import load_scraper_config
|
||||
from app.scrapers.http import fetch_soup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_CONFIG: dict = {
|
||||
"search": {
|
||||
"result": "div.search-result",
|
||||
"link": "div.title a",
|
||||
"image": "span.image img",
|
||||
"genres": "ul.additional li",
|
||||
},
|
||||
"details": {
|
||||
"title": "h1",
|
||||
"poster": ".slide-poster img",
|
||||
"synopsis": ".slide-desc",
|
||||
"genres": '.slide-top li.right a[href*="/genre/"]',
|
||||
"episode_badge": ".slide-poster .year",
|
||||
"season_li": "ul.slide-top li",
|
||||
},
|
||||
"episodes": {
|
||||
"option": "select.new_player_selector option",
|
||||
"button": "div.button_box",
|
||||
"content_prefix": "content_",
|
||||
},
|
||||
"latest": {
|
||||
"path": "/animes-vostfr-recement-ajoutees.html",
|
||||
"item": "div.movie-poster",
|
||||
"link": ".play a", # href = fiche, alt = titre
|
||||
"image": "span.image img",
|
||||
},
|
||||
# class CSS du lecteur → template d'URL ({} = valeur de #content_player_M ;
|
||||
# chaîne vide = la valeur est déjà une URL complète)
|
||||
"players": {
|
||||
"new_player_vip": "",
|
||||
"new_player_moevideo": "",
|
||||
"new_player_sibnet": "https://video.sibnet.ru/shell.php?videoid={}",
|
||||
"new_player_netu": "https://video.sibnet.ru/shell.php?videoid={}",
|
||||
"new_player_uqload": "https://uqload.com/embed-{}.html",
|
||||
"new_player_mp4": "https://www.mp4upload.com/embed-{}.html",
|
||||
"new_player_fembed": "https://www.fembed.com/v/{}",
|
||||
"new_player_mytv": "https://www.myvi.top/embed/{}",
|
||||
"new_player_myvi": "https://myvi.ru/player/embed/html/{}",
|
||||
"new_player_moevideo_mail": "",
|
||||
"new_player_rutube": "https://rutube.ru/play/embed/{}",
|
||||
"new_player_ok": "https://ok.ru/video/{}",
|
||||
"new_player_mail2": "https://my.mail.ru/video/embed/{}",
|
||||
"new_player_rapids": "https://rapidstream.co/embed-{}.html",
|
||||
"new_player_gtv": "https://iframedream.com/embed/{}.html",
|
||||
"new_player_cloudvideo": "https://cloudvideo.tv/embed-{}.html",
|
||||
"new_player_uptostream": "https://uptostream.com/iframe/{}",
|
||||
},
|
||||
}
|
||||
|
||||
_SLUG_RE = re.compile(r"([^/]+)\.html?$")
|
||||
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
|
||||
_SEASON_RE = re.compile(r"Saison\s*:?\s*(\d+)", re.IGNORECASE)
|
||||
_SAFE_FRAGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+$")
|
||||
|
||||
|
||||
def _merged_config() -> dict:
|
||||
merged = deepcopy(DEFAULT_CONFIG)
|
||||
for key, values in load_scraper_config("vostfree").items():
|
||||
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
||||
merged[key].update(values)
|
||||
else:
|
||||
merged[key] = values
|
||||
return merged
|
||||
|
||||
|
||||
@register_source
|
||||
class VostfreeScraper(SourceScraper):
|
||||
name = "vostfree"
|
||||
label = "Vostfree"
|
||||
base_url = "https://ipv4.vostfree.ws"
|
||||
media_types = ("anime",)
|
||||
|
||||
# ------------------------------------------------------------- helpers
|
||||
|
||||
def _title_url(self, source_id: str) -> str:
|
||||
return f"{self.base_url}/{source_id}.html"
|
||||
|
||||
@staticmethod
|
||||
def _slug_from_url(url: str) -> str | None:
|
||||
match = _SLUG_RE.search(url)
|
||||
return match.group(1) if match else None
|
||||
|
||||
@staticmethod
|
||||
def _text(element: Tag | None) -> str:
|
||||
return element.get_text(" ", strip=True) if element else ""
|
||||
|
||||
# ------------------------------------------------------------- search
|
||||
|
||||
async def search(self, query: str) -> list[SearchResult]:
|
||||
config = _merged_config()
|
||||
url = f"{self.base_url}/index.php?do=search&subaction=search&story={quote(query)}"
|
||||
soup = await fetch_soup(url)
|
||||
results: list[SearchResult] = []
|
||||
for block in soup.select(config["search"]["result"]):
|
||||
link = block.select_one(config["search"]["link"])
|
||||
href = link.get("href") if link else None
|
||||
if not href:
|
||||
logger.warning("vostfree : bloc de résultat sans lien, ignoré")
|
||||
continue
|
||||
source_id = self._slug_from_url(href)
|
||||
if not source_id:
|
||||
logger.warning("vostfree : slug introuvable dans %s", href)
|
||||
continue
|
||||
image = block.select_one(config["search"]["image"])
|
||||
genres_text = " ".join(
|
||||
li.get_text(" ", strip=True) for li in block.select(config["search"]["genres"])
|
||||
)
|
||||
results.append(
|
||||
SearchResult(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=self._text(link),
|
||||
url=href,
|
||||
image_url=image.get("src") if image else None,
|
||||
media_type="film"
|
||||
if re.search(r"\bfilm", genres_text, re.IGNORECASE)
|
||||
else "anime",
|
||||
)
|
||||
)
|
||||
logger.info("vostfree : %d résultats pour %r", len(results), query)
|
||||
return results
|
||||
|
||||
# ------------------------------------------------------------- latest
|
||||
|
||||
async def latest(self) -> list[SearchResult]:
|
||||
"""Ajouts récents — page « Animes VOSTFR récemment ajoutés »."""
|
||||
config = _merged_config()
|
||||
latest_cfg = config["latest"]
|
||||
soup = await fetch_soup(f"{self.base_url}{latest_cfg['path']}")
|
||||
results: list[SearchResult] = []
|
||||
for block in soup.select(latest_cfg["item"]):
|
||||
link = block.select_one(latest_cfg["link"])
|
||||
href = link.get("href") if link else None
|
||||
if not href:
|
||||
logger.warning("vostfree : bloc nouveauté sans lien, ignoré")
|
||||
continue
|
||||
source_id = self._slug_from_url(href)
|
||||
if not source_id:
|
||||
logger.warning("vostfree : slug introuvable dans %s", href)
|
||||
continue
|
||||
image = block.select_one(latest_cfg["image"])
|
||||
title = link.get("alt") or self._text(link)
|
||||
results.append(
|
||||
SearchResult(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=title,
|
||||
url=href,
|
||||
image_url=image.get("src") if image else None,
|
||||
)
|
||||
)
|
||||
logger.info("vostfree : %d nouveautés récupérées", len(results))
|
||||
return results
|
||||
# ------------------------------------------------------------- details
|
||||
|
||||
async def get_details(self, source_id: str) -> TitleDetails:
|
||||
config = _merged_config()
|
||||
url = self._title_url(source_id)
|
||||
soup = await fetch_soup(url)
|
||||
details_cfg = config["details"]
|
||||
|
||||
title_el = soup.select_one(details_cfg["title"])
|
||||
if title_el is None:
|
||||
raise ScrapeError(f"vostfree : fiche introuvable pour {source_id} ({url})")
|
||||
|
||||
synopsis_el = soup.select_one(details_cfg["synopsis"])
|
||||
if synopsis_el is not None:
|
||||
for cast in synopsis_el.select(".cast"):
|
||||
cast.decompose()
|
||||
synopsis = self._text(synopsis_el) or None
|
||||
|
||||
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
|
||||
badge_match = _NUMBER_RE.search(badge)
|
||||
|
||||
season = 1
|
||||
for li in soup.select(details_cfg["season_li"]):
|
||||
season_match = _SEASON_RE.search(self._text(li))
|
||||
if season_match:
|
||||
season = int(season_match.group(1))
|
||||
break
|
||||
|
||||
episodes = self._parse_episodes(soup, url, season)
|
||||
|
||||
return TitleDetails(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=self._text(title_el),
|
||||
url=url,
|
||||
synopsis=synopsis,
|
||||
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
|
||||
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
|
||||
episode_count=int(badge_match.group(1).replace(",", ".")) if badge_match else None,
|
||||
episodes=episodes,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------ episodes
|
||||
|
||||
async def list_episodes(self, source_id: str) -> list[Episode]:
|
||||
url = self._title_url(source_id)
|
||||
soup = await fetch_soup(url)
|
||||
season = self._find_season(soup)
|
||||
episodes = self._parse_episodes(soup, url, season)
|
||||
if not episodes:
|
||||
raise ScrapeError(f"vostfree : aucun épisode trouvé pour {source_id} ({url})")
|
||||
return episodes
|
||||
|
||||
def _find_season(self, soup: BeautifulSoup) -> int:
|
||||
config = _merged_config()
|
||||
for li in soup.select(config["details"]["season_li"]):
|
||||
season_match = _SEASON_RE.search(self._text(li))
|
||||
if season_match:
|
||||
return int(season_match.group(1))
|
||||
return 1
|
||||
|
||||
def _parse_episodes(self, soup: BeautifulSoup, page_url: str, season: int) -> list[Episode]:
|
||||
config = _merged_config()
|
||||
options = soup.select(config["episodes"]["option"])
|
||||
entries: list[tuple[str, str]] = []
|
||||
seen_ids: set[str] = set()
|
||||
for opt in options:
|
||||
value = opt.get("value", "")
|
||||
if value in seen_ids:
|
||||
continue # le site duplique parfois une option (ex. buttons_268)
|
||||
seen_ids.add(value)
|
||||
entries.append((value, opt.get_text(strip=True)))
|
||||
if not entries: # fiche sans sélecteur : on retombe sur les blocs de boutons
|
||||
entries = [
|
||||
(box.get("id", ""), f"Episode {index}")
|
||||
for index, box in enumerate(soup.select(config["episodes"]["button"]), start=1)
|
||||
]
|
||||
episodes: list[Episode] = []
|
||||
for index, (button_id, label) in enumerate(entries, start=1):
|
||||
number_match = _NUMBER_RE.search(label)
|
||||
number = (
|
||||
float(number_match.group(1).replace(",", ".")) if number_match else float(index)
|
||||
)
|
||||
episodes.append(
|
||||
Episode(
|
||||
number=number,
|
||||
title=label or f"Episode {index}",
|
||||
url=f"{page_url}#{button_id}" if button_id else page_url,
|
||||
season=season,
|
||||
)
|
||||
)
|
||||
return episodes
|
||||
|
||||
# -------------------------------------------------------------- embeds
|
||||
|
||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
||||
config = _merged_config()
|
||||
page_url, _, fragment = episode_url.partition("#")
|
||||
soup = await fetch_soup(page_url)
|
||||
button_cfg = config["episodes"]
|
||||
|
||||
button: Tag | None = None
|
||||
if fragment and _SAFE_FRAGMENT_RE.match(fragment):
|
||||
button = soup.select_one(f"#{fragment}")
|
||||
if button is None:
|
||||
logger.warning(
|
||||
"vostfree : fragment #%s sans bloc correspondant dans %s", fragment, page_url
|
||||
)
|
||||
if button is None:
|
||||
button = soup.select_one(button_cfg["button"])
|
||||
if button is None:
|
||||
raise ScrapeError(f"vostfree : aucun lecteur trouvé sur {episode_url}")
|
||||
|
||||
prefix = button_cfg["content_prefix"]
|
||||
links: list[str] = []
|
||||
for player in button.find_all("div", recursive=False):
|
||||
player_id = player.get("id")
|
||||
content = soup.select_one(f"#{prefix}{player_id}") if player_id else None
|
||||
if content is None:
|
||||
logger.warning(
|
||||
"vostfree : contenu absent pour le lecteur #%s (%s)", player_id, page_url
|
||||
)
|
||||
continue
|
||||
value = content.get_text(strip=True)
|
||||
if not value:
|
||||
continue
|
||||
player_class = (player.get("class") or [""])[0]
|
||||
template = config["players"].get(player_class)
|
||||
if value.startswith("http"):
|
||||
links.append(value)
|
||||
elif template is not None:
|
||||
links.append(template.format(value))
|
||||
else:
|
||||
logger.warning(
|
||||
"vostfree : pas de template pour le lecteur %r (valeur %r)", player_class, value
|
||||
)
|
||||
if not links:
|
||||
raise ScrapeError(f"vostfree : aucun lien embed extrait de {episode_url}")
|
||||
logger.info("vostfree : %d liens embed pour %s", len(links), episode_url)
|
||||
return links
|
||||
Reference in New Issue
Block a user