Roman
41566ab5fb
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
314 lines
13 KiB
Python
314 lines
13 KiB
Python
"""Source French-Manga (w16.french-manga.net) — animes VF/VOSTFR, moteur DataLife Engine.
|
|
|
|
Faits structurels (vérifiés en live) :
|
|
- La recherche native DLE est cassée (erreur MySQL en GET, « moins de 4 caractères » en POST) ;
|
|
le site utilise un endpoint AJAX : POST /engine/ajax/search.php avec `query=<q>&page=1`
|
|
→ blocs `.search-item` (lien dans l'attribut onclick).
|
|
- Nouveautés : /manga-streaming-1/ (« Récemment mises à jour ») → blocs `.short`
|
|
(lien `a.short-poster` href = `index.php?newsid=<id>`, titre `.short-title`, poster img).
|
|
- Fiche : `/<id>-<slug>.html` (alias canonique `index.php?newsid=<id>`), métadonnées dans
|
|
`.facts`, `.fdesc`, `.fposter` et `#serie-config` (data-title, data-news-id).
|
|
- Épisodes et lecteurs : GET /engine/ajax/manga_episodes_api.php?id=<newsid> → JSON
|
|
{"vf": {ep: {hoster: url}}, "vostfr": {...}, "info": {ep: {title, poster}}}.
|
|
- Pas de page par épisode : l'URL d'épisode est `<fiche>#ep=<version>-<numéro>`.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import re
|
|
from copy import deepcopy
|
|
from urllib.parse import urljoin
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup, Tag
|
|
|
|
from app.scrapers.base import (
|
|
Episode,
|
|
ScrapeError,
|
|
SearchResult,
|
|
SourceScraper,
|
|
TitleDetails,
|
|
register_source,
|
|
)
|
|
from app.scrapers.config_loader import load_scraper_config
|
|
from app.scrapers.http import fetch, fetch_soup, get_client
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_CONFIG: dict = {
|
|
"search": {
|
|
"endpoint": "/engine/ajax/search.php",
|
|
"result": ".search-item",
|
|
"title": ".search-title",
|
|
"image": ".search-poster img",
|
|
},
|
|
"details": {
|
|
"config_el": "#serie-config",
|
|
"poster": ".fposter img",
|
|
"synopsis": ".flist .fdesc p",
|
|
"release": ".facts .release",
|
|
"genres": ".facts .genres a",
|
|
"episode_badge": ".short-meta.short-label",
|
|
"episodes": {
|
|
"api": "/engine/ajax/manga_episodes_api.php",
|
|
"versions": ["vostfr", "vf"],
|
|
"fragment_prefix": "ep",
|
|
},
|
|
"latest": {
|
|
"path": "/manga-streaming-1/", # « Récemment mises à jour »
|
|
"item": ".short",
|
|
"link": "a.short-poster", # href = index.php?newsid=<id>, alt = titre
|
|
"title": ".short-title",
|
|
"image": "a.short-poster img",
|
|
},
|
|
},
|
|
}
|
|
|
|
_ID_RE = re.compile(r"newsid=(\d+)|/(\d+)-[^/]+\.html?")
|
|
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
|
|
_SEASON_RE = re.compile(r"Saison\s+(\d+)", re.IGNORECASE)
|
|
_YEAR_RE = re.compile(r"\((\d{4})\)\s*$")
|
|
|
|
|
|
def _merged_config() -> dict:
|
|
merged = deepcopy(DEFAULT_CONFIG)
|
|
for key, values in load_scraper_config("french_manga").items():
|
|
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
|
merged[key].update(values)
|
|
else:
|
|
merged[key] = values
|
|
return merged
|
|
|
|
|
|
@register_source
|
|
class FrenchMangaScraper(SourceScraper):
|
|
name = "french_manga"
|
|
label = "French-Manga"
|
|
base_url = "https://w16.french-manga.net"
|
|
media_types = ("anime",)
|
|
|
|
# ------------------------------------------------------------- helpers
|
|
|
|
@staticmethod
|
|
def _id_from_url(url: str) -> str | None:
|
|
match = _ID_RE.search(url)
|
|
if not match:
|
|
return None
|
|
return match.group(1) or match.group(2)
|
|
|
|
def _title_url(self, source_id: str) -> str:
|
|
return f"{self.base_url}/index.php?newsid={source_id}"
|
|
|
|
async def _post(self, url: str, data: dict[str, str]) -> str:
|
|
"""POST avec retries (fetch du socle est GET uniquement)."""
|
|
last_error: Exception | None = None
|
|
for attempt in range(3):
|
|
try:
|
|
response = await get_client().post(url, data=data)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
|
last_error = exc
|
|
logger.warning("french_manga POST %s — tentative %d/3 : %s", url, attempt + 1, exc)
|
|
await asyncio.sleep(1.0 * (attempt + 1))
|
|
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
|
|
|
async def _fetch_episodes_api(self, source_id: str) -> dict:
|
|
config = _merged_config()
|
|
api_url = urljoin(self.base_url + "/", config["episodes"]["api"])
|
|
html = await fetch(f"{api_url}?id={source_id}", referer=self._title_url(source_id))
|
|
try:
|
|
data = json.loads(html)
|
|
except json.JSONDecodeError as exc:
|
|
raise ScrapeError(
|
|
f"french_manga : réponse JSON invalide de l'API épisodes ({source_id}) : {exc}"
|
|
) from exc
|
|
if not isinstance(data, dict):
|
|
raise ScrapeError(f"french_manga : réponse inattendue de l'API épisodes ({source_id})")
|
|
return data
|
|
|
|
# ------------------------------------------------------------- search
|
|
|
|
async def search(self, query: str) -> list[SearchResult]:
|
|
config = _merged_config()
|
|
search_cfg = config["search"]
|
|
endpoint = urljoin(self.base_url + "/", search_cfg["endpoint"])
|
|
html = await self._post(endpoint, {"query": query, "page": "1"})
|
|
soup = BeautifulSoup(html, "lxml")
|
|
results: list[SearchResult] = []
|
|
for item in soup.select(search_cfg["result"]):
|
|
onclick = item.get("onclick", "")
|
|
match = re.search(r"location\.href='([^']+)'", onclick)
|
|
if not match:
|
|
logger.warning("french_manga : search-item sans lien (onclick=%r)", onclick[:80])
|
|
continue
|
|
url = urljoin(self.base_url + "/", match.group(1))
|
|
source_id = self._id_from_url(url)
|
|
if not source_id:
|
|
logger.warning("french_manga : identifiant introuvable dans %s", url)
|
|
continue
|
|
title_el = item.select_one(search_cfg["title"])
|
|
title = title_el.get_text(strip=True) if title_el else url
|
|
image = item.select_one(search_cfg["image"])
|
|
results.append(
|
|
SearchResult(
|
|
source=self.name,
|
|
source_id=source_id,
|
|
title=_YEAR_RE.sub("", title).strip() or title,
|
|
url=url,
|
|
image_url=image.get("src") if image else None,
|
|
media_type="film" if re.search(r"\bfilm\b", title, re.IGNORECASE) else "anime",
|
|
)
|
|
)
|
|
logger.info("french_manga : %d résultats pour %r", len(results), query)
|
|
return results
|
|
|
|
# ------------------------------------------------------------- latest
|
|
|
|
async def latest(self) -> list[SearchResult]:
|
|
"""Ajouts récents — section « Récemment mises à jour »."""
|
|
config = _merged_config()
|
|
latest_cfg = config["latest"]
|
|
url = urljoin(self.base_url + "/", latest_cfg["path"])
|
|
soup = await fetch_soup(url)
|
|
results: list[SearchResult] = []
|
|
for item in soup.select(latest_cfg["item"]):
|
|
link = item.select_one(latest_cfg["link"])
|
|
href = link.get("href") if link else None
|
|
if not href:
|
|
logger.warning("french_manga : bloc nouveauté sans lien, ignoré")
|
|
continue
|
|
absolute = urljoin(self.base_url + "/", href)
|
|
source_id = self._id_from_url(absolute)
|
|
if not source_id:
|
|
logger.warning("french_manga : identifiant introuvable dans %s", absolute)
|
|
continue
|
|
image = item.select_one(latest_cfg["image"])
|
|
title_el = item.select_one(latest_cfg["title"])
|
|
title = title_el.get_text(strip=True) if title_el else (link.get("alt") or absolute)
|
|
title = _YEAR_RE.sub("", title).strip() or title
|
|
results.append(
|
|
SearchResult(
|
|
source=self.name,
|
|
source_id=source_id,
|
|
title=title,
|
|
url=self._title_url(source_id),
|
|
image_url=image.get("src") if image else None,
|
|
)
|
|
)
|
|
logger.info("french_manga : %d nouveautés récupérées", len(results))
|
|
return results
|
|
|
|
# ------------------------------------------------------------- details
|
|
|
|
async def get_details(self, source_id: str) -> TitleDetails:
|
|
config = _merged_config()
|
|
url = self._title_url(source_id)
|
|
soup = await fetch_soup(url)
|
|
details_cfg = config["details"]
|
|
|
|
config_el = soup.select_one(details_cfg["config_el"])
|
|
news_id = config_el.get("data-news-id") if config_el else None
|
|
if news_id != source_id:
|
|
raise ScrapeError(f"french_manga : fiche introuvable pour {source_id} ({url})")
|
|
title = config_el.get("data-title") or (
|
|
soup.title.get_text(strip=True) if soup.title else source_id
|
|
)
|
|
|
|
release = self._text(soup.select_one(details_cfg["release"]))
|
|
release_match = re.search(r"(\d{4})", release)
|
|
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
|
|
badge_match = _NUMBER_RE.search(badge)
|
|
|
|
episodes = await self._episodes_from_api(source_id, title, url)
|
|
|
|
return TitleDetails(
|
|
source=self.name,
|
|
source_id=source_id,
|
|
title=title,
|
|
url=url,
|
|
synopsis=self._text(soup.select_one(details_cfg["synopsis"])) or None,
|
|
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
|
|
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
|
|
year=int(release_match.group(1)) if release_match else None,
|
|
episode_count=int(float(badge_match.group(1).replace(",", ".")))
|
|
if badge_match
|
|
else None,
|
|
episodes=episodes,
|
|
)
|
|
|
|
@staticmethod
|
|
def _text(element: Tag | None) -> str:
|
|
return element.get_text(" ", strip=True) if element else ""
|
|
|
|
# ------------------------------------------------------------ episodes
|
|
|
|
async def list_episodes(self, source_id: str) -> list[Episode]:
|
|
return await self._episodes_from_api(source_id, None, self._title_url(source_id))
|
|
|
|
async def _episodes_from_api(
|
|
self, source_id: str, title: str | None, page_url: str
|
|
) -> list[Episode]:
|
|
config = _merged_config()
|
|
data = await self._fetch_episodes_api(source_id)
|
|
versions: list[str] = config["episodes"]["versions"]
|
|
season = 1
|
|
if title:
|
|
season_match = _SEASON_RE.search(title)
|
|
if season_match:
|
|
season = int(season_match.group(1))
|
|
|
|
numbers: set[float] = set()
|
|
for version in versions:
|
|
for number_text in data.get(version) or {}:
|
|
try:
|
|
numbers.add(float(number_text))
|
|
except ValueError:
|
|
logger.warning("french_manga : numéro d'épisode invalide %r", number_text)
|
|
|
|
info = data.get("info") or {}
|
|
episodes: list[Episode] = []
|
|
for number in sorted(numbers):
|
|
number_text = str(int(number)) if number.is_integer() else str(number)
|
|
episode_info = info.get(number_text) or info.get(str(int(number)))
|
|
label = episode_info.get("title") if isinstance(episode_info, dict) else None
|
|
version = next((v for v in versions if number_text in (data.get(v) or {})), versions[0])
|
|
episodes.append(
|
|
Episode(
|
|
number=number,
|
|
title=label,
|
|
url=f"{page_url}#{config['episodes']['fragment_prefix']}={version}-{number_text}",
|
|
season=season,
|
|
)
|
|
)
|
|
if not episodes:
|
|
raise ScrapeError(f"french_manga : aucun épisode trouvé pour {source_id}")
|
|
return episodes
|
|
|
|
# -------------------------------------------------------------- embeds
|
|
|
|
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
|
config = _merged_config()
|
|
page_url, _, fragment = episode_url.partition("#")
|
|
prefix = config["episodes"]["fragment_prefix"] + "="
|
|
match = re.match(rf"{re.escape(prefix)}([A-Za-z0-9]+)-([\d.]+)$", fragment)
|
|
if not match:
|
|
raise ScrapeError(f"french_manga : fragment d'épisode invalide dans {episode_url}")
|
|
version, number_text = match.group(1), match.group(2)
|
|
if number_text.isdigit():
|
|
number_text = str(int(number_text))
|
|
|
|
source_id = self._id_from_url(page_url)
|
|
if not source_id:
|
|
raise ScrapeError(f"french_manga : identifiant introuvable dans {episode_url}")
|
|
data = await self._fetch_episodes_api(source_id)
|
|
hosters = (data.get(version) or {}).get(number_text)
|
|
if not hosters:
|
|
raise ScrapeError(
|
|
f"french_manga : aucun lecteur pour {version} épisode {number_text} ({episode_url})"
|
|
)
|
|
links = list(hosters.values())
|
|
logger.info("french_manga : %d liens embed pour %s", len(links), episode_url)
|
|
return links
|