Roman
41566ab5fb
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
328 lines
13 KiB
Python
328 lines
13 KiB
Python
"""Source Vostfree (ipv4.vostfree.ws) — moteur DataLife Engine, animes/films VF & VOSTFR.
|
|
|
|
Faits structurels (vérifiés en live) :
|
|
- Recherche : GET /index.php?do=search&subaction=search&story=<q> → blocs `div.search-result`.
|
|
- Nouveautés : /animes-vostfr-recement-ajoutees.html → blocs `div.movie-poster`
|
|
(lien `.play a`, alt = titre, poster `span.image img`).
|
|
- Fiche : `/444-telecharger-....html` — métadonnées dans `.slide-*`, épisodes dans un
|
|
`select.new_player_selector` (une `option value="buttons_N"` par épisode).
|
|
- Chaque `div#buttons_N` contient un ou plusieurs `div.new_player_<hoster>#player_M` ;
|
|
l'URL embed est le texte de `div#content_player_M` (URL complète ou simple ID selon
|
|
l'hébergeur — les templates de reconstruction viennent du JS `anime.js` du site).
|
|
- Pas de page par épisode : l'URL d'épisode est `<fiche>#buttons_N`.
|
|
"""
|
|
|
|
import logging
|
|
import re
|
|
from copy import deepcopy
|
|
from urllib.parse import quote
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
|
|
from app.scrapers.base import (
|
|
Episode,
|
|
ScrapeError,
|
|
SearchResult,
|
|
SourceScraper,
|
|
TitleDetails,
|
|
register_source,
|
|
)
|
|
from app.scrapers.config_loader import load_scraper_config
|
|
from app.scrapers.http import fetch_soup
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_CONFIG: dict = {
|
|
"search": {
|
|
"result": "div.search-result",
|
|
"link": "div.title a",
|
|
"image": "span.image img",
|
|
"genres": "ul.additional li",
|
|
},
|
|
"details": {
|
|
"title": "h1",
|
|
"poster": ".slide-poster img",
|
|
"synopsis": ".slide-desc",
|
|
"genres": '.slide-top li.right a[href*="/genre/"]',
|
|
"episode_badge": ".slide-poster .year",
|
|
"season_li": "ul.slide-top li",
|
|
},
|
|
"episodes": {
|
|
"option": "select.new_player_selector option",
|
|
"button": "div.button_box",
|
|
"content_prefix": "content_",
|
|
},
|
|
"latest": {
|
|
"path": "/animes-vostfr-recement-ajoutees.html",
|
|
"item": "div.movie-poster",
|
|
"link": ".play a", # href = fiche, alt = titre
|
|
"image": "span.image img",
|
|
},
|
|
# class CSS du lecteur → template d'URL ({} = valeur de #content_player_M ;
|
|
# chaîne vide = la valeur est déjà une URL complète)
|
|
"players": {
|
|
"new_player_vip": "",
|
|
"new_player_moevideo": "",
|
|
"new_player_sibnet": "https://video.sibnet.ru/shell.php?videoid={}",
|
|
"new_player_netu": "https://video.sibnet.ru/shell.php?videoid={}",
|
|
"new_player_uqload": "https://uqload.com/embed-{}.html",
|
|
"new_player_mp4": "https://www.mp4upload.com/embed-{}.html",
|
|
"new_player_fembed": "https://www.fembed.com/v/{}",
|
|
"new_player_mytv": "https://www.myvi.top/embed/{}",
|
|
"new_player_myvi": "https://myvi.ru/player/embed/html/{}",
|
|
"new_player_moevideo_mail": "",
|
|
"new_player_rutube": "https://rutube.ru/play/embed/{}",
|
|
"new_player_ok": "https://ok.ru/video/{}",
|
|
"new_player_mail2": "https://my.mail.ru/video/embed/{}",
|
|
"new_player_rapids": "https://rapidstream.co/embed-{}.html",
|
|
"new_player_gtv": "https://iframedream.com/embed/{}.html",
|
|
"new_player_cloudvideo": "https://cloudvideo.tv/embed-{}.html",
|
|
"new_player_uptostream": "https://uptostream.com/iframe/{}",
|
|
},
|
|
}
|
|
|
|
_SLUG_RE = re.compile(r"([^/]+)\.html?$")
|
|
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
|
|
_SEASON_RE = re.compile(r"Saison\s*:?\s*(\d+)", re.IGNORECASE)
|
|
_SAFE_FRAGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+$")
|
|
|
|
|
|
def _merged_config() -> dict:
|
|
merged = deepcopy(DEFAULT_CONFIG)
|
|
for key, values in load_scraper_config("vostfree").items():
|
|
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
|
merged[key].update(values)
|
|
else:
|
|
merged[key] = values
|
|
return merged
|
|
|
|
|
|
@register_source
|
|
class VostfreeScraper(SourceScraper):
|
|
name = "vostfree"
|
|
label = "Vostfree"
|
|
base_url = "https://ipv4.vostfree.ws"
|
|
media_types = ("anime",)
|
|
|
|
# ------------------------------------------------------------- helpers
|
|
|
|
def _title_url(self, source_id: str) -> str:
|
|
return f"{self.base_url}/{source_id}.html"
|
|
|
|
@staticmethod
|
|
def _slug_from_url(url: str) -> str | None:
|
|
match = _SLUG_RE.search(url)
|
|
return match.group(1) if match else None
|
|
|
|
@staticmethod
|
|
def _text(element: Tag | None) -> str:
|
|
return element.get_text(" ", strip=True) if element else ""
|
|
|
|
# ------------------------------------------------------------- search
|
|
|
|
async def search(self, query: str) -> list[SearchResult]:
|
|
config = _merged_config()
|
|
url = f"{self.base_url}/index.php?do=search&subaction=search&story={quote(query)}"
|
|
soup = await fetch_soup(url)
|
|
results: list[SearchResult] = []
|
|
for block in soup.select(config["search"]["result"]):
|
|
link = block.select_one(config["search"]["link"])
|
|
href = link.get("href") if link else None
|
|
if not href:
|
|
logger.warning("vostfree : bloc de résultat sans lien, ignoré")
|
|
continue
|
|
source_id = self._slug_from_url(href)
|
|
if not source_id:
|
|
logger.warning("vostfree : slug introuvable dans %s", href)
|
|
continue
|
|
image = block.select_one(config["search"]["image"])
|
|
genres_text = " ".join(
|
|
li.get_text(" ", strip=True) for li in block.select(config["search"]["genres"])
|
|
)
|
|
results.append(
|
|
SearchResult(
|
|
source=self.name,
|
|
source_id=source_id,
|
|
title=self._text(link),
|
|
url=href,
|
|
image_url=image.get("src") if image else None,
|
|
media_type="film"
|
|
if re.search(r"\bfilm", genres_text, re.IGNORECASE)
|
|
else "anime",
|
|
)
|
|
)
|
|
logger.info("vostfree : %d résultats pour %r", len(results), query)
|
|
return results
|
|
|
|
# ------------------------------------------------------------- latest
|
|
|
|
async def latest(self) -> list[SearchResult]:
|
|
"""Ajouts récents — page « Animes VOSTFR récemment ajoutés »."""
|
|
config = _merged_config()
|
|
latest_cfg = config["latest"]
|
|
soup = await fetch_soup(f"{self.base_url}{latest_cfg['path']}")
|
|
results: list[SearchResult] = []
|
|
for block in soup.select(latest_cfg["item"]):
|
|
link = block.select_one(latest_cfg["link"])
|
|
href = link.get("href") if link else None
|
|
if not href:
|
|
logger.warning("vostfree : bloc nouveauté sans lien, ignoré")
|
|
continue
|
|
source_id = self._slug_from_url(href)
|
|
if not source_id:
|
|
logger.warning("vostfree : slug introuvable dans %s", href)
|
|
continue
|
|
image = block.select_one(latest_cfg["image"])
|
|
title = link.get("alt") or self._text(link)
|
|
results.append(
|
|
SearchResult(
|
|
source=self.name,
|
|
source_id=source_id,
|
|
title=title,
|
|
url=href,
|
|
image_url=image.get("src") if image else None,
|
|
)
|
|
)
|
|
logger.info("vostfree : %d nouveautés récupérées", len(results))
|
|
return results
|
|
# ------------------------------------------------------------- details
|
|
|
|
async def get_details(self, source_id: str) -> TitleDetails:
|
|
config = _merged_config()
|
|
url = self._title_url(source_id)
|
|
soup = await fetch_soup(url)
|
|
details_cfg = config["details"]
|
|
|
|
title_el = soup.select_one(details_cfg["title"])
|
|
if title_el is None:
|
|
raise ScrapeError(f"vostfree : fiche introuvable pour {source_id} ({url})")
|
|
|
|
synopsis_el = soup.select_one(details_cfg["synopsis"])
|
|
if synopsis_el is not None:
|
|
for cast in synopsis_el.select(".cast"):
|
|
cast.decompose()
|
|
synopsis = self._text(synopsis_el) or None
|
|
|
|
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
|
|
badge_match = _NUMBER_RE.search(badge)
|
|
|
|
season = 1
|
|
for li in soup.select(details_cfg["season_li"]):
|
|
season_match = _SEASON_RE.search(self._text(li))
|
|
if season_match:
|
|
season = int(season_match.group(1))
|
|
break
|
|
|
|
episodes = self._parse_episodes(soup, url, season)
|
|
|
|
return TitleDetails(
|
|
source=self.name,
|
|
source_id=source_id,
|
|
title=self._text(title_el),
|
|
url=url,
|
|
synopsis=synopsis,
|
|
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
|
|
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
|
|
episode_count=int(badge_match.group(1).replace(",", ".")) if badge_match else None,
|
|
episodes=episodes,
|
|
)
|
|
|
|
# ------------------------------------------------------------ episodes
|
|
|
|
async def list_episodes(self, source_id: str) -> list[Episode]:
|
|
url = self._title_url(source_id)
|
|
soup = await fetch_soup(url)
|
|
season = self._find_season(soup)
|
|
episodes = self._parse_episodes(soup, url, season)
|
|
if not episodes:
|
|
raise ScrapeError(f"vostfree : aucun épisode trouvé pour {source_id} ({url})")
|
|
return episodes
|
|
|
|
def _find_season(self, soup: BeautifulSoup) -> int:
|
|
config = _merged_config()
|
|
for li in soup.select(config["details"]["season_li"]):
|
|
season_match = _SEASON_RE.search(self._text(li))
|
|
if season_match:
|
|
return int(season_match.group(1))
|
|
return 1
|
|
|
|
def _parse_episodes(self, soup: BeautifulSoup, page_url: str, season: int) -> list[Episode]:
|
|
config = _merged_config()
|
|
options = soup.select(config["episodes"]["option"])
|
|
entries: list[tuple[str, str]] = []
|
|
seen_ids: set[str] = set()
|
|
for opt in options:
|
|
value = opt.get("value", "")
|
|
if value in seen_ids:
|
|
continue # le site duplique parfois une option (ex. buttons_268)
|
|
seen_ids.add(value)
|
|
entries.append((value, opt.get_text(strip=True)))
|
|
if not entries: # fiche sans sélecteur : on retombe sur les blocs de boutons
|
|
entries = [
|
|
(box.get("id", ""), f"Episode {index}")
|
|
for index, box in enumerate(soup.select(config["episodes"]["button"]), start=1)
|
|
]
|
|
episodes: list[Episode] = []
|
|
for index, (button_id, label) in enumerate(entries, start=1):
|
|
number_match = _NUMBER_RE.search(label)
|
|
number = (
|
|
float(number_match.group(1).replace(",", ".")) if number_match else float(index)
|
|
)
|
|
episodes.append(
|
|
Episode(
|
|
number=number,
|
|
title=label or f"Episode {index}",
|
|
url=f"{page_url}#{button_id}" if button_id else page_url,
|
|
season=season,
|
|
)
|
|
)
|
|
return episodes
|
|
|
|
# -------------------------------------------------------------- embeds
|
|
|
|
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
|
config = _merged_config()
|
|
page_url, _, fragment = episode_url.partition("#")
|
|
soup = await fetch_soup(page_url)
|
|
button_cfg = config["episodes"]
|
|
|
|
button: Tag | None = None
|
|
if fragment and _SAFE_FRAGMENT_RE.match(fragment):
|
|
button = soup.select_one(f"#{fragment}")
|
|
if button is None:
|
|
logger.warning(
|
|
"vostfree : fragment #%s sans bloc correspondant dans %s", fragment, page_url
|
|
)
|
|
if button is None:
|
|
button = soup.select_one(button_cfg["button"])
|
|
if button is None:
|
|
raise ScrapeError(f"vostfree : aucun lecteur trouvé sur {episode_url}")
|
|
|
|
prefix = button_cfg["content_prefix"]
|
|
links: list[str] = []
|
|
for player in button.find_all("div", recursive=False):
|
|
player_id = player.get("id")
|
|
content = soup.select_one(f"#{prefix}{player_id}") if player_id else None
|
|
if content is None:
|
|
logger.warning(
|
|
"vostfree : contenu absent pour le lecteur #%s (%s)", player_id, page_url
|
|
)
|
|
continue
|
|
value = content.get_text(strip=True)
|
|
if not value:
|
|
continue
|
|
player_class = (player.get("class") or [""])[0]
|
|
template = config["players"].get(player_class)
|
|
if value.startswith("http"):
|
|
links.append(value)
|
|
elif template is not None:
|
|
links.append(template.format(value))
|
|
else:
|
|
logger.warning(
|
|
"vostfree : pas de template pour le lecteur %r (valeur %r)", player_class, value
|
|
)
|
|
if not links:
|
|
raise ScrapeError(f"vostfree : aucun lien embed extrait de {episode_url}")
|
|
logger.info("vostfree : %d liens embed pour %s", len(links), episode_url)
|
|
return links
|