v0.1.0 — Réécriture complète de OhmStreaming
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
This commit is contained in:
Roman
@@ -0,0 +1,327 @@
|
||||
"""Source Vostfree (ipv4.vostfree.ws) — moteur DataLife Engine, animes/films VF & VOSTFR.
|
||||
|
||||
Faits structurels (vérifiés en live) :
|
||||
- Recherche : GET /index.php?do=search&subaction=search&story=<q> → blocs `div.search-result`.
|
||||
- Nouveautés : /animes-vostfr-recement-ajoutees.html → blocs `div.movie-poster`
|
||||
(lien `.play a`, alt = titre, poster `span.image img`).
|
||||
- Fiche : `/444-telecharger-....html` — métadonnées dans `.slide-*`, épisodes dans un
|
||||
`select.new_player_selector` (une `option value="buttons_N"` par épisode).
|
||||
- Chaque `div#buttons_N` contient un ou plusieurs `div.new_player_<hoster>#player_M` ;
|
||||
l'URL embed est le texte de `div#content_player_M` (URL complète ou simple ID selon
|
||||
l'hébergeur — les templates de reconstruction viennent du JS `anime.js` du site).
|
||||
- Pas de page par épisode : l'URL d'épisode est `<fiche>#buttons_N`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from urllib.parse import quote
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from app.scrapers.base import (
|
||||
Episode,
|
||||
ScrapeError,
|
||||
SearchResult,
|
||||
SourceScraper,
|
||||
TitleDetails,
|
||||
register_source,
|
||||
)
|
||||
from app.scrapers.config_loader import load_scraper_config
|
||||
from app.scrapers.http import fetch_soup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_CONFIG: dict = {
|
||||
"search": {
|
||||
"result": "div.search-result",
|
||||
"link": "div.title a",
|
||||
"image": "span.image img",
|
||||
"genres": "ul.additional li",
|
||||
},
|
||||
"details": {
|
||||
"title": "h1",
|
||||
"poster": ".slide-poster img",
|
||||
"synopsis": ".slide-desc",
|
||||
"genres": '.slide-top li.right a[href*="/genre/"]',
|
||||
"episode_badge": ".slide-poster .year",
|
||||
"season_li": "ul.slide-top li",
|
||||
},
|
||||
"episodes": {
|
||||
"option": "select.new_player_selector option",
|
||||
"button": "div.button_box",
|
||||
"content_prefix": "content_",
|
||||
},
|
||||
"latest": {
|
||||
"path": "/animes-vostfr-recement-ajoutees.html",
|
||||
"item": "div.movie-poster",
|
||||
"link": ".play a", # href = fiche, alt = titre
|
||||
"image": "span.image img",
|
||||
},
|
||||
# class CSS du lecteur → template d'URL ({} = valeur de #content_player_M ;
|
||||
# chaîne vide = la valeur est déjà une URL complète)
|
||||
"players": {
|
||||
"new_player_vip": "",
|
||||
"new_player_moevideo": "",
|
||||
"new_player_sibnet": "https://video.sibnet.ru/shell.php?videoid={}",
|
||||
"new_player_netu": "https://video.sibnet.ru/shell.php?videoid={}",
|
||||
"new_player_uqload": "https://uqload.com/embed-{}.html",
|
||||
"new_player_mp4": "https://www.mp4upload.com/embed-{}.html",
|
||||
"new_player_fembed": "https://www.fembed.com/v/{}",
|
||||
"new_player_mytv": "https://www.myvi.top/embed/{}",
|
||||
"new_player_myvi": "https://myvi.ru/player/embed/html/{}",
|
||||
"new_player_moevideo_mail": "",
|
||||
"new_player_rutube": "https://rutube.ru/play/embed/{}",
|
||||
"new_player_ok": "https://ok.ru/video/{}",
|
||||
"new_player_mail2": "https://my.mail.ru/video/embed/{}",
|
||||
"new_player_rapids": "https://rapidstream.co/embed-{}.html",
|
||||
"new_player_gtv": "https://iframedream.com/embed/{}.html",
|
||||
"new_player_cloudvideo": "https://cloudvideo.tv/embed-{}.html",
|
||||
"new_player_uptostream": "https://uptostream.com/iframe/{}",
|
||||
},
|
||||
}
|
||||
|
||||
_SLUG_RE = re.compile(r"([^/]+)\.html?$")
|
||||
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
|
||||
_SEASON_RE = re.compile(r"Saison\s*:?\s*(\d+)", re.IGNORECASE)
|
||||
_SAFE_FRAGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+$")
|
||||
|
||||
|
||||
def _merged_config() -> dict:
|
||||
merged = deepcopy(DEFAULT_CONFIG)
|
||||
for key, values in load_scraper_config("vostfree").items():
|
||||
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
||||
merged[key].update(values)
|
||||
else:
|
||||
merged[key] = values
|
||||
return merged
|
||||
|
||||
|
||||
@register_source
|
||||
class VostfreeScraper(SourceScraper):
|
||||
name = "vostfree"
|
||||
label = "Vostfree"
|
||||
base_url = "https://ipv4.vostfree.ws"
|
||||
media_types = ("anime",)
|
||||
|
||||
# ------------------------------------------------------------- helpers
|
||||
|
||||
def _title_url(self, source_id: str) -> str:
|
||||
return f"{self.base_url}/{source_id}.html"
|
||||
|
||||
@staticmethod
|
||||
def _slug_from_url(url: str) -> str | None:
|
||||
match = _SLUG_RE.search(url)
|
||||
return match.group(1) if match else None
|
||||
|
||||
@staticmethod
|
||||
def _text(element: Tag | None) -> str:
|
||||
return element.get_text(" ", strip=True) if element else ""
|
||||
|
||||
# ------------------------------------------------------------- search
|
||||
|
||||
async def search(self, query: str) -> list[SearchResult]:
|
||||
config = _merged_config()
|
||||
url = f"{self.base_url}/index.php?do=search&subaction=search&story={quote(query)}"
|
||||
soup = await fetch_soup(url)
|
||||
results: list[SearchResult] = []
|
||||
for block in soup.select(config["search"]["result"]):
|
||||
link = block.select_one(config["search"]["link"])
|
||||
href = link.get("href") if link else None
|
||||
if not href:
|
||||
logger.warning("vostfree : bloc de résultat sans lien, ignoré")
|
||||
continue
|
||||
source_id = self._slug_from_url(href)
|
||||
if not source_id:
|
||||
logger.warning("vostfree : slug introuvable dans %s", href)
|
||||
continue
|
||||
image = block.select_one(config["search"]["image"])
|
||||
genres_text = " ".join(
|
||||
li.get_text(" ", strip=True) for li in block.select(config["search"]["genres"])
|
||||
)
|
||||
results.append(
|
||||
SearchResult(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=self._text(link),
|
||||
url=href,
|
||||
image_url=image.get("src") if image else None,
|
||||
media_type="film"
|
||||
if re.search(r"\bfilm", genres_text, re.IGNORECASE)
|
||||
else "anime",
|
||||
)
|
||||
)
|
||||
logger.info("vostfree : %d résultats pour %r", len(results), query)
|
||||
return results
|
||||
|
||||
# ------------------------------------------------------------- latest
|
||||
|
||||
async def latest(self) -> list[SearchResult]:
|
||||
"""Ajouts récents — page « Animes VOSTFR récemment ajoutés »."""
|
||||
config = _merged_config()
|
||||
latest_cfg = config["latest"]
|
||||
soup = await fetch_soup(f"{self.base_url}{latest_cfg['path']}")
|
||||
results: list[SearchResult] = []
|
||||
for block in soup.select(latest_cfg["item"]):
|
||||
link = block.select_one(latest_cfg["link"])
|
||||
href = link.get("href") if link else None
|
||||
if not href:
|
||||
logger.warning("vostfree : bloc nouveauté sans lien, ignoré")
|
||||
continue
|
||||
source_id = self._slug_from_url(href)
|
||||
if not source_id:
|
||||
logger.warning("vostfree : slug introuvable dans %s", href)
|
||||
continue
|
||||
image = block.select_one(latest_cfg["image"])
|
||||
title = link.get("alt") or self._text(link)
|
||||
results.append(
|
||||
SearchResult(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=title,
|
||||
url=href,
|
||||
image_url=image.get("src") if image else None,
|
||||
)
|
||||
)
|
||||
logger.info("vostfree : %d nouveautés récupérées", len(results))
|
||||
return results
|
||||
# ------------------------------------------------------------- details
|
||||
|
||||
async def get_details(self, source_id: str) -> TitleDetails:
|
||||
config = _merged_config()
|
||||
url = self._title_url(source_id)
|
||||
soup = await fetch_soup(url)
|
||||
details_cfg = config["details"]
|
||||
|
||||
title_el = soup.select_one(details_cfg["title"])
|
||||
if title_el is None:
|
||||
raise ScrapeError(f"vostfree : fiche introuvable pour {source_id} ({url})")
|
||||
|
||||
synopsis_el = soup.select_one(details_cfg["synopsis"])
|
||||
if synopsis_el is not None:
|
||||
for cast in synopsis_el.select(".cast"):
|
||||
cast.decompose()
|
||||
synopsis = self._text(synopsis_el) or None
|
||||
|
||||
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
|
||||
badge_match = _NUMBER_RE.search(badge)
|
||||
|
||||
season = 1
|
||||
for li in soup.select(details_cfg["season_li"]):
|
||||
season_match = _SEASON_RE.search(self._text(li))
|
||||
if season_match:
|
||||
season = int(season_match.group(1))
|
||||
break
|
||||
|
||||
episodes = self._parse_episodes(soup, url, season)
|
||||
|
||||
return TitleDetails(
|
||||
source=self.name,
|
||||
source_id=source_id,
|
||||
title=self._text(title_el),
|
||||
url=url,
|
||||
synopsis=synopsis,
|
||||
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
|
||||
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
|
||||
episode_count=int(badge_match.group(1).replace(",", ".")) if badge_match else None,
|
||||
episodes=episodes,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------ episodes
|
||||
|
||||
async def list_episodes(self, source_id: str) -> list[Episode]:
|
||||
url = self._title_url(source_id)
|
||||
soup = await fetch_soup(url)
|
||||
season = self._find_season(soup)
|
||||
episodes = self._parse_episodes(soup, url, season)
|
||||
if not episodes:
|
||||
raise ScrapeError(f"vostfree : aucun épisode trouvé pour {source_id} ({url})")
|
||||
return episodes
|
||||
|
||||
def _find_season(self, soup: BeautifulSoup) -> int:
|
||||
config = _merged_config()
|
||||
for li in soup.select(config["details"]["season_li"]):
|
||||
season_match = _SEASON_RE.search(self._text(li))
|
||||
if season_match:
|
||||
return int(season_match.group(1))
|
||||
return 1
|
||||
|
||||
def _parse_episodes(self, soup: BeautifulSoup, page_url: str, season: int) -> list[Episode]:
|
||||
config = _merged_config()
|
||||
options = soup.select(config["episodes"]["option"])
|
||||
entries: list[tuple[str, str]] = []
|
||||
seen_ids: set[str] = set()
|
||||
for opt in options:
|
||||
value = opt.get("value", "")
|
||||
if value in seen_ids:
|
||||
continue # le site duplique parfois une option (ex. buttons_268)
|
||||
seen_ids.add(value)
|
||||
entries.append((value, opt.get_text(strip=True)))
|
||||
if not entries: # fiche sans sélecteur : on retombe sur les blocs de boutons
|
||||
entries = [
|
||||
(box.get("id", ""), f"Episode {index}")
|
||||
for index, box in enumerate(soup.select(config["episodes"]["button"]), start=1)
|
||||
]
|
||||
episodes: list[Episode] = []
|
||||
for index, (button_id, label) in enumerate(entries, start=1):
|
||||
number_match = _NUMBER_RE.search(label)
|
||||
number = (
|
||||
float(number_match.group(1).replace(",", ".")) if number_match else float(index)
|
||||
)
|
||||
episodes.append(
|
||||
Episode(
|
||||
number=number,
|
||||
title=label or f"Episode {index}",
|
||||
url=f"{page_url}#{button_id}" if button_id else page_url,
|
||||
season=season,
|
||||
)
|
||||
)
|
||||
return episodes
|
||||
|
||||
# -------------------------------------------------------------- embeds
|
||||
|
||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
||||
config = _merged_config()
|
||||
page_url, _, fragment = episode_url.partition("#")
|
||||
soup = await fetch_soup(page_url)
|
||||
button_cfg = config["episodes"]
|
||||
|
||||
button: Tag | None = None
|
||||
if fragment and _SAFE_FRAGMENT_RE.match(fragment):
|
||||
button = soup.select_one(f"#{fragment}")
|
||||
if button is None:
|
||||
logger.warning(
|
||||
"vostfree : fragment #%s sans bloc correspondant dans %s", fragment, page_url
|
||||
)
|
||||
if button is None:
|
||||
button = soup.select_one(button_cfg["button"])
|
||||
if button is None:
|
||||
raise ScrapeError(f"vostfree : aucun lecteur trouvé sur {episode_url}")
|
||||
|
||||
prefix = button_cfg["content_prefix"]
|
||||
links: list[str] = []
|
||||
for player in button.find_all("div", recursive=False):
|
||||
player_id = player.get("id")
|
||||
content = soup.select_one(f"#{prefix}{player_id}") if player_id else None
|
||||
if content is None:
|
||||
logger.warning(
|
||||
"vostfree : contenu absent pour le lecteur #%s (%s)", player_id, page_url
|
||||
)
|
||||
continue
|
||||
value = content.get_text(strip=True)
|
||||
if not value:
|
||||
continue
|
||||
player_class = (player.get("class") or [""])[0]
|
||||
template = config["players"].get(player_class)
|
||||
if value.startswith("http"):
|
||||
links.append(value)
|
||||
elif template is not None:
|
||||
links.append(template.format(value))
|
||||
else:
|
||||
logger.warning(
|
||||
"vostfree : pas de template pour le lecteur %r (valeur %r)", player_class, value
|
||||
)
|
||||
if not links:
|
||||
raise ScrapeError(f"vostfree : aucun lien embed extrait de {episode_url}")
|
||||
logger.info("vostfree : %d liens embed pour %s", len(links), episode_url)
|
||||
return links
|
||||
Reference in New Issue
Block a user