Ajout de la source French-Stream (french-stream.lat) : séries et films
- Scraper DataLife Engine : recherche AJAX, fiches par saison, épisodes via ep-data.php (VF/VOSTFR), films via film_api.php, nouveautés /series/ - Config YAML externalisée (sélecteurs réparables sans coder) - Domaines étendus : uqload.vc, vidzy.cc/.live - 11 tests (126 au total)
This commit is contained in:
@@ -111,7 +111,8 @@ Variables d'environnement (préfixe `OHM_`, voir `.env.example`) :
|
|||||||
|
|
||||||
## Fonctionnalités
|
## Fonctionnalités
|
||||||
|
|
||||||
- **Recherche unifiée** sur plusieurs sources (Vostfree, French-Manga, VoirAnime) — chaque source
|
- **Recherche unifiée** sur plusieurs sources (Vostfree, French-Manga, VoirAnime pour les
|
||||||
|
animes ; French-Stream pour les séries et films VF/VOSTFR) — chaque source
|
||||||
est un module interchangeable activable/désactivable à chaud (page Admin), dont l'URL
|
est un module interchangeable activable/désactivable à chaud (page Admin), dont l'URL
|
||||||
est modifiable à la volée (utile si un site change de domaine).
|
est modifiable à la volée (utile si un site change de domaine).
|
||||||
- **Extraction en 2 niveaux** : page d'épisode → lecteurs embarqués → URL directe
|
- **Extraction en 2 niveaux** : page d'épisode → lecteurs embarqués → URL directe
|
||||||
@@ -200,7 +201,7 @@ app/
|
|||||||
├── scrapers/
|
├── scrapers/
|
||||||
│ ├── base.py # contrats SourceScraper/HosterExtractor + registres
|
│ ├── base.py # contrats SourceScraper/HosterExtractor + registres
|
||||||
│ ├── configs/ # sélecteurs YAML externalisés (réparer sans coder)
|
│ ├── configs/ # sélecteurs YAML externalisés (réparer sans coder)
|
||||||
│ ├── sources/ # vostfree, french_manga, voiranime
|
│ ├── sources/ # vostfree, french_manga, voiranime, french_stream
|
||||||
├── services/ # downloads, kitsu, discover, sonarr, torznab, settings
|
├── services/ # downloads, kitsu, discover, sonarr, torznab, settings
|
||||||
└── templates/ + static/ # UI htmx + Alpine.js, thème sombre
|
└── templates/ + static/ # UI htmx + Alpine.js, thème sombre
|
||||||
```
|
```
|
||||||
@@ -211,6 +212,6 @@ app/
|
|||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv run pytest # 86 tests
|
uv run pytest # 126 tests
|
||||||
uv run ruff check . # lint
|
uv run ruff check . # lint
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# Sélecteurs/endpoints French-Stream (french-stream.lat, DataLife Engine) — surchargeables
|
||||||
|
# sans toucher au code. Structure vérifiée en live (films & séries VF/VOSTFR).
|
||||||
|
|
||||||
|
endpoints:
|
||||||
|
search: "/engine/ajax/search.php" # POST query=<q>&page=1
|
||||||
|
episodes: "/ep-data.php?id={newsid}&format=js" # JSON {vf, vostfr, vo, info}
|
||||||
|
film: "/engine/ajax/film_api.php?id={newsid}" # JSON {players: {hoster: {version: url}}}
|
||||||
|
|
||||||
|
search:
|
||||||
|
item: "div.search-item" # bloc résultat (lien dans onclick="location.href='...'")
|
||||||
|
title: ".search-title"
|
||||||
|
poster: ".search-poster img"
|
||||||
|
|
||||||
|
details:
|
||||||
|
title: "h1#s-title"
|
||||||
|
synopsis: "div.fdesc" # le boilerplate p.desc-text est retiré
|
||||||
|
synopsis_boilerplate: "p.desc-text"
|
||||||
|
genres: ".facts .genres"
|
||||||
|
year: ".facts .release"
|
||||||
|
poster_serie: ".fposter img"
|
||||||
|
poster_film: "#film-data" # attribut data-affiche
|
||||||
|
serie_marker: "#serie-config" # présent = fiche série (sinon film)
|
||||||
|
|
||||||
|
latest:
|
||||||
|
path: "/series/" # mix films/séries ; « Saison » dans le titre = série
|
||||||
|
item: "div.short"
|
||||||
|
link: "a.short-poster" # href = fiche (/index.php?newsid=N), alt = titre
|
||||||
|
image: "img"
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Extracteur Uqload (uqload.to/.co/.com/.io) — page embed → mp4/m3u8.
|
"""Extracteur Uqload (uqload.to/.co/.com/.io/.vc) — page embed → mp4/m3u8.
|
||||||
|
|
||||||
La page embed contient un jwplayer configuré dans du JS packé
|
La page embed contient un jwplayer configuré dans du JS packé
|
||||||
(p,a,c,k,e,d) : `sources:[{file:"https://.../master.m3u8?..."}]`.
|
(p,a,c,k,e,d) : `sources:[{file:"https://.../master.m3u8?..."}]`.
|
||||||
@@ -24,7 +24,7 @@ _PATTERNS = (
|
|||||||
@register_hoster
|
@register_hoster
|
||||||
class UqloadExtractor(HosterExtractor):
|
class UqloadExtractor(HosterExtractor):
|
||||||
name = "uqload"
|
name = "uqload"
|
||||||
domains = ("uqload.to", "uqload.co", "uqload.com", "uqload.io")
|
domains = ("uqload.to", "uqload.co", "uqload.com", "uqload.io", "uqload.vc")
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
async def extract(self, embed_url: str) -> VideoLink:
|
||||||
html = await fetch(embed_url)
|
html = await fetch(embed_url)
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Extracteur Vidzy (vidzy.org) — page embed videojs → m3u8.
|
"""Extracteur Vidzy (vidzy.org/.cc/.live) — page embed videojs → m3u8.
|
||||||
|
|
||||||
La page embed ne contient pas l'URL vidéo en clair : le script videojs appelle
|
La page embed ne contient pas l'URL vidéo en clair : le script videojs appelle
|
||||||
une fonction de décodage inline `atob(s)` + reverse + XOR, avec une graine
|
une fonction de décodage inline `atob(s)` + reverse + XOR, avec une graine
|
||||||
@@ -45,7 +45,7 @@ def _decode(b64: str, hostname: str) -> str | None:
|
|||||||
@register_hoster
|
@register_hoster
|
||||||
class VidzyExtractor(HosterExtractor):
|
class VidzyExtractor(HosterExtractor):
|
||||||
name = "vidzy"
|
name = "vidzy"
|
||||||
domains = ("vidzy.org",)
|
domains = ("vidzy.org", "vidzy.cc", "vidzy.live")
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
async def extract(self, embed_url: str) -> VideoLink:
|
||||||
html = await fetch(embed_url)
|
html = await fetch(embed_url)
|
||||||
|
|||||||
@@ -42,15 +42,19 @@ async def fetch(
|
|||||||
referer: str | None = None,
|
referer: str | None = None,
|
||||||
retries: int = 2,
|
retries: int = 2,
|
||||||
headers: dict[str, str] | None = None,
|
headers: dict[str, str] | None = None,
|
||||||
|
data: dict[str, str] | None = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""GET d'une page avec retries ; lève ScrapeError en cas d'échec définitif."""
|
"""GET (ou POST si `data` est fourni) avec retries ; lève ScrapeError en cas d'échec définitif."""
|
||||||
request_headers = dict(headers) if headers else {}
|
request_headers = dict(headers) if headers else {}
|
||||||
if referer:
|
if referer:
|
||||||
request_headers.setdefault("Referer", referer)
|
request_headers.setdefault("Referer", referer)
|
||||||
last_error: Exception | None = None
|
last_error: Exception | None = None
|
||||||
for attempt in range(retries + 1):
|
for attempt in range(retries + 1):
|
||||||
try:
|
try:
|
||||||
|
if data is None:
|
||||||
response = await get_client().get(url, headers=request_headers)
|
response = await get_client().get(url, headers=request_headers)
|
||||||
|
else:
|
||||||
|
response = await get_client().post(url, headers=request_headers, data=data)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return response.text
|
return response.text
|
||||||
except httpx.HTTPStatusError as exc:
|
except httpx.HTTPStatusError as exc:
|
||||||
|
|||||||
@@ -0,0 +1,348 @@
|
|||||||
|
"""Source French-Stream (french-stream.lat) — films & séries VF/VOSTFR, moteur DataLife Engine.
|
||||||
|
|
||||||
|
Faits structurels (vérifiés en live) :
|
||||||
|
- Recherche : POST /engine/ajax/search.php (query, page) → blocs `div.search-item`
|
||||||
|
(lien dans onclick="location.href='...'", poster `.search-poster img`).
|
||||||
|
- Fiche : `/index.php?newsid=<id>` (ou la jolie URL `/<id>-<slug>.html`).
|
||||||
|
`source_id` = newsid numérique.
|
||||||
|
- Séries : une fiche par saison (« Titre - Saison N »). Épisodes via
|
||||||
|
GET /ep-data.php?id=<newsid>&format=js → JSON
|
||||||
|
`{"vf": {"1": {"vidzy": url, "uqload": url, ...}}, "vostfr": {...}, "info": {...}}`.
|
||||||
|
- Films : lecteurs via GET /engine/ajax/film_api.php?id=<newsid> → JSON
|
||||||
|
`{"players": {"vidzy": {"default": url, "vff": url, "vostfr": url, ...}}}`.
|
||||||
|
- URL d'épisode interne : `<fiche>#vf-3` (série) ou `<fiche>#film` (film).
|
||||||
|
- Nouveautés : /series/ → blocs `div.short` (mix films/séries, « Saison » dans le
|
||||||
|
titre = série).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from copy import deepcopy
|
||||||
|
|
||||||
|
from bs4 import BeautifulSoup, Tag
|
||||||
|
|
||||||
|
from app.scrapers.base import (
|
||||||
|
Episode,
|
||||||
|
ScrapeError,
|
||||||
|
SearchResult,
|
||||||
|
SourceScraper,
|
||||||
|
TitleDetails,
|
||||||
|
register_source,
|
||||||
|
)
|
||||||
|
from app.scrapers.config_loader import load_scraper_config
|
||||||
|
from app.scrapers.http import fetch, fetch_soup
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_CONFIG: dict = {
|
||||||
|
"endpoints": {
|
||||||
|
"search": "/engine/ajax/search.php",
|
||||||
|
"episodes": "/ep-data.php?id={newsid}&format=js",
|
||||||
|
"film": "/engine/ajax/film_api.php?id={newsid}",
|
||||||
|
},
|
||||||
|
"search": {
|
||||||
|
"item": "div.search-item",
|
||||||
|
"title": ".search-title",
|
||||||
|
"poster": ".search-poster img",
|
||||||
|
},
|
||||||
|
"details": {
|
||||||
|
"title": "h1#s-title",
|
||||||
|
"synopsis": "div.fdesc",
|
||||||
|
"synopsis_boilerplate": "p.desc-text",
|
||||||
|
"genres": ".facts .genres",
|
||||||
|
"year": ".facts .release",
|
||||||
|
"poster_serie": ".fposter img",
|
||||||
|
"poster_film": "#film-data",
|
||||||
|
"serie_marker": "#serie-config",
|
||||||
|
},
|
||||||
|
"latest": {
|
||||||
|
"path": "/series/",
|
||||||
|
"item": "div.short",
|
||||||
|
"link": "a.short-poster",
|
||||||
|
"image": "img",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_NEWSID_RE = re.compile(r"(?:newsid=|/)(\d+)(?:-[^/]*)?\.?html?$|(?:newsid=)(\d+)")
|
||||||
|
_ONCLICK_URL_RE = re.compile(r"location\.href='([^']+)'")
|
||||||
|
_SEASON_RE = re.compile(r"saison\s*:?\s*(\d+)", re.IGNORECASE)
|
||||||
|
_YEAR_RE = re.compile(r"(\d{4})")
|
||||||
|
_FRAGMENT_RE = re.compile(r"^(vf|vostfr|vo)-(\d+(?:\.\d+)?)$")
|
||||||
|
_VERSION_ORDER = {"vf": 0, "vostfr": 1, "vo": 2}
|
||||||
|
|
||||||
|
|
||||||
|
def _merged_config() -> dict:
|
||||||
|
merged = deepcopy(DEFAULT_CONFIG)
|
||||||
|
for key, values in load_scraper_config("french_stream").items():
|
||||||
|
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
||||||
|
merged[key].update(values)
|
||||||
|
else:
|
||||||
|
merged[key] = values
|
||||||
|
return merged
|
||||||
|
|
||||||
|
|
||||||
|
def _newsid_from_url(url: str) -> str | None:
|
||||||
|
match = _NEWSID_RE.search(url)
|
||||||
|
if not match:
|
||||||
|
return None
|
||||||
|
return match.group(1) or match.group(2)
|
||||||
|
|
||||||
|
|
||||||
|
def _media_type(title: str, url: str) -> str:
|
||||||
|
if _SEASON_RE.search(title) or "-saison-" in url:
|
||||||
|
return "serie"
|
||||||
|
return "film"
|
||||||
|
|
||||||
|
|
||||||
|
@register_source
|
||||||
|
class FrenchStreamScraper(SourceScraper):
|
||||||
|
name = "french_stream"
|
||||||
|
label = "French-Stream"
|
||||||
|
base_url = "https://french-stream.lat"
|
||||||
|
media_types = ("serie", "film")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- helpers
|
||||||
|
|
||||||
|
def _title_url(self, source_id: str) -> str:
|
||||||
|
return f"{self.base_url}/index.php?newsid={source_id}"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _text(element: Tag | None) -> str:
|
||||||
|
return element.get_text(" ", strip=True) if element else ""
|
||||||
|
|
||||||
|
def _is_serie(self, soup: BeautifulSoup, title: str, url: str) -> bool:
|
||||||
|
config = _merged_config()
|
||||||
|
if soup.select_one(config["details"]["serie_marker"]):
|
||||||
|
return True
|
||||||
|
return _media_type(title, url) == "serie"
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- search
|
||||||
|
|
||||||
|
async def search(self, query: str) -> list[SearchResult]:
|
||||||
|
config = _merged_config()
|
||||||
|
url = f"{self.base_url}{config['endpoints']['search']}"
|
||||||
|
html = await fetch(
|
||||||
|
url, referer=f"{self.base_url}/", data={"query": query, "page": "1"}
|
||||||
|
)
|
||||||
|
soup = BeautifulSoup(html, "lxml")
|
||||||
|
results: list[SearchResult] = []
|
||||||
|
for block in soup.select(config["search"]["item"]):
|
||||||
|
onclick = block.get("onclick", "")
|
||||||
|
url_match = _ONCLICK_URL_RE.search(onclick)
|
||||||
|
if not url_match:
|
||||||
|
logger.warning("french_stream : résultat sans lien, ignoré")
|
||||||
|
continue
|
||||||
|
href = url_match.group(1)
|
||||||
|
source_id = _newsid_from_url(href)
|
||||||
|
if not source_id:
|
||||||
|
logger.warning("french_stream : newsid introuvable dans %s", href)
|
||||||
|
continue
|
||||||
|
title = self._text(block.select_one(config["search"]["title"]))
|
||||||
|
image = block.select_one(config["search"]["poster"])
|
||||||
|
results.append(
|
||||||
|
SearchResult(
|
||||||
|
source=self.name,
|
||||||
|
source_id=source_id,
|
||||||
|
title=title,
|
||||||
|
url=f"{self.base_url}{href}" if href.startswith("/") else href,
|
||||||
|
image_url=image.get("src") if image else None,
|
||||||
|
media_type=_media_type(title, href),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logger.info("french_stream : %d résultats pour %r", len(results), query)
|
||||||
|
return results
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- latest
|
||||||
|
|
||||||
|
async def latest(self) -> list[SearchResult]:
|
||||||
|
"""Ajouts récents — page /series/ (mix films & séries)."""
|
||||||
|
config = _merged_config()
|
||||||
|
latest_cfg = config["latest"]
|
||||||
|
soup = await fetch_soup(f"{self.base_url}{latest_cfg['path']}")
|
||||||
|
results: list[SearchResult] = []
|
||||||
|
for block in soup.select(latest_cfg["item"]):
|
||||||
|
link = block.select_one(latest_cfg["link"])
|
||||||
|
href = link.get("href") if link else None
|
||||||
|
if not href:
|
||||||
|
continue
|
||||||
|
source_id = _newsid_from_url(href)
|
||||||
|
if not source_id:
|
||||||
|
logger.warning("french_stream : newsid introuvable dans %s", href)
|
||||||
|
continue
|
||||||
|
image = link.select_one(latest_cfg["image"])
|
||||||
|
title = link.get("alt") or self._text(link)
|
||||||
|
results.append(
|
||||||
|
SearchResult(
|
||||||
|
source=self.name,
|
||||||
|
source_id=source_id,
|
||||||
|
title=title,
|
||||||
|
url=f"{self.base_url}{href}" if href.startswith("/") else href,
|
||||||
|
image_url=image.get("src") if image else None,
|
||||||
|
media_type=_media_type(title, href),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logger.info("french_stream : %d nouveautés récupérées", len(results))
|
||||||
|
return results
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- details
|
||||||
|
|
||||||
|
async def get_details(self, source_id: str) -> TitleDetails:
|
||||||
|
config = _merged_config()
|
||||||
|
details_cfg = config["details"]
|
||||||
|
url = self._title_url(source_id)
|
||||||
|
soup = await fetch_soup(url)
|
||||||
|
|
||||||
|
title = " ".join(self._text(soup.select_one(details_cfg["title"])).split())
|
||||||
|
if not title:
|
||||||
|
raise ScrapeError(f"french_stream : fiche introuvable pour {source_id} ({url})")
|
||||||
|
|
||||||
|
synopsis_el = soup.select_one(details_cfg["synopsis"])
|
||||||
|
if synopsis_el is not None:
|
||||||
|
for boilerplate in synopsis_el.select(details_cfg["synopsis_boilerplate"]):
|
||||||
|
boilerplate.decompose()
|
||||||
|
synopsis = self._text(synopsis_el) or None
|
||||||
|
|
||||||
|
genres_el = soup.select_one(details_cfg["genres"])
|
||||||
|
if genres_el and genres_el.select("a"):
|
||||||
|
genres = [a.get_text(strip=True) for a in genres_el.select("a") if a.get_text(strip=True)]
|
||||||
|
else:
|
||||||
|
genres = [g.strip() for g in self._text(genres_el).split(",") if g.strip()]
|
||||||
|
|
||||||
|
year_match = _YEAR_RE.search(self._text(soup.select_one(details_cfg["year"])))
|
||||||
|
if year_match is None:
|
||||||
|
year_match = _YEAR_RE.search(title)
|
||||||
|
|
||||||
|
poster_el = soup.select_one(details_cfg["poster_serie"])
|
||||||
|
poster = poster_el.get("src") if poster_el else None
|
||||||
|
if not poster:
|
||||||
|
film_data = soup.select_one(details_cfg["poster_film"])
|
||||||
|
poster = film_data.get("data-affiche") if film_data else None
|
||||||
|
|
||||||
|
is_serie = self._is_serie(soup, title, url)
|
||||||
|
episodes = await self._fetch_episodes(source_id, url, title) if is_serie else []
|
||||||
|
|
||||||
|
return TitleDetails(
|
||||||
|
source=self.name,
|
||||||
|
source_id=source_id,
|
||||||
|
title=title,
|
||||||
|
url=url,
|
||||||
|
synopsis=synopsis,
|
||||||
|
image_url=poster,
|
||||||
|
genres=genres,
|
||||||
|
year=int(year_match.group(1)) if year_match else None,
|
||||||
|
episode_count=len(episodes) if is_serie else 1,
|
||||||
|
episodes=episodes if is_serie else [self._film_episode(url)],
|
||||||
|
media_type="serie" if is_serie else "film",
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ episodes
|
||||||
|
|
||||||
|
async def list_episodes(self, source_id: str) -> list[Episode]:
|
||||||
|
url = self._title_url(source_id)
|
||||||
|
soup = await fetch_soup(url)
|
||||||
|
title = " ".join(
|
||||||
|
self._text(soup.select_one(_merged_config()["details"]["title"])).split()
|
||||||
|
)
|
||||||
|
if not self._is_serie(soup, title, url):
|
||||||
|
return [self._film_episode(url)]
|
||||||
|
episodes = await self._fetch_episodes(source_id, url, title)
|
||||||
|
if not episodes:
|
||||||
|
raise ScrapeError(f"french_stream : aucun épisode trouvé pour {source_id} ({url})")
|
||||||
|
return episodes
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _film_episode(page_url: str) -> Episode:
|
||||||
|
return Episode(number=1, title="Film", url=f"{page_url}#film", season=1)
|
||||||
|
|
||||||
|
async def _fetch_episodes(
|
||||||
|
self, source_id: str, page_url: str, title: str
|
||||||
|
) -> list[Episode]:
|
||||||
|
"""Épisodes d'une saison via l'API JSON du site (versions vf/vostfr/vo)."""
|
||||||
|
config = _merged_config()
|
||||||
|
endpoint = config["endpoints"]["episodes"].format(newsid=source_id)
|
||||||
|
raw = await fetch(f"{self.base_url}{endpoint}", referer=page_url)
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ScrapeError(
|
||||||
|
f"french_stream : JSON d'épisodes invalide pour {source_id}"
|
||||||
|
) from exc
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return []
|
||||||
|
|
||||||
|
season_match = _SEASON_RE.search(title)
|
||||||
|
season = int(season_match.group(1)) if season_match else 1
|
||||||
|
info = data.get("info") if isinstance(data.get("info"), dict) else {}
|
||||||
|
|
||||||
|
episodes: list[Episode] = []
|
||||||
|
for version, eps in data.items():
|
||||||
|
if version == "info" or not isinstance(eps, dict):
|
||||||
|
continue
|
||||||
|
for number_key in eps:
|
||||||
|
number_match = re.match(r"^(\d+(?:\.\d+)?)$", number_key)
|
||||||
|
if not number_match:
|
||||||
|
continue
|
||||||
|
number = float(number_key)
|
||||||
|
ep_info = info.get(number_key) or info.get(str(int(number))) or {}
|
||||||
|
episodes.append(
|
||||||
|
Episode(
|
||||||
|
number=number,
|
||||||
|
title=ep_info.get("title") if isinstance(ep_info, dict) else None,
|
||||||
|
url=f"{page_url}#{version}-{number_key}",
|
||||||
|
season=season,
|
||||||
|
version=version,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
episodes.sort(
|
||||||
|
key=lambda e: (e.number, _VERSION_ORDER.get(e.version or "", 99))
|
||||||
|
)
|
||||||
|
return episodes
|
||||||
|
|
||||||
|
# -------------------------------------------------------------- embeds
|
||||||
|
|
||||||
|
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
||||||
|
config = _merged_config()
|
||||||
|
page_url, _, fragment = episode_url.partition("#")
|
||||||
|
source_id = _newsid_from_url(page_url)
|
||||||
|
if not source_id:
|
||||||
|
raise ScrapeError(f"french_stream : newsid introuvable dans {episode_url}")
|
||||||
|
|
||||||
|
if fragment == "film":
|
||||||
|
endpoint = config["endpoints"]["film"].format(newsid=source_id)
|
||||||
|
raw = await fetch(f"{self.base_url}{endpoint}", referer=page_url)
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ScrapeError(
|
||||||
|
f"french_stream : JSON film invalide pour {source_id}"
|
||||||
|
) from exc
|
||||||
|
links: list[str] = []
|
||||||
|
for hoster_urls in (data.get("players") or {}).values():
|
||||||
|
if not isinstance(hoster_urls, dict):
|
||||||
|
continue
|
||||||
|
for embed in hoster_urls.values():
|
||||||
|
if isinstance(embed, str) and embed.startswith("http") and embed not in links:
|
||||||
|
links.append(embed)
|
||||||
|
else:
|
||||||
|
fragment_match = _FRAGMENT_RE.match(fragment)
|
||||||
|
if not fragment_match:
|
||||||
|
raise ScrapeError(
|
||||||
|
f"french_stream : fragment d'épisode invalide dans {episode_url}"
|
||||||
|
)
|
||||||
|
version, number = fragment_match.group(1), fragment_match.group(2)
|
||||||
|
endpoint = config["endpoints"]["episodes"].format(newsid=source_id)
|
||||||
|
raw = await fetch(f"{self.base_url}{endpoint}", referer=page_url)
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ScrapeError(
|
||||||
|
f"french_stream : JSON d'épisodes invalide pour {source_id}"
|
||||||
|
) from exc
|
||||||
|
hosters = (data.get(version) or {}).get(number) or {}
|
||||||
|
links = [u for u in hosters.values() if isinstance(u, str) and u.startswith("http")]
|
||||||
|
|
||||||
|
if not links:
|
||||||
|
raise ScrapeError(f"french_stream : aucun lien embed extrait de {episode_url}")
|
||||||
|
logger.info("french_stream : %d liens embed pour %s", len(links), episode_url)
|
||||||
|
return links
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
"""Tests du scraper French-Stream (séries/films VF-VOSTFR) sur fixtures réelles."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.scrapers.base import ScrapeError, get_source, import_all_scrapers
|
||||||
|
from app.scrapers.sources.french_stream import FrenchStreamScraper
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ fixtures
|
||||||
|
|
||||||
|
SEARCH_HTML = """
|
||||||
|
<div class='search-item' onclick="location.href='/9562-breaking-bad-saison-2-streaming-complet-vf-vostfr.html'">
|
||||||
|
<div class='search-poster'><img src='https://image.tmdb.org/t/p/w400/bb.jpg' alt='Breaking Bad - Saison 2'></div>
|
||||||
|
<div class='search-info'><div class='search-title'>Breaking Bad - Saison 2</div></div>
|
||||||
|
</div>
|
||||||
|
<div class='search-item' onclick="location.href='/1022-inception-streaming-complet-vf-vostfr.html'">
|
||||||
|
<div class='search-poster'><img src='https://image.tmdb.org/t/p/w400/inc.jpg' alt='Inception'></div>
|
||||||
|
<div class='search-info'><div class='search-title'>Inception (2010)</div></div>
|
||||||
|
</div>
|
||||||
|
<div class='search-item'><div class='search-info'><div class='search-title'>Sans lien</div></div></div>
|
||||||
|
"""
|
||||||
|
|
||||||
|
SERIE_DETAILS_HTML = """
|
||||||
|
<div id="film-data" data-newsid="9562" data-affiche="https://image.tmdb.org/t/p/w300/bb.jpg"></div>
|
||||||
|
<div class="fmain"><div class="fcols fx-row"><div class="fmid">
|
||||||
|
<h1 id="s-title">Breaking Bad - Saison 2 </h1>
|
||||||
|
<div class="facts">
|
||||||
|
<span class="certification">-12</span> - <span class="release">2008 -</span>
|
||||||
|
<span class="genres">Drame, Crime</span> - <span class="runtime">48 min</span>
|
||||||
|
</div>
|
||||||
|
<div class="flist clearfix"><div class="fdesc"><p>Walter et Jesse montent leur propre affaire.</p></div></div>
|
||||||
|
</div></div></div>
|
||||||
|
<div class="fleft"><div class="fposter"><img src="https://image.tmdb.org/t/p/w400/bb.jpg" alt="Breaking Bad - Saison 2"></div></div>
|
||||||
|
<div id="serie-config" style="display:none" data-title="Breaking Bad - Saison 2" data-news-id="9562"></div>
|
||||||
|
"""
|
||||||
|
|
||||||
|
FILM_DETAILS_HTML = """
|
||||||
|
<div id="film-data" data-newsid="1022" data-affiche="https://image.tmdb.org/t/p/w300/inc.jpg"></div>
|
||||||
|
<div class="fmain"><div class="fcols fx-row"><div class="fmid">
|
||||||
|
<h1 id="s-title"> Inception - 2010 <span class="tag release_date"></span> </h1>
|
||||||
|
<div class="facts">
|
||||||
|
<span class="genres"><a href="/index.php?do=xfsearch&xf=Action">Action</a><a href="/index.php?do=xfsearch&xf=Science-Fiction">Science-Fiction</a></span>
|
||||||
|
<span class="runtime">- 2h28</span>
|
||||||
|
</div>
|
||||||
|
<div class="fdesc clearfix slice-this" id="s-desc">
|
||||||
|
<p class="desc-text">Résumé du film Inception en streaming complet vf et vostfr hd vod gratuit</p>
|
||||||
|
Dom Cobb est un voleur expérimenté.
|
||||||
|
</div>
|
||||||
|
</div></div></div>
|
||||||
|
"""
|
||||||
|
|
||||||
|
EP_DATA = {
|
||||||
|
"vf": {
|
||||||
|
"1": {"vidzy": "https://vidzy.cc/embed-aaa.html", "uqload": "https://uqload.vc/embed-bbb.html"},
|
||||||
|
"2": {"vidzy": "https://vidzy.cc/embed-ccc.html"},
|
||||||
|
},
|
||||||
|
"vostfr": {
|
||||||
|
"1": {"vidzy": "https://vidzy.cc/embed-ddd.html"},
|
||||||
|
"2": {"vidzy": "https://vidzy.cc/embed-eee.html"},
|
||||||
|
},
|
||||||
|
"vo": {},
|
||||||
|
"info": {
|
||||||
|
"1": {"title": "Traqués", "synopsis": "Walt et Jesse...", "poster": "https://img/ep1.jpg"},
|
||||||
|
"2": {"title": "Chasse à l'homme"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
FILM_API = {
|
||||||
|
"players": {
|
||||||
|
"vidzy": {
|
||||||
|
"default": "https://vidzy.live/embed-fff.html",
|
||||||
|
"vff": "https://vidzy.live/embed-fff.html",
|
||||||
|
"vostfr": "https://vidzy.live/embed-ggg.html",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
LATEST_HTML = """
|
||||||
|
<div class="short"><div class="short-in nl">
|
||||||
|
<a class="short-poster img-box with-mask" href="/index.php?newsid=15137274" alt="The Drop - Saison 1">
|
||||||
|
<img src="https://image.tmdb.org/t/p/w300/drop.jpg">
|
||||||
|
</a>
|
||||||
|
</div></div>
|
||||||
|
<div class="short"><div class="short-in nl">
|
||||||
|
<a class="short-poster img-box with-mask" href="/index.php?newsid=15139056" alt="La Rumeur">
|
||||||
|
<img src="https://image.tmdb.org/t/p/w300/rumeur.jpg">
|
||||||
|
</a>
|
||||||
|
</div></div>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _soup(html: str):
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
return BeautifulSoup(html, "lxml")
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ source
|
||||||
|
|
||||||
|
|
||||||
|
async def test_search(monkeypatch):
|
||||||
|
async def fake_fetch(url, **kwargs):
|
||||||
|
assert url.endswith("/engine/ajax/search.php")
|
||||||
|
assert kwargs["data"] == {"query": "breaking bad", "page": "1"}
|
||||||
|
return SEARCH_HTML
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch", fake_fetch)
|
||||||
|
results = await FrenchStreamScraper().search("breaking bad")
|
||||||
|
assert len(results) == 2
|
||||||
|
serie, film = results
|
||||||
|
assert serie.source_id == "9562"
|
||||||
|
assert serie.media_type == "serie"
|
||||||
|
assert serie.title == "Breaking Bad - Saison 2"
|
||||||
|
assert serie.url == "https://french-stream.lat/9562-breaking-bad-saison-2-streaming-complet-vf-vostfr.html"
|
||||||
|
assert serie.image_url == "https://image.tmdb.org/t/p/w400/bb.jpg"
|
||||||
|
assert film.media_type == "film"
|
||||||
|
assert film.source_id == "1022"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_latest(monkeypatch):
|
||||||
|
async def fake_fetch_soup(url, **kwargs):
|
||||||
|
assert url.endswith("/series/")
|
||||||
|
return _soup(LATEST_HTML)
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch_soup", fake_fetch_soup)
|
||||||
|
results = await FrenchStreamScraper().latest()
|
||||||
|
assert [(r.source_id, r.media_type) for r in results] == [
|
||||||
|
("15137274", "serie"),
|
||||||
|
("15139056", "film"),
|
||||||
|
]
|
||||||
|
assert results[0].title == "The Drop - Saison 1"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_details_serie(monkeypatch):
|
||||||
|
async def fake_fetch_soup(url, **kwargs):
|
||||||
|
assert "newsid=9562" in url
|
||||||
|
return _soup(SERIE_DETAILS_HTML)
|
||||||
|
|
||||||
|
async def fake_fetch(url, **kwargs):
|
||||||
|
assert "ep-data.php?id=9562" in url
|
||||||
|
return json.dumps(EP_DATA)
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch_soup", fake_fetch_soup)
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch", fake_fetch)
|
||||||
|
details = await FrenchStreamScraper().get_details("9562")
|
||||||
|
assert details.media_type == "serie"
|
||||||
|
assert details.title == "Breaking Bad - Saison 2"
|
||||||
|
assert details.year == 2008
|
||||||
|
assert details.genres == ["Drame", "Crime"]
|
||||||
|
assert details.synopsis == "Walter et Jesse montent leur propre affaire."
|
||||||
|
assert details.image_url == "https://image.tmdb.org/t/p/w400/bb.jpg"
|
||||||
|
assert details.episode_count == 4
|
||||||
|
vf1 = details.episodes[0]
|
||||||
|
assert (vf1.season, vf1.number, vf1.version, vf1.title) == (2, 1.0, "vf", "Traqués")
|
||||||
|
assert vf1.url.endswith("#vf-1")
|
||||||
|
versions = {(e.number, e.version) for e in details.episodes}
|
||||||
|
assert versions == {(1.0, "vf"), (1.0, "vostfr"), (2.0, "vf"), (2.0, "vostfr")}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_details_film(monkeypatch):
|
||||||
|
async def fake_fetch_soup(url, **kwargs):
|
||||||
|
return _soup(FILM_DETAILS_HTML)
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch_soup", fake_fetch_soup)
|
||||||
|
details = await FrenchStreamScraper().get_details("1022")
|
||||||
|
assert details.media_type == "film"
|
||||||
|
assert details.title == "Inception - 2010"
|
||||||
|
assert details.year == 2010
|
||||||
|
assert details.genres == ["Action", "Science-Fiction"]
|
||||||
|
assert "Résumé du film" not in details.synopsis
|
||||||
|
assert details.episode_count == 1
|
||||||
|
assert details.episodes[0].url.endswith("#film")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_episodes_film(monkeypatch):
|
||||||
|
async def fake_fetch_soup(url, **kwargs):
|
||||||
|
return _soup(FILM_DETAILS_HTML)
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch_soup", fake_fetch_soup)
|
||||||
|
episodes = await FrenchStreamScraper().list_episodes("1022")
|
||||||
|
assert len(episodes) == 1
|
||||||
|
assert episodes[0].number == 1
|
||||||
|
|
||||||
|
|
||||||
|
async def test_extract_embed_links_episode(monkeypatch):
|
||||||
|
async def fake_fetch(url, **kwargs):
|
||||||
|
return json.dumps(EP_DATA)
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch", fake_fetch)
|
||||||
|
links = await FrenchStreamScraper().extract_embed_links(
|
||||||
|
"https://french-stream.lat/index.php?newsid=9562#vf-1"
|
||||||
|
)
|
||||||
|
assert links == ["https://vidzy.cc/embed-aaa.html", "https://uqload.vc/embed-bbb.html"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_extract_embed_links_film_dedup(monkeypatch):
|
||||||
|
async def fake_fetch(url, **kwargs):
|
||||||
|
assert "film_api.php?id=1022" in url
|
||||||
|
return json.dumps(FILM_API)
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch", fake_fetch)
|
||||||
|
links = await FrenchStreamScraper().extract_embed_links(
|
||||||
|
"https://french-stream.lat/index.php?newsid=1022#film"
|
||||||
|
)
|
||||||
|
assert links == ["https://vidzy.live/embed-fff.html", "https://vidzy.live/embed-ggg.html"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_extract_embed_links_bad_fragment():
|
||||||
|
with pytest.raises(ScrapeError):
|
||||||
|
await FrenchStreamScraper().extract_embed_links(
|
||||||
|
"https://french-stream.lat/index.php?newsid=9562#nimportequoi"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_extract_embed_links_invalid_json(monkeypatch):
|
||||||
|
async def fake_fetch(url, **kwargs):
|
||||||
|
return "<html>oops</html>"
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch", fake_fetch)
|
||||||
|
with pytest.raises(ScrapeError):
|
||||||
|
await FrenchStreamScraper().extract_embed_links(
|
||||||
|
"https://french-stream.lat/index.php?newsid=9562#vf-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_unknown_detail_page(monkeypatch):
|
||||||
|
async def fake_fetch_soup(url, **kwargs):
|
||||||
|
return _soup("<html><body>404</body></html>")
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.sources.french_stream.fetch_soup", fake_fetch_soup)
|
||||||
|
with pytest.raises(ScrapeError):
|
||||||
|
await FrenchStreamScraper().get_details("999999999")
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered():
|
||||||
|
import_all_scrapers()
|
||||||
|
source = get_source("french_stream")
|
||||||
|
assert source.media_types == ("serie", "film")
|
||||||
Reference in New Issue
Block a user