Roman
7399f4b781
- Scraper catalogue : recherche /catalogue?q=, fiches + épisodes (cartes S·E, multi-saisons), chaîne de lecteurs prepare/fallback avec dédoublonnage par id de source, latest via carrousels accueil. - Extracteur hoster : endpoint prepare (AJAX) → MP4 direct décodé du payload (Referer d'origine) ou HLS relayé par le proxy du site (CDN signés pour leur backend → accès direct en 403). - fetch() accepte désormais des en-têtes additionnels (X-Requested-With). - 9 tests sur fixtures HTML réelles ; README mis à jour.
67 lines
1.9 KiB
Python
67 lines
1.9 KiB
Python
"""Client HTTP partagé pour le scraping (httpx async, headers navigateur, retries)."""
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from app.config import get_settings
|
|
from app.scrapers.base import ScrapeError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_client: httpx.AsyncClient | None = None
|
|
|
|
|
|
def get_client() -> httpx.AsyncClient:
|
|
global _client
|
|
if _client is None:
|
|
settings = get_settings()
|
|
_client = httpx.AsyncClient(
|
|
timeout=settings.http_timeout,
|
|
follow_redirects=True,
|
|
headers={
|
|
"User-Agent": settings.user_agent,
|
|
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
|
},
|
|
)
|
|
return _client
|
|
|
|
|
|
async def close_client() -> None:
|
|
global _client
|
|
if _client is not None:
|
|
await _client.aclose()
|
|
_client = None
|
|
|
|
|
|
async def fetch(
|
|
url: str,
|
|
*,
|
|
referer: str | None = None,
|
|
retries: int = 2,
|
|
headers: dict[str, str] | None = None,
|
|
) -> str:
|
|
"""GET d'une page avec retries ; lève ScrapeError en cas d'échec définitif."""
|
|
request_headers = dict(headers) if headers else {}
|
|
if referer:
|
|
request_headers.setdefault("Referer", referer)
|
|
last_error: Exception | None = None
|
|
for attempt in range(retries + 1):
|
|
try:
|
|
response = await get_client().get(url, headers=request_headers)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
|
last_error = exc
|
|
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
|
|
if attempt < retries:
|
|
await asyncio.sleep(1.0 * (attempt + 1))
|
|
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
|
|
|
|
|
async def fetch_soup(url: str, *, referer: str | None = None) -> BeautifulSoup:
|
|
html = await fetch(url, referer=referer)
|
|
return BeautifulSoup(html, "lxml")
|