Roman
79a44297fd
- Scraper DataLife Engine : recherche AJAX, fiches par saison, épisodes via ep-data.php (VF/VOSTFR), films via film_api.php, nouveautés /series/ - Config YAML externalisée (sélecteurs réparables sans coder) - Domaines étendus : uqload.vc, vidzy.cc/.live - 11 tests (126 au total)
79 lines
2.6 KiB
Python
79 lines
2.6 KiB
Python
"""Client HTTP partagé pour le scraping (httpx async, headers navigateur, retries)."""
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from app.config import get_settings
|
|
from app.scrapers.base import ScrapeError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_client: httpx.AsyncClient | None = None
|
|
|
|
|
|
def get_client() -> httpx.AsyncClient:
|
|
global _client
|
|
if _client is None:
|
|
settings = get_settings()
|
|
_client = httpx.AsyncClient(
|
|
timeout=settings.http_timeout,
|
|
follow_redirects=True,
|
|
headers={
|
|
"User-Agent": settings.user_agent,
|
|
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
|
},
|
|
)
|
|
return _client
|
|
|
|
|
|
async def close_client() -> None:
|
|
global _client
|
|
if _client is not None:
|
|
await _client.aclose()
|
|
_client = None
|
|
|
|
|
|
async def fetch(
|
|
url: str,
|
|
*,
|
|
referer: str | None = None,
|
|
retries: int = 2,
|
|
headers: dict[str, str] | None = None,
|
|
data: dict[str, str] | None = None,
|
|
) -> str:
|
|
"""GET (ou POST si `data` est fourni) avec retries ; lève ScrapeError en cas d'échec définitif."""
|
|
request_headers = dict(headers) if headers else {}
|
|
if referer:
|
|
request_headers.setdefault("Referer", referer)
|
|
last_error: Exception | None = None
|
|
for attempt in range(retries + 1):
|
|
try:
|
|
if data is None:
|
|
response = await get_client().get(url, headers=request_headers)
|
|
else:
|
|
response = await get_client().post(url, headers=request_headers, data=data)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except httpx.HTTPStatusError as exc:
|
|
status = exc.response.status_code
|
|
if 400 <= status < 500 and status not in (408, 429):
|
|
raise ScrapeError(f"Échec de récupération de {url} : HTTP {status} (définitif)") from exc
|
|
last_error = exc
|
|
logger.warning("fetch %s — HTTP %d, tentative %d/%d", url, status, attempt + 1, retries + 1)
|
|
if attempt < retries:
|
|
await asyncio.sleep(1.0 * (attempt + 1))
|
|
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
|
last_error = exc
|
|
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
|
|
if attempt < retries:
|
|
await asyncio.sleep(1.0 * (attempt + 1))
|
|
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
|
|
|
|
|
async def fetch_soup(url: str, *, referer: str | None = None) -> BeautifulSoup:
|
|
html = await fetch(url, referer=referer)
|
|
return BeautifulSoup(html, "lxml")
|