Roman
41566ab5fb
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
64 lines
1.8 KiB
Python
64 lines
1.8 KiB
Python
"""Client HTTP partagé pour le scraping (httpx async, headers navigateur, retries)."""
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from app.config import get_settings
|
|
from app.scrapers.base import ScrapeError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_client: httpx.AsyncClient | None = None
|
|
|
|
|
|
def get_client() -> httpx.AsyncClient:
|
|
global _client
|
|
if _client is None:
|
|
settings = get_settings()
|
|
_client = httpx.AsyncClient(
|
|
timeout=settings.http_timeout,
|
|
follow_redirects=True,
|
|
headers={
|
|
"User-Agent": settings.user_agent,
|
|
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
|
},
|
|
)
|
|
return _client
|
|
|
|
|
|
async def close_client() -> None:
|
|
global _client
|
|
if _client is not None:
|
|
await _client.aclose()
|
|
_client = None
|
|
|
|
|
|
async def fetch(
|
|
url: str,
|
|
*,
|
|
referer: str | None = None,
|
|
retries: int = 2,
|
|
) -> str:
|
|
"""GET d'une page avec retries ; lève ScrapeError en cas d'échec définitif."""
|
|
headers = {"Referer": referer} if referer else {}
|
|
last_error: Exception | None = None
|
|
for attempt in range(retries + 1):
|
|
try:
|
|
response = await get_client().get(url, headers=headers)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
|
last_error = exc
|
|
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
|
|
if attempt < retries:
|
|
await asyncio.sleep(1.0 * (attempt + 1))
|
|
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
|
|
|
|
|
async def fetch_soup(url: str, *, referer: str | None = None) -> BeautifulSoup:
|
|
html = await fetch(url, referer=referer)
|
|
return BeautifulSoup(html, "lxml")
|