v0.1.0 — Réécriture complète de OhmStreaming
Nouvelle version réécrite de zéro : recherche multi-sources (Vostfree, French-Manga), extraction 2 niveaux, proxy vidéo intégré, streaming/téléchargement HLS, métadonnées Kitsu, bibliothèque locale, comptes JWT + administration, découverte fusionnée, indexeur Torznab (Sonarr/Prowlarr).
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
"""Client HTTP partagé pour le scraping (httpx async, headers navigateur, retries)."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.config import get_settings
|
||||
from app.scrapers.base import ScrapeError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_client: httpx.AsyncClient | None = None
|
||||
|
||||
|
||||
def get_client() -> httpx.AsyncClient:
|
||||
global _client
|
||||
if _client is None:
|
||||
settings = get_settings()
|
||||
_client = httpx.AsyncClient(
|
||||
timeout=settings.http_timeout,
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"User-Agent": settings.user_agent,
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
||||
},
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
async def close_client() -> None:
|
||||
global _client
|
||||
if _client is not None:
|
||||
await _client.aclose()
|
||||
_client = None
|
||||
|
||||
|
||||
async def fetch(
|
||||
url: str,
|
||||
*,
|
||||
referer: str | None = None,
|
||||
retries: int = 2,
|
||||
) -> str:
|
||||
"""GET d'une page avec retries ; lève ScrapeError en cas d'échec définitif."""
|
||||
headers = {"Referer": referer} if referer else {}
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(retries + 1):
|
||||
try:
|
||||
response = await get_client().get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
||||
last_error = exc
|
||||
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
|
||||
if attempt < retries:
|
||||
await asyncio.sleep(1.0 * (attempt + 1))
|
||||
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
||||
|
||||
|
||||
async def fetch_soup(url: str, *, referer: str | None = None) -> BeautifulSoup:
|
||||
html = await fetch(url, referer=referer)
|
||||
return BeautifulSoup(html, "lxml")
|
||||
Reference in New Issue
Block a user