Corrections : test Torznab périmé et retries sur erreurs définitives
- test_torznab_search_requires_query attendait un 400 mais le commit
853b4e0 a volontairement fait servir les nouveautés quand q est
absent (RSS sync Sonarr) — le test valide désormais ce comportement.
- fetch() ne retente plus les 4xx définitifs (sauf 408/429) : un
lecteur VoirAnime en échec 422 coûtait 2 requêtes et 3 s de pauses
avant de basculer sur le lecteur suivant.
- 4 nouveaux tests unitaires du client HTTP (fail-fast + retries).
This commit is contained in:
@@ -53,6 +53,14 @@ async def fetch(
|
|||||||
response = await get_client().get(url, headers=request_headers)
|
response = await get_client().get(url, headers=request_headers)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return response.text
|
return response.text
|
||||||
|
except httpx.HTTPStatusError as exc:
|
||||||
|
status = exc.response.status_code
|
||||||
|
if 400 <= status < 500 and status not in (408, 429):
|
||||||
|
raise ScrapeError(f"Échec de récupération de {url} : HTTP {status} (définitif)") from exc
|
||||||
|
last_error = exc
|
||||||
|
logger.warning("fetch %s — HTTP %d, tentative %d/%d", url, status, attempt + 1, retries + 1)
|
||||||
|
if attempt < retries:
|
||||||
|
await asyncio.sleep(1.0 * (attempt + 1))
|
||||||
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
||||||
last_error = exc
|
last_error = exc
|
||||||
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
|
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
"""Tests unitaires : format interne, sanitisation, auth, registres scrapers."""
|
"""Tests unitaires : format interne, sanitisation, auth, registres scrapers."""
|
||||||
|
|
||||||
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app import auth
|
from app import auth
|
||||||
|
from app.scrapers import http as http_module
|
||||||
from app.scrapers.base import (
|
from app.scrapers.base import (
|
||||||
|
ScrapeError,
|
||||||
decode_internal_url,
|
decode_internal_url,
|
||||||
encode_internal_url,
|
encode_internal_url,
|
||||||
get_source,
|
get_source,
|
||||||
@@ -133,3 +136,43 @@ def test_hoster_resolution():
|
|||||||
assert resolve_hoster("https://vidmoly.to/embed-x.html").name == "vidmoly"
|
assert resolve_hoster("https://vidmoly.to/embed-x.html").name == "vidmoly"
|
||||||
assert resolve_hoster("https://sendvid.com/embed/x").name == "sendvid"
|
assert resolve_hoster("https://sendvid.com/embed/x").name == "sendvid"
|
||||||
assert resolve_hoster("https://inconnu.example.com/v/1") is None
|
assert resolve_hoster("https://inconnu.example.com/v/1") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ client HTTP
|
||||||
|
|
||||||
|
|
||||||
|
class _StatusClient:
|
||||||
|
"""Fake client HTTP : sert une série de codes statut puis un corps 200."""
|
||||||
|
|
||||||
|
def __init__(self, statuses: list[int]):
|
||||||
|
self._statuses = statuses
|
||||||
|
self.calls = 0
|
||||||
|
|
||||||
|
async def get(self, url, headers=None):
|
||||||
|
self.calls += 1
|
||||||
|
request = httpx.Request("GET", url)
|
||||||
|
status = self._statuses.pop(0) if self._statuses else 200
|
||||||
|
response = httpx.Response(status, request=request, text="ok")
|
||||||
|
response.raise_for_status()
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
async def test_fetch_fails_fast_on_definitive_4xx(monkeypatch):
|
||||||
|
client = _StatusClient([422])
|
||||||
|
monkeypatch.setattr("app.scrapers.http.get_client", lambda: client)
|
||||||
|
with pytest.raises(ScrapeError, match="422"):
|
||||||
|
await http_module.fetch("https://site.example/lecteur/prepare/1?content=2&episode=3")
|
||||||
|
assert client.calls == 1 # aucun retry sur erreur définitive
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("status", [429, 500, 503])
|
||||||
|
async def test_fetch_retries_transitory_errors(monkeypatch, status):
|
||||||
|
client = _StatusClient([status, 200])
|
||||||
|
monkeypatch.setattr("app.scrapers.http.get_client", lambda: client)
|
||||||
|
|
||||||
|
async def no_sleep(_delay):
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scrapers.http.asyncio.sleep", no_sleep)
|
||||||
|
assert await http_module.fetch("https://site.example/page", retries=2) == "ok"
|
||||||
|
assert client.calls == 2
|
||||||
|
|||||||
+19
-2
@@ -128,9 +128,26 @@ async def test_torznab_tvsearch_all_episodes(client, apikey, fake_source):
|
|||||||
assert "Frieren S02E01 VOSTFR WEB-DL" in titles
|
assert "Frieren S02E01 VOSTFR WEB-DL" in titles
|
||||||
|
|
||||||
|
|
||||||
async def test_torznab_search_requires_query(client, apikey):
|
async def test_torznab_search_without_query_serves_latest(client, apikey, fake_source):
|
||||||
|
# Sans q : flux RSS des nouveautés (RSS sync / test Sonarr), pas une erreur 400.
|
||||||
|
async def fake_latest():
|
||||||
|
return [
|
||||||
|
SearchResult(
|
||||||
|
source="fake",
|
||||||
|
source_id="frieren-1",
|
||||||
|
title="Frieren",
|
||||||
|
url="https://fake.example/frieren",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
fake_source.latest = fake_latest
|
||||||
r = await client.get("/torznab/api", params={"t": "search", "apikey": apikey})
|
r = await client.get("/torznab/api", params={"t": "search", "apikey": apikey})
|
||||||
assert r.status_code == 400
|
assert r.status_code == 200
|
||||||
|
root = ET.fromstring(r.text)
|
||||||
|
items = root.findall(".//item")
|
||||||
|
assert len(items) == 1
|
||||||
|
# seul le dernier épisode de la nouveauté est publié (S02E01 > S01E02 > S01E01)
|
||||||
|
assert items[0].find("title").text == "Frieren S02E01 VOSTFR WEB-DL"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- grab
|
# ---------------------------------------------------------------- grab
|
||||||
|
|||||||
Reference in New Issue
Block a user