Compare commits
44
Commits
v0.2.0
..
3cf2f8eca5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3cf2f8eca5 | ||
|
|
8b7a419b4c | ||
|
|
2e0af00278 | ||
|
|
414a89b7a5 | ||
|
|
90dc884ef9 | ||
|
|
fcf099ebb4 | ||
|
|
5fa55fe1a2 | ||
|
|
2482a1fe58 | ||
|
|
da5403a307 | ||
|
|
c6be191699 | ||
|
|
6fcfb3f812 | ||
|
|
7dabce1c3c | ||
|
|
764b4e2edd | ||
|
|
d82bec92b4 | ||
|
|
ef72e221be | ||
|
|
c1c31d7685 | ||
|
|
5e50081b58 | ||
|
|
4d280b5239 | ||
|
|
3afad41d46 | ||
|
|
1fe7392063 | ||
|
|
92ef76ed2a | ||
|
|
63af6fd4d9 | ||
|
|
bfd5269ff7 | ||
|
|
944d13a4c9 | ||
|
|
b27c331d1c | ||
|
|
adb43ee371 | ||
|
|
3d7a17d0d7 | ||
|
|
30f79789ee | ||
|
|
8f9f544d47 | ||
|
|
55bb85b56f | ||
|
|
a32ea205a4 | ||
|
|
f527a335de | ||
|
|
01792e8a58 | ||
|
|
81f1b7708c | ||
|
|
f13ad6abbd | ||
|
|
eb870d89c2 | ||
|
|
785147b1b1 | ||
|
|
5805f1036f | ||
|
|
d2e1bd8ab0 | ||
|
|
6168e9ed60 | ||
|
|
20cad0b4fe | ||
|
|
40977438ff | ||
|
|
c977306020 | ||
|
|
cb3ea8d926 |
@@ -1,20 +0,0 @@
|
|||||||
# Contexte de build minimal : ni secrets, ni données, ni caches
|
|
||||||
.git
|
|
||||||
.gitignore
|
|
||||||
.env
|
|
||||||
.env.example
|
|
||||||
.venv
|
|
||||||
.plasma
|
|
||||||
.pytest_cache
|
|
||||||
.ruff_cache
|
|
||||||
data/
|
|
||||||
downloads/
|
|
||||||
tests/
|
|
||||||
__pycache__/
|
|
||||||
*.pyc
|
|
||||||
Dockerfile
|
|
||||||
docker-compose.yml
|
|
||||||
docker-compose.yml.example
|
|
||||||
README.md
|
|
||||||
Projet_descriptions.md
|
|
||||||
scripts/
|
|
||||||
+24
-27
@@ -1,34 +1,31 @@
|
|||||||
# Copier en .env et adapter. Toutes les variables applicatives sont préfixées OHM_.
|
# Ohm Stream Downloader Environment Configuration
|
||||||
|
|
||||||
# OBLIGATOIRE en production : clé de signature des tokens (32+ caractères)
|
# Application
|
||||||
OHM_SECRET_KEY=change-me-in-production
|
APP_NAME=Ohm Stream Downloader
|
||||||
|
APP_VERSION=2.2
|
||||||
|
DEBUG=false
|
||||||
|
|
||||||
# Chemins
|
# Server Configuration
|
||||||
# OHM_DATA_DIR=./data
|
HOST=0.0.0.0
|
||||||
# OHM_DOWNLOAD_DIR=./downloads
|
PORT=3000
|
||||||
# OHM_DATABASE_PATH=./data/ohm.db
|
RELOAD=true
|
||||||
|
|
||||||
# Téléchargements
|
# Download Settings
|
||||||
# OHM_MAX_PARALLEL_DOWNLOADS=3
|
DOWNLOAD_DIR=downloads
|
||||||
|
MAX_PARALLEL_DOWNLOADS=3
|
||||||
|
CHUNK_SIZE=1048576
|
||||||
|
|
||||||
# Scraping
|
# CORS Origins (comma-separated)
|
||||||
# OHM_HTTP_TIMEOUT=20
|
CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000,http://192.168.1.204:3000
|
||||||
# OHM_USER_AGENT="Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
|
|
||||||
|
|
||||||
# Métadonnées Kitsu
|
# Storage Paths
|
||||||
# OHM_METADATA_CACHE_TTL_HOURS=72
|
FAVORITES_STORAGE_PATH=favorites.json
|
||||||
|
SONARR_CONFIG_PATH=config/sonarr.json
|
||||||
|
SONARR_MAPPINGS_PATH=config/sonarr_mappings.json
|
||||||
|
|
||||||
# Auth
|
# API Timeouts
|
||||||
# OHM_ACCESS_TOKEN_TTL_MINUTES=15
|
HTTP_TIMEOUT=10.0
|
||||||
# OHM_REFRESH_TOKEN_TTL_DAYS=30
|
DOWNLOAD_TIMEOUT=300
|
||||||
|
|
||||||
# OHM_DEBUG=false
|
# Logging
|
||||||
|
LOG_LEVEL=INFO
|
||||||
# ── Déploiement Docker (docker-compose.yml) ────────────────────────────────
|
|
||||||
# Secret partagé entre OhmStreaming et Watchtower pour déclencher les mises
|
|
||||||
# à jour depuis la page Admin. OBLIGATOIRE en Docker.
|
|
||||||
# Générer : openssl rand -hex 24
|
|
||||||
WATCHTOWER_TOKEN=change-me-watchtower
|
|
||||||
|
|
||||||
# Port hôte exposé par docker compose (défaut 8777)
|
|
||||||
# OHM_PORT=8777
|
|
||||||
|
|||||||
+52
-5
@@ -1,9 +1,56 @@
|
|||||||
.venv/
|
# Python
|
||||||
__pycache__/
|
__pycache__/
|
||||||
*.pyc
|
*.py[cod]
|
||||||
data/
|
*$py.class
|
||||||
|
*.so
|
||||||
|
.Python
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
.venv
|
||||||
|
|
||||||
|
# FastAPI
|
||||||
|
uploads/
|
||||||
|
streams/
|
||||||
downloads/
|
downloads/
|
||||||
|
|
||||||
|
# Environment
|
||||||
.env
|
.env
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# Testing
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
.ruff_cache/
|
.coverage
|
||||||
.plasma/
|
.coverage.*
|
||||||
|
htmlcov/
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
.hypothesis/
|
||||||
|
|
||||||
|
# Project data
|
||||||
|
data/
|
||||||
|
favorites.json
|
||||||
|
*.db
|
||||||
|
*.sqlite
|
||||||
|
ohm_streaming.db
|
||||||
|
|
||||||
|
# Config (runtime-generated)
|
||||||
|
config/anime_sama_domain.json
|
||||||
|
config/metadata_cache.json
|
||||||
|
data/
|
||||||
|
favorites.json
|
||||||
|
*.db
|
||||||
|
*.sqlite
|
||||||
|
ohm_streaming.db
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
# AGENTS.md - Agentic Coding Guidelines
|
||||||
|
|
||||||
|
This file provides guidance for AI agents working in this repository.
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Setup
|
||||||
|
python3 -m venv venv && source venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
# Run dev server
|
||||||
|
uvicorn main:app --reload --host 0.0.0.0 --port 3000
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build, Lint & Test Commands
|
||||||
|
|
||||||
|
### Running Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# All tests
|
||||||
|
pytest
|
||||||
|
|
||||||
|
# With coverage
|
||||||
|
pytest --cov=app --cov-report=html
|
||||||
|
|
||||||
|
# Unit only (fast)
|
||||||
|
pytest -m "unit"
|
||||||
|
|
||||||
|
# Exclude slow tests
|
||||||
|
pytest -m "not slow"
|
||||||
|
|
||||||
|
# Verbose with print debugging
|
||||||
|
pytest -v -s
|
||||||
|
```
|
||||||
|
|
||||||
|
### Running Single Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Specific file
|
||||||
|
pytest tests/test_sonarr.py -v
|
||||||
|
|
||||||
|
# Specific class
|
||||||
|
pytest tests/test_sonarr.py::TestSonarrHandler -v
|
||||||
|
|
||||||
|
# Specific test
|
||||||
|
pytest tests/test_sonarr.py::TestSonarrHandler::test_add_mapping -v
|
||||||
|
|
||||||
|
# Pattern match
|
||||||
|
pytest -k "test_download" -v
|
||||||
|
```
|
||||||
|
|
||||||
|
## Code Style
|
||||||
|
|
||||||
|
### Imports (PEP 8 order)
|
||||||
|
1. Standard library (`os`, `json`, `asyncio`)
|
||||||
|
2. Third-party (`httpx`, `beautifulsoup4`, `fastapi`)
|
||||||
|
3. Local app (`app.config`, `app.utils`)
|
||||||
|
|
||||||
|
```python
|
||||||
|
import os
|
||||||
|
import asyncio
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.models import DownloadTask, DownloadStatus
|
||||||
|
```
|
||||||
|
|
||||||
|
### Formatting
|
||||||
|
- **Line length**: 120 chars max
|
||||||
|
- **Indentation**: 4 spaces
|
||||||
|
- **Blank lines**: 2 between top-level, 1 between inline
|
||||||
|
|
||||||
|
### Type Annotations
|
||||||
|
- Use explicit types
|
||||||
|
- Use `Optional[X]` not `X | None`
|
||||||
|
- Use `list[X]`, `dict[X, Y]`
|
||||||
|
|
||||||
|
```python
|
||||||
|
# Good
|
||||||
|
async def get_download_link(url: str, target_filename: Optional[str] = None) -> tuple[str, str]:
|
||||||
|
results: list[dict[str, str]] = []
|
||||||
|
|
||||||
|
# Avoid
|
||||||
|
async def get_download_link(url, target_filename=None):
|
||||||
|
results = []
|
||||||
|
```
|
||||||
|
|
||||||
|
### Naming Conventions
|
||||||
|
|
||||||
|
| Element | Convention | Example |
|
||||||
|
|---------|------------|---------|
|
||||||
|
| Modules | snake_case | `download_manager.py` |
|
||||||
|
| Classes | PascalCase | `DownloadManager` |
|
||||||
|
| Functions | snake_case | `get_download_link()` |
|
||||||
|
| Constants | UPPER_SNAKE | `MAX_PARALLEL_DOWNLOADS` |
|
||||||
|
| Variables | snake_case | `download_task` |
|
||||||
|
| Enums | PascalCase | `DownloadStatus` |
|
||||||
|
| Enum values | UPPER_SNAKE | `DownloadStatus.PENDING` |
|
||||||
|
|
||||||
|
### Async/Await
|
||||||
|
- Always use for I/O operations
|
||||||
|
- Close clients properly to avoid leaks
|
||||||
|
|
||||||
|
```python
|
||||||
|
async def close(self):
|
||||||
|
await self.client.aclose()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Error Handling
|
||||||
|
- Use try/except for recoverable errors
|
||||||
|
- Raise specific exceptions (`HTTPException`, `ValueError`)
|
||||||
|
- Never use empty except blocks
|
||||||
|
- Log errors appropriately
|
||||||
|
|
||||||
|
```python
|
||||||
|
try:
|
||||||
|
result = await client.get(url)
|
||||||
|
except httpx.TimeoutException:
|
||||||
|
logger.warning(f"Request timeout for {url}")
|
||||||
|
raise HTTPException(status_code=504, detail="Request timeout")
|
||||||
|
```
|
||||||
|
|
||||||
|
### File Operations
|
||||||
|
- Always sanitize filenames: `app.utils.sanitize_filename()`
|
||||||
|
- Validate paths: `app.utils.is_safe_filename()`
|
||||||
|
|
||||||
|
### Testing
|
||||||
|
- Use pytest with pytest-asyncio
|
||||||
|
- Mark tests: `@pytest.mark.unit`, `@pytest.mark.integration`
|
||||||
|
- Use fixtures from `tests/conftest.py`
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.unit
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_download_manager():
|
||||||
|
manager = DownloadManager(max_parallel=3)
|
||||||
|
assert manager.max_parallel == 3
|
||||||
|
```
|
||||||
|
|
||||||
|
### Security
|
||||||
|
- Never hardcode secrets - use environment variables
|
||||||
|
- Validate all inputs (URLs, filenames)
|
||||||
|
- Use HMAC for webhook verification when configured
|
||||||
|
- Limit CORS origins - never use `*` in production
|
||||||
|
|
||||||
|
## Architecture Patterns
|
||||||
|
|
||||||
|
**Three-Tier Downloader:**
|
||||||
|
1. `app/downloaders/anime_sites/` - Anime catalogs
|
||||||
|
2. `app/downloaders/series_sites/` - TV series catalogs
|
||||||
|
3. `app/downloaders/video_players/` - File hosting
|
||||||
|
|
||||||
|
Each has base class and factory. When adding providers:
|
||||||
|
1. Inherit from appropriate base class
|
||||||
|
2. Implement required methods
|
||||||
|
3. Register in factory
|
||||||
|
4. Add to providers config in `app/providers.py`
|
||||||
|
|
||||||
|
**URL Convention**: Pipe-separated format preserves metadata:
|
||||||
|
```
|
||||||
|
video_url|anime_page_url|episode_title
|
||||||
|
```
|
||||||
|
|
||||||
|
## Key Files
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `main.py` | FastAPI app, endpoints |
|
||||||
|
| `app/config.py` | Pydantic Settings |
|
||||||
|
| `app/download_manager.py` | Download queue |
|
||||||
|
| `app/utils.py` | sanitize_filename |
|
||||||
|
| `app/auth.py` | JWT auth |
|
||||||
|
| `app/models/__init__.py` | Pydantic models |
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
- Use `.env` from `.env.example`
|
||||||
|
- JWT_SECRET_KEY must change in production
|
||||||
@@ -0,0 +1,664 @@
|
|||||||
|
# CLAUDE.md
|
||||||
|
|
||||||
|
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||||
|
|
||||||
|
## Project Overview
|
||||||
|
|
||||||
|
Ohm Stream Downloader is a FastAPI-based web application for downloading anime episodes and media files from various file hosting services (1fichier, Doodstream, Rapidfile, Uptobox, VidMoly, SendVid, Sibnet, Lpayer, Vidzy, LuLuvid, Uqload) and streaming platforms (Anime-Sama, Neko-Sama, Anime-Ultime, Vostfree, French-Manga, FS7). It features a modern web interface, parallel downloads, pause/resume support, video streaming, personalized recommendations, JWT authentication, and Sonarr webhook integration for automated downloads.
|
||||||
|
|
||||||
|
## Development Commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Create and activate virtual environment
|
||||||
|
python3 -m venv venv
|
||||||
|
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
# Run development server (auto-reload)
|
||||||
|
uvicorn main:app --reload --host 0.0.0.0 --port 3000
|
||||||
|
|
||||||
|
# Access web interface
|
||||||
|
# Open http://localhost:3000/web in browser
|
||||||
|
|
||||||
|
# Run all tests
|
||||||
|
pytest
|
||||||
|
|
||||||
|
# Run tests with coverage report
|
||||||
|
pytest --cov=app --cov-report=html
|
||||||
|
|
||||||
|
# Run only unit tests (fast, isolated)
|
||||||
|
pytest -m "unit"
|
||||||
|
|
||||||
|
# Run only integration tests
|
||||||
|
pytest -m "integration"
|
||||||
|
|
||||||
|
# Exclude slow tests
|
||||||
|
pytest -m "not slow"
|
||||||
|
|
||||||
|
# Verbose output
|
||||||
|
pytest -v
|
||||||
|
|
||||||
|
# Show print debugging
|
||||||
|
pytest -s
|
||||||
|
```
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
**Directory Structure:**
|
||||||
|
```
|
||||||
|
Ohm_streaming/
|
||||||
|
├── main.py # FastAPI application & API endpoints
|
||||||
|
├── app/
|
||||||
|
│ ├── models/ # Pydantic models (DownloadTask, AnimeMetadata, Sonarr, etc.)
|
||||||
|
│ ├── downloaders/ # Host-specific downloaders (organized structure)
|
||||||
|
│ │ ├── base.py # BaseDownloader abstract class (legacy, kept for compatibility)
|
||||||
|
│ │ ├── __init__.py # Factory function (three-tier: anime sites → series sites → video players)
|
||||||
|
│ │ ├── anime_sites/ # Anime streaming sites (catalogs)
|
||||||
|
│ │ │ ├── base.py # BaseAnimeSite abstract class
|
||||||
|
│ │ │ ├── __init__.py # Anime site factory
|
||||||
|
│ │ │ ├── animesama.py # Anime-Sama (anime provider)
|
||||||
|
│ │ │ ├── animeultime.py # Anime-Ultime (anime provider)
|
||||||
|
│ │ │ ├── nekosama.py # Neko-Sama (anime provider)
|
||||||
|
│ │ │ ├── vostfree.py # Vostfree (anime provider)
|
||||||
|
│ │ │ └── frenchmanga.py # French-Manga (anime provider)
|
||||||
|
│ │ ├── series_sites/ # TV series streaming sites (catalogs)
|
||||||
|
│ │ │ ├── base.py # BaseSeriesSite abstract class
|
||||||
|
│ │ │ ├── __init__.py # Series site factory
|
||||||
|
│ │ │ └── fs7.py # FS7 (French Stream)
|
||||||
|
│ │ └── video_players/ # File hosting services (players)
|
||||||
|
│ │ ├── base.py # BaseVideoPlayer abstract class
|
||||||
|
│ │ ├── __init__.py # Video player factory
|
||||||
|
│ │ ├── unfichier.py # 1fichier.com handler
|
||||||
|
│ │ ├── doodstream.py # Doodstream handler
|
||||||
|
│ │ ├── rapidfile.py # Rapidfile handler
|
||||||
|
│ │ ├── uptobox.py # Uptobox handler
|
||||||
|
│ │ ├── vidmoly.py # VidMoly handler
|
||||||
|
│ │ ├── sendvid.py # SendVid handler
|
||||||
|
│ │ ├── sibnet.py # Sibnet handler
|
||||||
|
│ │ ├── lpayer.py # Lpayer handler
|
||||||
|
│ │ ├── vidzy.py # Vidzy handler
|
||||||
|
│ │ ├── luluv.py # LuLuvid handler
|
||||||
|
│ │ └── uqload.py # Uqload handler
|
||||||
|
│ ├── providers.py # Provider configuration (domains, icons, colors)
|
||||||
|
│ ├── config.py # Environment-based configuration (Pydantic Settings)
|
||||||
|
│ ├── utils.py # Security utilities (sanitize_filename, is_safe_filename)
|
||||||
|
│ ├── download_manager.py # Manages download queue, progress, parallel downloads
|
||||||
|
│ ├── favorites.py # Favorites management system (JSON-based)
|
||||||
|
│ ├── recommendation_engine.py # Analyzes download history for personalized recommendations
|
||||||
|
│ ├── recommendations.py # Fetches latest releases from anime sources
|
||||||
|
│ ├── kitsu_api.py # Kitsu API integration for anime metadata
|
||||||
|
│ ├── sonarr_handler.py # Sonarr webhook integration handler
|
||||||
|
│ ├── auth.py # JWT authentication system
|
||||||
|
│ └── models/
|
||||||
|
│ ├── __init__.py # Core models (DownloadTask, AnimeMetadata, etc.)
|
||||||
|
│ └── sonarr.py # Sonarr Pydantic models
|
||||||
|
├── downloads/ # Downloaded files storage
|
||||||
|
├── templates/
|
||||||
|
│ ├── index.html # Main web interface
|
||||||
|
│ ├── player.html # Video player page
|
||||||
|
│ └── base.html # Base template
|
||||||
|
├── static/ # Static assets (CSS, JS, images)
|
||||||
|
└── tests/ # Test suite with fixtures
|
||||||
|
```
|
||||||
|
|
||||||
|
**Core Components:**
|
||||||
|
|
||||||
|
### 0. Configuration (`app/config.py`)
|
||||||
|
- `Settings` class using Pydantic Settings for environment-based configuration
|
||||||
|
- Loads from `.env` file with sensible defaults
|
||||||
|
- Provides `get_settings()` function for accessing configuration globally
|
||||||
|
|
||||||
|
### 1. DownloadManager (`app/download_manager.py`)
|
||||||
|
- Manages all download tasks with parallel download limit (default: 3 concurrent)
|
||||||
|
- Handles pause/resume/cancel operations
|
||||||
|
- Tracks progress, speed, and file chunks for resume support
|
||||||
|
- Uses `asyncio.Semaphore` to limit concurrent downloads
|
||||||
|
- Auto-restores completed downloads from disk on server startup
|
||||||
|
|
||||||
|
### 2. Downloaders (`app/downloaders/`)
|
||||||
|
|
||||||
|
**Architecture:**
|
||||||
|
The downloaders are organized into three categories with separate base classes:
|
||||||
|
|
||||||
|
**Anime Sites** (`app/downloaders/anime_sites/`):
|
||||||
|
- Provide anime catalogs, metadata, and episode listings
|
||||||
|
- Link to video players for actual file hosting
|
||||||
|
- Inherit from `BaseAnimeSite` abstract class
|
||||||
|
- Factory: `get_anime_site(url)` in `anime_sites/__init__.py`
|
||||||
|
- Implement: `search_anime()`, `get_episodes()`, `get_anime_metadata()`, `get_download_link()`
|
||||||
|
|
||||||
|
**Series Sites** (`app/downloaders/series_sites/`):
|
||||||
|
- Provide TV series catalogs, metadata, and episode listings
|
||||||
|
- Similar to anime sites but for general TV series content
|
||||||
|
- Inherit from `BaseSeriesSite` abstract class
|
||||||
|
- Factory: `get_series_site(url)` in `series_sites/__init__.py`
|
||||||
|
- Implement: `search_anime()`, `get_episodes()`, `get_anime_metadata()`, `get_download_link()`
|
||||||
|
|
||||||
|
**Video Players** (`app/downloaders/video_players/`):
|
||||||
|
- Host actual video files and provide direct download links
|
||||||
|
- Extract URLs from embedded players and handle file downloads
|
||||||
|
- Inherit from `BaseVideoPlayer` abstract class
|
||||||
|
- Factory: `get_video_player(url)` in `video_players/__init__.py`
|
||||||
|
- Implement: `get_download_link(url, target_filename=None)`
|
||||||
|
|
||||||
|
**Three-Tier Factory Pattern:**
|
||||||
|
- `get_downloader(url)` in main `__init__.py` checks: anime sites → series sites → video players
|
||||||
|
- Falls back to `GenericDownloader` if no match
|
||||||
|
- This separation allows anime/series sites to delegate to video players for actual downloads
|
||||||
|
|
||||||
|
**BaseAnimeSite Interface:**
|
||||||
|
- `can_handle(url)` - Check if this anime site can handle the URL
|
||||||
|
- `search_anime(query, lang)` - Search for anime, returns list with title, url, cover_image
|
||||||
|
- `get_episodes(anime_url, lang)` - Get episode list with episode_number, url, title, host
|
||||||
|
- `get_anime_metadata(anime_url)` - Get metadata dict (synopsis, genres, rating, release_year, studio, poster_image, total_episodes, status)
|
||||||
|
- `get_download_link(url)` - Get video player URL from episode page (NOT direct download link)
|
||||||
|
|
||||||
|
**BaseSeriesSite Interface:**
|
||||||
|
- `can_handle(url)` - Check if this series site can handle the URL
|
||||||
|
- `search_anime(query, lang)` - Search for series, returns list with title, url, cover_image, lang
|
||||||
|
- `get_episodes(anime_url, lang)` - Get episode list with episode_number, url, title, host
|
||||||
|
- `get_anime_metadata(anime_url)` - Get metadata dict (title, synopsis, genres, rating, release_year, studio, poster_image, total_episodes, status, languages)
|
||||||
|
- `get_download_link(url)` - Get video player URL from episode page (NOT direct download link)
|
||||||
|
|
||||||
|
**BaseVideoPlayer Interface:**
|
||||||
|
- `can_handle(url)` - Check if this player can handle the URL
|
||||||
|
- `get_download_link(url, target_filename=None)` - Extract direct download link and filename
|
||||||
|
- Note: `target_filename` parameter is optional but MUST be supported for VidMoly/SendVid compatibility
|
||||||
|
- Always use `sanitize_filename()` on extracted filenames!
|
||||||
|
|
||||||
|
**Key Patterns:**
|
||||||
|
- All downloaders use httpx.AsyncClient for HTTP requests
|
||||||
|
- BeautifulSoup with lxml for HTML parsing
|
||||||
|
- Async/await throughout for non-blocking I/O
|
||||||
|
- Fuzzy search using jieba for Chinese text segmentation and typo tolerance
|
||||||
|
- Security: Filename sanitization enforced via `app.utils` functions
|
||||||
|
|
||||||
|
**URL Format Convention:**
|
||||||
|
- **Pipe-separated format**: `video_url|anime_page_url|episode_title`
|
||||||
|
- Preserves metadata through the download process
|
||||||
|
- Example: `https://vidmoly.to/abc123|https://anime-sama.si/catalogue/naruto/s1/vostfr/|Episode+1`
|
||||||
|
- `target_filename` parameter allows anime/series sites to suggest filenames
|
||||||
|
- Video players extract the final download link and filename
|
||||||
|
|
||||||
|
### 3. Provider Configuration (`app/providers.py`)
|
||||||
|
- `ANIME_PROVIDERS` - Anime streaming sites configuration
|
||||||
|
- `FILE_HOSTS` - File hosting services configuration
|
||||||
|
- Each provider has: name, domains, icon, color, url_pattern
|
||||||
|
- `detect_provider_from_url(url)` - Identify provider from URL
|
||||||
|
|
||||||
|
### 4. API Endpoints
|
||||||
|
|
||||||
|
**Download Management:**
|
||||||
|
- `POST /api/download` - Create new download task
|
||||||
|
- `GET /api/downloads` - List all download tasks
|
||||||
|
- `GET /api/download/{task_id}` - Get task details
|
||||||
|
- `POST /api/download/{task_id}/pause` - Pause download
|
||||||
|
- `POST /api/download/{task_id}/resume` - Resume download
|
||||||
|
- `DELETE /api/download/{task_id}` - Delete task (keeps completed files)
|
||||||
|
- `GET /api/download/{task_id}/file` - Download completed file
|
||||||
|
|
||||||
|
**Anime Features:**
|
||||||
|
- `GET /api/anime/search` - Unified search across all providers
|
||||||
|
- `GET /api/anime/metadata` - Get anime metadata
|
||||||
|
- `GET /api/anime/episodes` - Get episode list
|
||||||
|
- `POST /api/anime/download` - Download single episode
|
||||||
|
- `POST /api/anime/download-season` - Download entire season
|
||||||
|
|
||||||
|
**Video Streaming:**
|
||||||
|
- `GET /video/{task_id}` - Stream video with Range support
|
||||||
|
- `GET /stream/{filename}` - Stream by filename
|
||||||
|
- `GET /player/{task_id}` - Video player page
|
||||||
|
- `GET /watch/{filename}` - Player by filename
|
||||||
|
|
||||||
|
**Recommendations & Favorites:**
|
||||||
|
- `GET /api/recommendations` - Personalized recommendations
|
||||||
|
- `GET /api/releases/latest` - Latest anime releases
|
||||||
|
- `GET /api/favorites` - List favorites
|
||||||
|
- `POST /api/favorites` - Add favorite
|
||||||
|
- `DELETE /api/favorites/{anime_id}` - Remove favorite
|
||||||
|
|
||||||
|
**Sonarr Integration:**
|
||||||
|
- `POST /api/webhook/sonarr` - Receive Sonarr webhooks
|
||||||
|
- `GET /api/sonarr/config` - Get Sonarr configuration
|
||||||
|
- `PUT /api/sonarr/config` - Update Sonarr configuration
|
||||||
|
- `GET /api/sonarr/mappings` - List Sonarr to anime mappings
|
||||||
|
- `POST /api/sonarr/mappings` - Create/update mapping
|
||||||
|
- `DELETE /api/sonarr/mappings/{series_id}` - Delete mapping
|
||||||
|
- `GET /api/sonarr/search` - Search anime for mapping
|
||||||
|
- `GET /api/sonarr/episodes` - Get episode list
|
||||||
|
- `GET /api/sonarr/suggest` - Suggest anime matches
|
||||||
|
- `POST /api/sonarr/download` - Manually trigger download
|
||||||
|
|
||||||
|
### 5. Web Interface
|
||||||
|
- Single-page app at `/web` (templates/index.html)
|
||||||
|
- Auto-refreshes every second to show progress
|
||||||
|
- Video player with seeking support (HTTP Range headers)
|
||||||
|
- Dark theme with gradients and animations
|
||||||
|
|
||||||
|
### 6. Security Utilities (`app/utils.py`)
|
||||||
|
- `sanitize_filename(filename, max_length=255)` - Sanitize filenames to prevent path traversal
|
||||||
|
- Removes dangerous characters: `\ / : * ? " < > |`
|
||||||
|
- Strips path separators and leading dots/dashes
|
||||||
|
- Limits filename length while preserving extension
|
||||||
|
- `is_safe_filename(filename)` - Validate filename safety
|
||||||
|
- Checks for path traversal patterns (`..`, `/`, `\`)
|
||||||
|
- Detects absolute paths and drive letters
|
||||||
|
- Used throughout the codebase for file operations
|
||||||
|
|
||||||
|
### 7. Authentication System (`app/auth.py`)
|
||||||
|
- **UserManager** - JSON-based user storage in `config/users.json`
|
||||||
|
- User registration with bcrypt password hashing
|
||||||
|
- Password truncated to 72 bytes (bcrypt limitation)
|
||||||
|
- User authentication and last login tracking
|
||||||
|
- **JWT Tokens** - Stateless authentication
|
||||||
|
- 7-day token expiration (configurable via `ACCESS_TOKEN_EXPIRE_MINUTES`)
|
||||||
|
- HS256 algorithm with JWT_SECRET_KEY (change in production!)
|
||||||
|
- Token verification and user extraction
|
||||||
|
- **Password Security**
|
||||||
|
- bcrypt hashing with passlib
|
||||||
|
- Automatic deprecated scheme migration
|
||||||
|
- **Configuration**
|
||||||
|
- `JWT_SECRET_KEY` environment variable (default: dev-secret-change-in-production)
|
||||||
|
- Users stored in `config/users.json`
|
||||||
|
|
||||||
|
**Authentication Endpoints:**
|
||||||
|
- `POST /api/auth/register` - User registration
|
||||||
|
- `POST /api/auth/login` - Login and receive JWT token
|
||||||
|
- `GET /api/auth/me` - Get current user profile
|
||||||
|
- `PUT /api/auth/me` - Update user profile
|
||||||
|
|
||||||
|
### 8. Recommendation Engine (`app/recommendation_engine.py`)
|
||||||
|
- Analyzes download history to generate personalized recommendations
|
||||||
|
- Tracks genre preferences and viewing patterns
|
||||||
|
- Scores anime based on user's download history
|
||||||
|
- Used by `/api/recommendations` endpoint
|
||||||
|
|
||||||
|
### 9. Kitsu API (`app/kitsu_api.py`)
|
||||||
|
- Integrates with Kitsu anime database for metadata
|
||||||
|
- Fetches anime information by title or ID
|
||||||
|
- Provides enriched metadata (synopsis, genres, ratings, poster images)
|
||||||
|
- Used as fallback when provider metadata is incomplete
|
||||||
|
|
||||||
|
### 10. Watchlist & Auto-Download System
|
||||||
|
|
||||||
|
**WatchlistManager** (`app/watchlist.py`):
|
||||||
|
- JSON-based storage in `config/watchlist.json`
|
||||||
|
- Per-user watchlist management (multi-tenant)
|
||||||
|
- CRUD operations for tracked anime
|
||||||
|
- Statistics and queries
|
||||||
|
- Settings management in `config/watchlist_settings.json`
|
||||||
|
|
||||||
|
**EpisodeChecker** (`app/episode_checker.py`):
|
||||||
|
- Checks for new episodes for anime in watchlist
|
||||||
|
- Downloads episodes automatically when detected
|
||||||
|
- Integrates with existing downloaders
|
||||||
|
- Handles errors and retries
|
||||||
|
- Lazy initialization to avoid circular imports
|
||||||
|
|
||||||
|
**AutoDownloadScheduler** (`app/auto_download_scheduler.py`):
|
||||||
|
- APScheduler-based periodic checking
|
||||||
|
- Configurable intervals (1-168 hours)
|
||||||
|
- Start/stop control via API
|
||||||
|
- Next run tracking
|
||||||
|
- Background task execution
|
||||||
|
|
||||||
|
**Watchlist Models** (`app/models/watchlist.py`):
|
||||||
|
- `WatchlistItem` - Tracked anime with settings
|
||||||
|
- `WatchlistStatus` - ACTIVE, PAUSED, COMPLETED, ARCHIVED
|
||||||
|
- `QualityPreference` - AUTO, 1080p, 720p, 480p
|
||||||
|
- `WatchlistSettings` - Global configuration
|
||||||
|
- `AutoDownloadResult` - Operation results
|
||||||
|
|
||||||
|
**Watchlist Endpoints:**
|
||||||
|
- `GET /api/watchlist` - List user's watchlist (with status filter)
|
||||||
|
- `POST /api/watchlist` - Add anime to watchlist
|
||||||
|
- `GET /api/watchlist/{item_id}` - Get specific item
|
||||||
|
- `PUT /api/watchlist/{item_id}` - Update watchlist item
|
||||||
|
- `DELETE /api/watchlist/{item_id}` - Remove from watchlist
|
||||||
|
- `POST /api/watchlist/{item_id}/check` - Check specific anime
|
||||||
|
- `POST /api/watchlist/check-all` - Check all due items
|
||||||
|
- `POST /api/watchlist/{item_id}/pause` - Pause tracking
|
||||||
|
- `POST /api/watchlist/{item_id}/resume` - Resume tracking
|
||||||
|
- `GET /api/watchlist/settings` - Get global settings
|
||||||
|
- `PUT /api/watchlist/settings` - Update settings
|
||||||
|
- `GET /api/watchlist/stats` - Get watchlist statistics
|
||||||
|
- `GET /api/watchlist/scheduler/status` - Get scheduler status
|
||||||
|
- `POST /api/watchlist/scheduler/start` - Start scheduler
|
||||||
|
- `POST /api/watchlist/scheduler/stop` - Stop scheduler
|
||||||
|
|
||||||
|
### 11. Pydantic Models (`app/models/`)
|
||||||
|
- **`__init__.py`** - Core models:
|
||||||
|
- `DownloadStatus` - Enum for task states (PENDING, DOWNLOADING, PAUSED, COMPLETED, FAILED, CANCELLED)
|
||||||
|
- `HostType` - Enum for file host types (RAPIDFILE, UNFICHIER, DOODSTREAM, OTHER)
|
||||||
|
- `DownloadTask` - Main task model with progress tracking
|
||||||
|
- `DownloadRequest` - Request model for creating downloads
|
||||||
|
- `AnimeMetadata` - Anime information (synopsis, genres, rating, release_year, studio, etc.)
|
||||||
|
- `AnimeSearchResult` - Enhanced search result with metadata
|
||||||
|
- **`sonarr.py`** - Sonarr-specific models:
|
||||||
|
- `SonarrWebhookPayload` - Complete webhook payload schema
|
||||||
|
- `SonarrEventType` - Enum for event types (Grab, Download, Rename, Delete, Test)
|
||||||
|
- `SonarrMapping` - Mapping between Sonarr series and anime providers
|
||||||
|
- `SonarrConfig` - Webhook configuration (enabled, secret, auto-download, etc.)
|
||||||
|
- **`auth.py`** - Authentication models:
|
||||||
|
- `UserCreate` - User registration request
|
||||||
|
- `UserLogin` - Login request
|
||||||
|
- `User` - User profile
|
||||||
|
- `Token` - JWT token response
|
||||||
|
- **`watchlist.py`** - Watchlist models:
|
||||||
|
- `WatchlistItem` - Tracked anime item
|
||||||
|
- `WatchlistItemCreate` - Create request
|
||||||
|
- `WatchlistItemUpdate` - Update request
|
||||||
|
- `WatchlistStatus` - Status enum
|
||||||
|
- `WatchlistSettings` - Global settings
|
||||||
|
|
||||||
|
## Test Structure
|
||||||
|
|
||||||
|
**Test Organization (tests/):**
|
||||||
|
- `conftest.py` - Pytest configuration and fixtures
|
||||||
|
- `test_models.py` - Pydantic model tests
|
||||||
|
- `test_downloaders.py` - Downloader tests
|
||||||
|
- `test_download_manager.py` - DownloadManager tests
|
||||||
|
- `test_favorites.py` - Favorites system tests
|
||||||
|
- `test_api.py` - FastAPI endpoint tests
|
||||||
|
- `test_sonarr.py` - Sonarr integration tests
|
||||||
|
- `test_anime_sama_seasons.py` - Anime-Sama season handling tests
|
||||||
|
- `test_translate_api.py` - Translation API tests
|
||||||
|
- `test_delete_and_restore.py` - Delete and restore functionality tests
|
||||||
|
- `test_french_manga.py` - French-Manga provider tests
|
||||||
|
|
||||||
|
**Fixtures in conftest.py:**
|
||||||
|
- `temp_dir` - Temporary directory
|
||||||
|
- `temp_download_dir` - Temporary download directory
|
||||||
|
- `download_manager` - DownloadManager instance
|
||||||
|
- `favorites_manager` - FavoritesManager instance
|
||||||
|
- `mock_httpx_client` - Mock for httpx.AsyncClient
|
||||||
|
- `sample_download_task` - Sample task data
|
||||||
|
- `sample_anime_metadata` - Sample metadata
|
||||||
|
|
||||||
|
**Test Markers:**
|
||||||
|
- `unit` - Unit tests (isolated, fast) - auto-applied
|
||||||
|
- `integration` - Integration tests (API endpoints) - auto-applied
|
||||||
|
- `asyncio` - Async tests - auto-applied
|
||||||
|
- `slow` - Slow tests - manual
|
||||||
|
- `network` - Requires network - manual
|
||||||
|
|
||||||
|
**pytest.ini Configuration:**
|
||||||
|
- Auto-applies markers for async and integration tests
|
||||||
|
- Coverage enabled by default (`--cov=app`)
|
||||||
|
- HTML coverage report generated in `htmlcov/`
|
||||||
|
- Verbose output with local variables in tracebacks
|
||||||
|
- 300-second timeout for tests
|
||||||
|
- `asyncio_mode = auto` for async test support
|
||||||
|
|
||||||
|
**Running Single Test:**
|
||||||
|
```bash
|
||||||
|
# Run specific test file
|
||||||
|
pytest tests/test_sonarr.py -v
|
||||||
|
|
||||||
|
# Run specific test class
|
||||||
|
pytest tests/test_sonarr.py::TestSonarrHandler -v
|
||||||
|
|
||||||
|
# Run specific test
|
||||||
|
pytest tests/test_sonarr.py::TestSonarrHandler::test_add_mapping -v
|
||||||
|
```
|
||||||
|
|
||||||
|
## Adding New Host Support
|
||||||
|
|
||||||
|
To add support for a new file hosting service:
|
||||||
|
|
||||||
|
1. Create new file in `app/downloaders/video_players/` (e.g., `myhost.py`)
|
||||||
|
2. Inherit from `BaseVideoPlayer`
|
||||||
|
3. Implement required methods (`can_handle`, `get_download_link`)
|
||||||
|
4. Add to imports in `app/downloaders/video_players/__init__.py`
|
||||||
|
5. Add to `players` list in `get_video_player()`
|
||||||
|
6. Add configuration to `FILE_HOSTS` in `app/providers.py`
|
||||||
|
|
||||||
|
Example:
|
||||||
|
```python
|
||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
class MyHostDownloader(BaseVideoPlayer):
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return "myhost.com" in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: Optional[str] = None) -> tuple[str, str]:
|
||||||
|
soup = BeautifulSoup(await self._fetch_page(url), 'lxml')
|
||||||
|
# ... extraction logic ...
|
||||||
|
# IMPORTANT: Always sanitize filenames!
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
filename = sanitize_filename(extracted_filename)
|
||||||
|
return download_url, filename
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
# IMPORTANT: Always close the HTTP client
|
||||||
|
await self.client.aclose()
|
||||||
|
```
|
||||||
|
|
||||||
|
**Important:**
|
||||||
|
- Always close the HTTP client in your downloader to avoid resource leaks
|
||||||
|
- Use `sanitize_filename()` from `app.utils` when extracting filenames from URLs
|
||||||
|
- Use `is_safe_filename()` to validate filenames before file operations
|
||||||
|
- The `target_filename` parameter is required for compatibility with anime/series sites
|
||||||
|
|
||||||
|
## Adding New Series Site
|
||||||
|
|
||||||
|
To add a new TV series streaming provider (similar to anime sites but for general TV series):
|
||||||
|
|
||||||
|
1. Create new file in `app/downloaders/series_sites/` (e.g., `mysite.py`)
|
||||||
|
2. Inherit from `BaseSeriesSite`
|
||||||
|
3. Implement series-specific methods:
|
||||||
|
- `search_anime(query, lang)` - Return list of series with title, url, cover_image, lang
|
||||||
|
- `get_episodes(anime_url, lang)` - Return list of episodes
|
||||||
|
- `get_anime_metadata(anime_url)` - Return metadata dict (should include languages field)
|
||||||
|
- `get_download_link(url)` - Return video player URL from episode page
|
||||||
|
4. Add to imports in `app/downloaders/series_sites/__init__.py`
|
||||||
|
5. Add to `sites` list in `get_series_site()`
|
||||||
|
|
||||||
|
BaseSeriesSite is nearly identical to BaseAnimeSite but designed for general TV series content rather than anime-specific content.
|
||||||
|
|
||||||
|
## Sonarr Integration
|
||||||
|
|
||||||
|
The application includes full Sonarr webhook support for automated anime downloads.
|
||||||
|
|
||||||
|
### Architecture
|
||||||
|
|
||||||
|
**SonarrHandler (`app/sonarr_handler.py`):**
|
||||||
|
- Processes incoming webhooks from Sonarr
|
||||||
|
- Manages series mappings (Sonarr TVDB ID → Anime Provider URL)
|
||||||
|
- Supports HMAC SHA256 signature verification for security
|
||||||
|
- Auto-triggers downloads on Grab events
|
||||||
|
- Provides search and suggestion APIs for mapping setup
|
||||||
|
|
||||||
|
**Sonarr Models (`app/models/sonarr.py`):**
|
||||||
|
- `SonarrWebhookPayload` - Complete webhook payload schema
|
||||||
|
- `SonarrEventType` - Enum for event types (Grab, Download, Rename, Delete, Test)
|
||||||
|
- `SonarrMapping` - Mapping between Sonarr series and anime providers
|
||||||
|
- `SonarrConfig` - Webhook configuration (enabled, secret, auto-download, etc.)
|
||||||
|
|
||||||
|
### Workflow
|
||||||
|
|
||||||
|
1. **Setup in Sonarr:**
|
||||||
|
- Configure webhook: Settings > Connect > Sonarr > Webhook
|
||||||
|
- URL: `http://your-server:3000/api/webhook/sonarr`
|
||||||
|
- Enable "Grab" event
|
||||||
|
|
||||||
|
2. **Create Mappings:**
|
||||||
|
- Get Sonarr series TVDB ID from series details
|
||||||
|
- Search anime: `GET /api/sonarr/search?q={title}`
|
||||||
|
- Create mapping: `POST /api/sonarr/mappings`
|
||||||
|
|
||||||
|
3. **Automatic Download:**
|
||||||
|
- Sonarr grabs new episode → Sends webhook
|
||||||
|
- Ohm Stream Downloader receives webhook
|
||||||
|
- Looks up mapping by TVDB ID
|
||||||
|
- Finds matching episode on anime provider
|
||||||
|
- Creates and starts download task
|
||||||
|
|
||||||
|
### Configuration Files
|
||||||
|
|
||||||
|
- `config/sonarr.json` - Webhook configuration
|
||||||
|
- `config/sonarr_mappings.json` - Series mappings
|
||||||
|
|
||||||
|
### Example Mapping
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"sonarr_series_id": 79644,
|
||||||
|
"sonarr_title": "Naruto Shippuden",
|
||||||
|
"anime_provider": "anime-sama",
|
||||||
|
"anime_url": "https://anime-sama.si/catalogue/naruto-shippuden/saison1/vostfr/",
|
||||||
|
"anime_title": "Naruto Shippuden",
|
||||||
|
"lang": "vostfr",
|
||||||
|
"quality_preference": "1080p",
|
||||||
|
"auto_download": true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- Optional HMAC SHA256 signature verification
|
||||||
|
- Configure secret in both Sonarr and Ohm Stream Downloader
|
||||||
|
- Enable with `verify_hmac: true` in config
|
||||||
|
|
||||||
|
### Testing
|
||||||
|
|
||||||
|
- Test endpoint: `POST /api/webhook/test/sonarr`
|
||||||
|
- Manual trigger: `POST /api/sonarr/download`
|
||||||
|
- Get suggestions: `GET /api/sonarr/suggest?sonarr_title={title}`
|
||||||
|
|
||||||
|
**Documentation:** See `docs/SONARR_INTEGRATION.md` for complete setup guide.
|
||||||
|
|
||||||
|
## Adding New Anime Provider
|
||||||
|
|
||||||
|
To add a new anime streaming provider:
|
||||||
|
|
||||||
|
1. Create new file in `app/downloaders/anime_sites/` (e.g., `mysite.py`)
|
||||||
|
2. Inherit from `BaseAnimeSite`
|
||||||
|
3. Implement anime-specific methods:
|
||||||
|
- `search_anime(query, lang)` - Return list of anime with title, url, cover_image
|
||||||
|
- `get_episodes(anime_url, lang)` - Return list of episodes
|
||||||
|
- `get_anime_metadata(anime_url)` - Return metadata dict
|
||||||
|
- `get_download_link(url)` - Return video player URL from episode page
|
||||||
|
4. Add to imports in `app/downloaders/anime_sites/__init__.py`
|
||||||
|
5. Add to `sites` list in `get_anime_site()`
|
||||||
|
6. Add to `ANIME_PROVIDERS` in `app/providers.py`
|
||||||
|
7. Update `main.py` to include in unified search
|
||||||
|
|
||||||
|
Metadata should include:
|
||||||
|
- synopsis, genres, rating, release_year, studio, poster_image, total_episodes, status
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
The application uses environment variables for configuration via `app/config.py` (Pydantic Settings).
|
||||||
|
|
||||||
|
**Environment Variables (.env):**
|
||||||
|
```bash
|
||||||
|
# Copy the example file
|
||||||
|
cp .env.example .env
|
||||||
|
|
||||||
|
# Edit .env to configure:
|
||||||
|
APP_NAME=Ohm Stream Downloader # Application name
|
||||||
|
DEBUG=false # Debug mode
|
||||||
|
HOST=0.0.0.0 # Server host
|
||||||
|
PORT=3000 # Server port
|
||||||
|
DOWNLOAD_DIR=downloads # Download storage location
|
||||||
|
MAX_PARALLEL_DOWNLOADS=3 # Maximum concurrent downloads
|
||||||
|
CHUNK_SIZE=1048576 # Download chunk size (1MB)
|
||||||
|
CORS_ORIGINS=... # Comma-separated allowed origins
|
||||||
|
HTTP_TIMEOUT=10.0 # HTTP request timeout (seconds)
|
||||||
|
DOWNLOAD_TIMEOUT=300 # Download timeout (seconds)
|
||||||
|
LOG_LEVEL=INFO # Logging level
|
||||||
|
JWT_SECRET_KEY=change-me-in-production # JWT signing key for auth
|
||||||
|
```
|
||||||
|
|
||||||
|
**Configuration Files:**
|
||||||
|
- `.env` - Environment configuration (create from .env.example)
|
||||||
|
- `config/users.json` - User authentication database (created automatically)
|
||||||
|
- `config/sonarr.json` - Sonarr webhook configuration (created automatically)
|
||||||
|
- `config/sonarr_mappings.json` - Sonarr to anime provider mappings (created automatically)
|
||||||
|
- `config/watchlist.json` - User watchlist items (created automatically)
|
||||||
|
- `config/watchlist_settings.json` - Watchlist global settings (created automatically)
|
||||||
|
- `config/.gitkeep` - Ensures config directory is tracked in git
|
||||||
|
- Example files: `config/sonarr.example.json`, `config/sonarr_mappings.example.json`
|
||||||
|
|
||||||
|
**Documentation:**
|
||||||
|
- `README.md` - User-facing features and roadmap
|
||||||
|
- `CLAUDE.md` - This file (developer guide)
|
||||||
|
- `docs/SONARR_INTEGRATION.md` - Complete Sonarr setup guide
|
||||||
|
- `docs/SONARR_IMPLEMENTATION.md` - Technical implementation summary
|
||||||
|
- `docs/IMPROVEMENTS_2024-01-24.md` - Recent security and quality improvements
|
||||||
|
- `docs/WATCHLIST_AUTO_DOWNLOAD.md` - Watchlist system documentation
|
||||||
|
|
||||||
|
## Security
|
||||||
|
|
||||||
|
**Filename Sanitization (`app/utils.py`):**
|
||||||
|
- `sanitize_filename()` - Removes dangerous characters (`\ / : * ? " < > |`)
|
||||||
|
- `is_safe_filename()` - Validates against path traversal patterns
|
||||||
|
- Used throughout the codebase for all file operations
|
||||||
|
- Prevents `../../../etc/passwd` style attacks
|
||||||
|
- Limits filename length to 255 characters
|
||||||
|
|
||||||
|
**CORS Configuration:**
|
||||||
|
- Restricted origins (not `*`) in production
|
||||||
|
- Specific allowed methods (GET, POST, PUT, DELETE, PATCH, OPTIONS)
|
||||||
|
- Configured in `main.py` via environment variables
|
||||||
|
|
||||||
|
**Authentication:**
|
||||||
|
- JWT token-based authentication with 7-day expiration
|
||||||
|
- bcrypt password hashing with passlib
|
||||||
|
- Passwords truncated to 72 bytes (bcrypt limitation)
|
||||||
|
- Credentials stored in `config/users.json`
|
||||||
|
|
||||||
|
## Key Implementation Details
|
||||||
|
|
||||||
|
**Resume Support:**
|
||||||
|
- Downloads use HTTP Range headers to resume from last byte
|
||||||
|
- Files downloaded in 1MB chunks
|
||||||
|
- Partial files cleaned up on cancel
|
||||||
|
- Resume position tracked in `downloaded_bytes` field
|
||||||
|
|
||||||
|
**Domain Handling:**
|
||||||
|
- Anime providers use dynamic domain detection (e.g., Anime-Sama fetches current domain from anime-sama.pw)
|
||||||
|
- Multiple domains per provider supported in configuration
|
||||||
|
- Domain detection via `detect_provider_from_url(url)` in providers.py
|
||||||
|
|
||||||
|
**Task Lifecycle:**
|
||||||
|
- PENDING → DOWNLOADING → PAUSED / COMPLETED / CANCELLED / FAILED
|
||||||
|
- Active downloads tracked in `active_downloads` dict
|
||||||
|
- All tasks stored in `tasks` dict with UUID keys
|
||||||
|
- Completed files preserved when deleting tasks (only partial files removed)
|
||||||
|
|
||||||
|
**Video Streaming:**
|
||||||
|
- Range header support for seeking in video player
|
||||||
|
- Serves from `/downloads` directory via StaticFiles
|
||||||
|
- Video extensions: .mp4, .mkv, .avi, .mov, .wmv, .flv, .webm
|
||||||
|
|
||||||
|
**Error Handling:**
|
||||||
|
- Graceful degradation with status tracking
|
||||||
|
- Network errors caught and reported in task status
|
||||||
|
- Automatic retry on resume
|
||||||
|
- Downloads > 1MB considered complete to skip small error files
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
**Core:**
|
||||||
|
- fastapi - Web framework
|
||||||
|
- uvicorn - ASGI server
|
||||||
|
- httpx - Async HTTP client
|
||||||
|
- beautifulsoup4, lxml - HTML parsing
|
||||||
|
- aiofiles - Async file operations
|
||||||
|
- jieba - Chinese text segmentation for fuzzy search
|
||||||
|
- passlib[bcrypt] - Password hashing
|
||||||
|
- python-jose[cryptography] - JWT token handling
|
||||||
|
- apscheduler - Task scheduling for auto-download
|
||||||
|
|
||||||
|
**Testing:**
|
||||||
|
- pytest - Test framework
|
||||||
|
- pytest-asyncio - Async test support
|
||||||
|
- pytest-cov - Coverage reporting
|
||||||
|
- pytest-mock - Mocking support
|
||||||
|
- pytest-timeout - Test timeout handling
|
||||||
|
- pytest-html - HTML test reports
|
||||||
-56
@@ -1,56 +0,0 @@
|
|||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Étape 1 — dépendances Python via uv (cache couche par couche)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
|
|
||||||
ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
|
|
||||||
WORKDIR /opt/ohm
|
|
||||||
|
|
||||||
# D'abord les métadonnées seules : couche réutilisable tant que uv.lock ne bouge pas
|
|
||||||
COPY pyproject.toml uv.lock ./
|
|
||||||
RUN uv sync --frozen --no-dev --no-install-project --no-cache
|
|
||||||
|
|
||||||
# Puis le code
|
|
||||||
COPY app ./app
|
|
||||||
RUN uv sync --frozen --no-dev --no-cache
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Étape 2 — image d'exécution minimale
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
FROM python:3.13-slim-bookworm
|
|
||||||
|
|
||||||
# ffmpeg (téléchargements HLS), ca-certificates (scraping HTTPS),
|
|
||||||
# gosu (bascule utilisateur non-root dans l'entrypoint)
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends ffmpeg ca-certificates gosu \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Utilisateur non-root
|
|
||||||
RUN useradd --create-home --uid 1000 ohm
|
|
||||||
|
|
||||||
WORKDIR /opt/ohm
|
|
||||||
COPY --from=builder --chown=ohm:ohm /opt/ohm/.venv ./.venv
|
|
||||||
COPY --chown=ohm:ohm app ./app
|
|
||||||
COPY --chown=ohm:ohm docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
|
||||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
|
||||||
|
|
||||||
ENV PATH="/opt/ohm/.venv/bin:$PATH" \
|
|
||||||
PYTHONUNBUFFERED=1 \
|
|
||||||
# Chemins montés en volumes par docker-compose
|
|
||||||
OHM_DATA_DIR=/data \
|
|
||||||
OHM_DOWNLOAD_DIR=/downloads \
|
|
||||||
OHM_DATABASE_PATH=/data/ohm.db
|
|
||||||
|
|
||||||
# Version cuite dans l'image par scripts/release.sh (build-arg VERSION)
|
|
||||||
ARG VERSION=dev
|
|
||||||
ENV OHM_VERSION=${VERSION}
|
|
||||||
|
|
||||||
RUN mkdir -p /data /downloads && chown -R ohm:ohm /opt/ohm /data /downloads
|
|
||||||
# Root par défaut : l'entrypoint chown les volumes puis passe en « ohm »
|
|
||||||
|
|
||||||
EXPOSE 8777
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
|
||||||
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8777/health', timeout=4)"]
|
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
|
||||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8777"]
|
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
# 🔧 Correction Import Error - VidMoly
|
||||||
|
|
||||||
|
## Problème
|
||||||
|
|
||||||
|
Quand on tentait un téléchargement depuis le web avec une URL Anime-Sama qui pointait vers VidMoly:
|
||||||
|
```
|
||||||
|
Error extracting AnimeSama link: Error extracting from vidmoly:
|
||||||
|
No module named 'app.downloaders.anime_sites.vidmoly'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Cause Racine
|
||||||
|
|
||||||
|
Après la restructuration, les players vidéo ont été déplacés de `app/downloaders/` vers `app/downloaders/video_players/`, mais `AnimeSamaDownloader` essayait encore d'importer `VidMolyDownloader` depuis `anime_sites/`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# ❌ Ancien import (ne fonctionne plus)
|
||||||
|
from .vidmoly import VidMolyDownloader
|
||||||
|
```
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
|
||||||
|
Corriger tous les imports de players vidéo dans `AnimeSamaDownloader`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# ✅ Nouvel import (correct)
|
||||||
|
from ..video_players.vidmoly import VidMolyDownloader
|
||||||
|
from ..video_players.sendvid import SendVidDownloader
|
||||||
|
from ..video_players.sibnet import SibnetDownloader
|
||||||
|
from ..video_players.lpayer import LpayerDownloader
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers Modifiés
|
||||||
|
|
||||||
|
**`app/downloaders/anime_sites/animesama.py`**:
|
||||||
|
- Ligne 195: `from ..video_players.vidmoly import VidMolyDownloader`
|
||||||
|
- Ligne 257: `from ..video_players.sendvid import SendVidDownloader`
|
||||||
|
- Ligne 304: `from ..video_players.sibnet import SibnetDownloader`
|
||||||
|
- Ligne 401: `from ..video_players.lpayer import LpayerDownloader`
|
||||||
|
|
||||||
|
## Vérification
|
||||||
|
|
||||||
|
✅ **23/23 tests passants**
|
||||||
|
✅ **Téléchargement test**: Anime-Sama → VidMoly fonctionne
|
||||||
|
✅ **API endpoint**: `/api/download` fonctionne correctement
|
||||||
|
✅ **Imports**: Tous les paths sont corrects
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
```python
|
||||||
|
# Test d'un téléchargement complet
|
||||||
|
POST /api/download
|
||||||
|
{
|
||||||
|
"url": "https://anime-sama.si/catalogue/naruto/saison1/vostfr/episode-1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Réponse: 200 OK
|
||||||
|
{
|
||||||
|
"task_id": "...",
|
||||||
|
"status": "pending",
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Autres Sites Anime
|
||||||
|
|
||||||
|
✅ **NekoSama**: Aucun import de video player (OK)
|
||||||
|
✅ **AnimeUltime**: Aucun import de video player (OK)
|
||||||
|
✅ **Vostfree**: Aucun import de video player (OK)
|
||||||
|
|
||||||
|
Seul `AnimeSama` utilise des imports directs de video players.
|
||||||
|
|
||||||
|
---
|
||||||
|
**Statut**: ✅ Corrigé et testé
|
||||||
|
**Impact**: Le téléchargement depuis le web fonctionne maintenant
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# ✅ Verification Frontend - Restructuration
|
||||||
|
|
||||||
|
## Tests Effectués
|
||||||
|
|
||||||
|
### 1. ✅ Application Startup
|
||||||
|
- Import de `main.py`: ✅ réussi
|
||||||
|
- 59 routes chargées: ✅
|
||||||
|
- Routes clés présentes:
|
||||||
|
- `/api/download` ✅
|
||||||
|
- `/api/downloads` ✅
|
||||||
|
- `/api/anime/search` ✅
|
||||||
|
- `/web` ✅
|
||||||
|
|
||||||
|
### 2. ✅ Providers API
|
||||||
|
- **Endpoint**: `GET /api/providers`
|
||||||
|
- **Status**: 200 ✅
|
||||||
|
- **Anime providers**: 4 (Anime-Sama, Neko-Sama, Anime-Ultime, Vostfree)
|
||||||
|
- **File hosts**: 8 (1fichier, Uptobox, Doodstream, Rapidfile, VidMoly, SendVid, Sibnet, Lplayer)
|
||||||
|
|
||||||
|
### 3. ✅ Downloader Routing
|
||||||
|
Tous les downloaders sont correctement routés:
|
||||||
|
- DoodStreamDownloader ✅
|
||||||
|
- AnimeSamaDownloader ✅
|
||||||
|
- NekoSamaDownloader ✅
|
||||||
|
- SibnetDownloader ✅
|
||||||
|
- VidMolyDownloader ✅
|
||||||
|
- SendVidDownloader ✅
|
||||||
|
- UnFichierDownloader ✅
|
||||||
|
- UptoboxDownloader ✅
|
||||||
|
- RapidFileDownloader ✅
|
||||||
|
- LpayerDownloader ✅
|
||||||
|
|
||||||
|
### 4. ✅ Frontend Pages
|
||||||
|
- **Page d'accueil** (`/web`): Status 200, HTML valide ✅
|
||||||
|
- **API downloads** (`/api/downloads`): Status 200, retourne dict ✅
|
||||||
|
|
||||||
|
## Modifications Apportées
|
||||||
|
|
||||||
|
### `app/providers.py`
|
||||||
|
Ajout des 4 nouveaux file hosts qui manquaient:
|
||||||
|
- VidMoly (vidmoly.to, vidmoly.org, vidmoly.biz)
|
||||||
|
- SendVid (sendvid.com, sendvid.io)
|
||||||
|
- Sibnet (sibnet.ru, video.sibnet.ru)
|
||||||
|
- Lplayer (lpayer.embed4me.com, lpayer.com, lplayer.fr)
|
||||||
|
|
||||||
|
## Conclusion
|
||||||
|
|
||||||
|
✅ **Le frontend fonctionne parfaitement avec la nouvelle structure!**
|
||||||
|
|
||||||
|
Aucune rupture de fonctionnalité détectée. Tous les endpoints API sont opérationnels et le frontend peut accéder à tous les providers.
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
# ✅ Rapport Final - Vérification Frontend
|
||||||
|
|
||||||
|
## Date: 2026-01-24
|
||||||
|
|
||||||
|
## 🎯 Conclusion
|
||||||
|
|
||||||
|
**🎉 Le frontend est 100% cohérent et fonctionnel!**
|
||||||
|
|
||||||
|
Aucune erreur ou incohérence détectée.
|
||||||
|
|
||||||
|
## 📊 Fichiers Vérifiés
|
||||||
|
|
||||||
|
### Static Files (11 fichiers)
|
||||||
|
✅ **JavaScript (7 fichiers)**:
|
||||||
|
- api.js (3,545 octets)
|
||||||
|
- utils.js (2,429 octets)
|
||||||
|
- downloads.js (14,380 octets)
|
||||||
|
- anime.js (14,085 octets)
|
||||||
|
- anime-details.js (18,829 octets)
|
||||||
|
- recommendations.js (11,008 octets)
|
||||||
|
- main.js (7,494 octets)
|
||||||
|
|
||||||
|
✅ **CSS (1 fichier)**:
|
||||||
|
- style.css (31,976 octets)
|
||||||
|
|
||||||
|
✅ **Templates HTML (3 fichiers)**:
|
||||||
|
- index.html (286 octets)
|
||||||
|
- base.html (834 octets)
|
||||||
|
- player.html (6,082 octets)
|
||||||
|
|
||||||
|
## 🔗 Vérifications Effectuées
|
||||||
|
|
||||||
|
### 1. ✅ Intégrité des Fichiers
|
||||||
|
- Tous les fichiers JS/CSS/HTML sont présents
|
||||||
|
- Tous les fichiers référencés dans base.html existent
|
||||||
|
- Aucun lien cassé
|
||||||
|
|
||||||
|
### 2. ✅ Cohérence Frontend/Backend
|
||||||
|
Tous les endpoints API fonctionnent:
|
||||||
|
- `GET /web` → 200 ✅
|
||||||
|
- `GET /api/providers` → 200 ✅
|
||||||
|
- `GET /api/downloads` → 200 ✅
|
||||||
|
- `POST /api/download` → 200 ✅
|
||||||
|
|
||||||
|
### 3. ✅ Providers Configurés
|
||||||
|
**8 File hosts** (tous complets avec name, domains, icon, color):
|
||||||
|
1. 1fichier ✅
|
||||||
|
2. Uptobox ✅
|
||||||
|
3. Doodstream ✅
|
||||||
|
4. Rapidfile ✅
|
||||||
|
5. VidMoly ✅
|
||||||
|
6. SendVid ✅
|
||||||
|
7. Sibnet ✅
|
||||||
|
8. Lplayer ✅
|
||||||
|
|
||||||
|
**4 Anime sites**:
|
||||||
|
1. Anime-Sama ✅
|
||||||
|
2. Neko-Sama ✅
|
||||||
|
3. Anime-Ultime ✅
|
||||||
|
4. Vostfree ✅
|
||||||
|
|
||||||
|
### 4. ✅ Imports JavaScript
|
||||||
|
- Tous les imports entre modules JS sont valides
|
||||||
|
- Les appels API utilisent les bons endpoints
|
||||||
|
- Les références aux providers sont cohérentes
|
||||||
|
|
||||||
|
### 5. ✅ Structure HTML/CSS
|
||||||
|
- base.html référence correctement tous les scripts
|
||||||
|
- Les IDs et classes CSS sont cohérents
|
||||||
|
- Les styles sont correctement chargés
|
||||||
|
|
||||||
|
## 📝 Tests Réalisés
|
||||||
|
|
||||||
|
| Test | Résultat | Détails |
|
||||||
|
|------|----------|---------|
|
||||||
|
| Fichiers statiques | ✅ | 11/11 présents |
|
||||||
|
| Références HTML | ✅ | Tous les liens valides |
|
||||||
|
| Endpoints API | ✅ | 4/4 fonctionnels |
|
||||||
|
| Providers | ✅ | 12/12 complets |
|
||||||
|
| Imports JS | ✅ | Aucune erreur |
|
||||||
|
| Cohérence CSS | ✅ | Styles chargés |
|
||||||
|
|
||||||
|
## ✨ Points Forts du Frontend
|
||||||
|
|
||||||
|
1. **Code propre**: Gestion d'erreur présente dans tous les fichiers JS
|
||||||
|
2. **Modulaire**: Séparation claire (api, utils, downloads, anime, etc.)
|
||||||
|
3. **Complet**: Tous les endpoints backend sont accessibles
|
||||||
|
4. **Maintenable**: Structure claire et bien organisée
|
||||||
|
5. **Robuste**: Gestion d'erreur à tous les niveaux
|
||||||
|
|
||||||
|
## 🚀 Après Restructuration
|
||||||
|
|
||||||
|
La restructuration des downloaders n'a **AUCUN IMPACT** négatif sur le frontend:
|
||||||
|
- Tous les endpoints API fonctionnent identiquement
|
||||||
|
- Les providers sont tous accessibles
|
||||||
|
- L'interface web est pleinement fonctionnelle
|
||||||
|
- Aucune modification nécessaire dans le code JS
|
||||||
|
|
||||||
|
---
|
||||||
|
**Vérifié par**: Claude Code
|
||||||
|
**Date**: 2026-01-24
|
||||||
|
**Statut**: ✅ Frontend 100% valide
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
# ✅ Rapport de Vérification - Imports Complets
|
||||||
|
|
||||||
|
## Date: 2026-01-24
|
||||||
|
|
||||||
|
## 🔍 Vérifications Effectuées
|
||||||
|
|
||||||
|
### 1. ✅ Analyse Statique du Code
|
||||||
|
- **14 fichiers Python** vérifiés dans la nouvelle structure
|
||||||
|
- **0 erreur** d'import détectée
|
||||||
|
- Fichiers vérifiés:
|
||||||
|
- `anime_sites/`: animesama.py, nekosama.py, animeultime.py, vostfree.py, base.py
|
||||||
|
- `video_players/`: doodstream.py, sibnet.py, vidmoly.py, sendvid.py, lpayer.py, unfichier.py, uptobox.py, rapidfile.py, base.py
|
||||||
|
|
||||||
|
### 2. ✅ Test des Imports Python
|
||||||
|
Tous les imports testés avec succès:
|
||||||
|
|
||||||
|
**Imports principaux:**
|
||||||
|
```python
|
||||||
|
from app.downloaders import (
|
||||||
|
get_downloader, BaseDownloader, GenericDownloader,
|
||||||
|
# Video players (8)
|
||||||
|
BaseVideoPlayer, DoodStreamDownloader, SibnetDownloader,
|
||||||
|
VidMolyDownloader, SendVidDownloader, LpayerDownloader,
|
||||||
|
UnFichierDownloader, UptoboxDownloader, RapidFileDownloader,
|
||||||
|
# Anime sites (4)
|
||||||
|
BaseAnimeSite, AnimeSamaDownloader, NekoSamaDownloader,
|
||||||
|
AnimeUltimeDownloader, VostfreeDownloader
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Imports factories:**
|
||||||
|
```python
|
||||||
|
from app.downloaders.video_players import get_video_player
|
||||||
|
from app.downloaders.anime_sites import get_anime_site
|
||||||
|
```
|
||||||
|
|
||||||
|
**Imports directs (modules individuels):**
|
||||||
|
```python
|
||||||
|
from app.downloaders.video_players.vidmoly import VidMolyDownloader
|
||||||
|
from app.downloaders.video_players.sendvid import SendVidDownloader
|
||||||
|
from app.downloaders.video_players.sibnet import SibnetDownloader
|
||||||
|
from app.downloaders.video_players.lpayer import LpayerDownloader
|
||||||
|
from app.downloaders.anime_sites.animesama import AnimeSamaDownloader
|
||||||
|
from app.downloaders.anime_sites.nekosama import NekoSamaDownloader
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. ✅ Test d'Instanciation et Typage
|
||||||
|
Toutes les classes s'instancient correctement:
|
||||||
|
- `VidMolyDownloader()` → instance de `BaseVideoPlayer` ✅
|
||||||
|
- `SendVidDownloader()` → instance de `BaseVideoPlayer` ✅
|
||||||
|
- `AnimeSamaDownloader()` → instance de `BaseAnimeSite` ✅
|
||||||
|
- `NekoSamaDownloader()` → instance de `BaseAnimeSite` ✅
|
||||||
|
|
||||||
|
### 4. ✅ Test des Imports Croisés
|
||||||
|
L'import croisé critique fonctionne:
|
||||||
|
```python
|
||||||
|
# Dans AnimeSamaDownloader._extract_from_vidmoly():
|
||||||
|
from ..video_players.vidmoly import VidMolyDownloader # ✅ CORRECT
|
||||||
|
```
|
||||||
|
|
||||||
|
Autres imports croisés dans AnimeSama:
|
||||||
|
- `from ..video_players.sendvid import SendVidDownloader` ✅
|
||||||
|
- `from ..video_players.sibnet import SibnetDownloader` ✅
|
||||||
|
- `from ..video_players.lpayer import LpayerDownloader` ✅
|
||||||
|
|
||||||
|
### 5. ✅ Tests Frontend
|
||||||
|
Tous les endpoints API fonctionnent:
|
||||||
|
|
||||||
|
| Endpoint | Status | Résultat |
|
||||||
|
|----------|--------|----------|
|
||||||
|
| `GET /web` | 200 | ✅ Page HTML chargée |
|
||||||
|
| `GET /api/providers` | 200 | ✅ 4 anime + 8 hosts |
|
||||||
|
| `POST /api/download` | 200 | ✅ Task créé |
|
||||||
|
| `GET /api/downloads` | 200 | ✅ Liste téléchargements |
|
||||||
|
|
||||||
|
### 6. ✅ Tests Pytest
|
||||||
|
```bash
|
||||||
|
pytest tests/test_downloaders.py -v
|
||||||
|
======================== 23 passed, 3 warnings in 1.56s ========================
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📊 Résultat Global
|
||||||
|
|
||||||
|
| Catégorie | Status | Détails |
|
||||||
|
|-----------|--------|---------|
|
||||||
|
| **Structure** | ✅ | 12 fichiers déplacés correctement |
|
||||||
|
| **Imports** | ✅ | Tous les imports fonctionnent |
|
||||||
|
| **Typage** | ✅ | Héritage correct (BaseVideoPlayer, BaseAnimeSite) |
|
||||||
|
| **Frontend** | ✅ | Tous les endpoints API opérationnels |
|
||||||
|
| **Tests** | ✅ | 23/23 tests passants |
|
||||||
|
| **Imports croisés** | ✅ | AnimeSama → VideoPlayers fonctionne |
|
||||||
|
|
||||||
|
## 🎯 Imports Corrigés
|
||||||
|
|
||||||
|
Fichier: `app/downloaders/anime_sites/animesama.py`
|
||||||
|
|
||||||
|
| Ligne | Avant | Après |
|
||||||
|
|-------|-------|-------|
|
||||||
|
| 195 | `from .vidmoly import` | `from ..video_players.vidmoly import` |
|
||||||
|
| 257 | `from .sendvid import` | `from ..video_players.sendvid import` |
|
||||||
|
| 304 | `from .sibnet import` | `from ..video_players.sibnet import` |
|
||||||
|
| 401 | `from .lpayer import` | `from ..video_players.lpayer import` |
|
||||||
|
|
||||||
|
## ✨ Conclusion
|
||||||
|
|
||||||
|
🎉 **Tous les imports sont corrects et fonctionnels!**
|
||||||
|
|
||||||
|
- Aucune erreur d'import détectée
|
||||||
|
- La structure est propre et maintenable
|
||||||
|
- Le frontend fonctionne parfaitement
|
||||||
|
- Tous les tests passent
|
||||||
|
- Les imports croisés (anime_sites → video_players) fonctionnent
|
||||||
|
|
||||||
|
**La restructuration est complète et 100% opérationnelle!**
|
||||||
|
|
||||||
|
---
|
||||||
|
**Vérifié par**: Claude Code
|
||||||
|
**Date**: 2026-01-24
|
||||||
|
**Statut**: ✅ Validé
|
||||||
@@ -1,96 +0,0 @@
|
|||||||
# Ohm Stream Downloader — Description fonctionnelle
|
|
||||||
|
|
||||||
> Cahier des charges pour une réécriture à partir de zéro.
|
|
||||||
|
|
||||||
## 🎯 Le but du projet
|
|
||||||
|
|
||||||
Une **application web auto-hébergée** (pensée pour un homelab, usage personnel) qui sert de **centre de contrôle unique** pour les contenus vidéo en français (animes, séries TV, VOSTFR) :
|
|
||||||
|
|
||||||
1. **Découvrir** du contenu en cherchant sur plusieurs sites/sources en même temps,
|
|
||||||
2. **Regarder** directement dans l'application via un lecteur intégré,
|
|
||||||
3. **Télécharger** des épisodes ou des saisons complètes,
|
|
||||||
4. **Ne rien rater** : suivre des titres et télécharger automatiquement chaque nouvel épisode dès sa sortie.
|
|
||||||
|
|
||||||
L'application agrège donc le rôle de moteur de recherche, de plateforme de streaming, de gestionnaire de téléchargements et d'outil d'automatisation.
|
|
||||||
|
|
||||||
## 👤 Parcours utilisateur type
|
|
||||||
|
|
||||||
1. L'utilisateur se connecte (ou s'inscrit).
|
|
||||||
2. Il lance une **recherche unifiée** : une seule requête interroge tous les sites d'animes et de séries activés, les résultats sont fusionnés.
|
|
||||||
3. Il consulte la **fiche détaillée** d'un titre : poster, bannière, synopsis, genres, note, année, nombre d'épisodes, liste des saisons et épisodes.
|
|
||||||
4. Depuis la fiche, il peut :
|
|
||||||
- **streamer** un épisode dans le lecteur intégré,
|
|
||||||
- **télécharger** un épisode précis ou une **saison entière** d'un coup,
|
|
||||||
- **l'ajouter à sa watchlist** pour un suivi automatique,
|
|
||||||
- **l'ajouter aux favoris**.
|
|
||||||
5. Il suit l'avancement des téléchargements en temps réel, puis **regarde les fichiers téléchargés** dans une bibliothèque locale (streaming depuis le serveur, reprise de lecture, épisode suivant).
|
|
||||||
|
|
||||||
## ⚙️ Fonctionnalités détaillées
|
|
||||||
|
|
||||||
### 1. Recherche multi-sources
|
|
||||||
- Sources « animes » : environ 5 sites francophones (Anime-Sama, Neko-Sama, Anime-Ultime, Vostfree, French-Manga).
|
|
||||||
- Sources « séries/films » : environ 2 sites (FS7/French-Stream, Zone-Téléchargement).
|
|
||||||
- Chaque source est un **module interchangeable** avec le même contrat : chercher, lister les épisodes, récupérer les métadonnées, extraire le lien vidéo réel.
|
|
||||||
- Résolution automatique : pour une URL donnée, le système trouve tout seul le bon module (site → hébergeur vidéo → générique).
|
|
||||||
- **Extraction en 2 niveaux** : la page de l'épisode contient des lecteurs embarqués hébergés ailleurs (une quinzaine d'hébergeurs supportés : DoodStream, VidMoly, 1fichier, Uptobox, Sibnet, Uqload, SendVid, etc.). Le système sait résoudre l'URL directe du fichier pour chacun d'eux.
|
|
||||||
|
|
||||||
### 2. Métadonnées enrichies
|
|
||||||
- Les infos de base viennent du site scrappé, puis sont **enrichies/complétées** via une base externe de données anime (Kitsu, en alternative à MAL) : synopsis, genres, notes, année, images.
|
|
||||||
- Fusion intelligente : on ne remplit que les champs manquants, avec normalisation des formats entre sources.
|
|
||||||
- **Cache des métadonnées** pour limiter les appels externes.
|
|
||||||
|
|
||||||
### 3. Streaming et lecteur
|
|
||||||
- Lecteur vidéo intégré pour les liens extraits des hébergeurs (avec contournement des protections courantes de ces hébergeurs).
|
|
||||||
- **Bibliothèque locale** : tous les fichiers téléchargés sont streamables depuis le serveur (page « watch » avec reprise de lecture, navigation épisode suivant, streaming avec support des requêtes de plage pour l'avance rapide).
|
|
||||||
|
|
||||||
### 4. Gestionnaire de téléchargements
|
|
||||||
- File d'attente avec **téléchargements parallèles limités** (~3–5 simultanés).
|
|
||||||
- Statuts : en attente, en cours, en pause, terminé, échec, annulé.
|
|
||||||
- **Pause / reprise** (via requêtes de plage HTTP), **retry**, **annulation**, « tout annuler », nettoyage des tâches finies.
|
|
||||||
- **Progression temps réel** : pourcentage, vitesse, temps restant — rafraîchie dynamiquement côté interface.
|
|
||||||
- **Anti-doublons** : si un téléchargement de la même source est déjà actif/en attente, on retourne la tâche existante au lieu d'en créer une.
|
|
||||||
- **Persistance** : au démarrage, le dossier de téléchargements est scanné pour **restaurer** les tâches terminées.
|
|
||||||
- Noms de fichiers **nettoyés et sécurisés** automatiquement (suppression de caractères interdits, protection contre la traversée de répertoires).
|
|
||||||
- Les URLs circulent au format interne `video_url|page_url|titre` pour ne pas perdre le contexte entre l'extraction et le téléchargement.
|
|
||||||
|
|
||||||
### 5. Watchlist + téléchargement automatique (le cœur de l'app)
|
|
||||||
- L'utilisateur suit un titre ; chaque item a un **statut** : actif, en pause, terminé, archivé.
|
|
||||||
- Mémorisation du **dernier épisode téléchargé** et du dernier épisode disponible.
|
|
||||||
- Un **planificateur interne** vérifie périodiquement (intervalle configurable, d'1 h à 168 h) tous les titres « dus » :
|
|
||||||
- détecte les nouveaux épisodes par comparaison au dernier connu,
|
|
||||||
- les **télécharge automatiquement**,
|
|
||||||
- journalise un rapport (nouveaux épisodes trouvés / téléchargés).
|
|
||||||
- Réglages globaux de la watchlist (activation de l'auto-download, intervalle, etc.), statistiques de suivi.
|
|
||||||
- Vérification manuelle possible à la demande (« vérifier maintenant »).
|
|
||||||
|
|
||||||
### 6. Intégration Sonarr (homelab)
|
|
||||||
- Réception de **webhooks Sonarr** (événements type « épisode importé/téléchargé ») avec **vérification d'authenticité** (signature HMAC + secret) optionnelle.
|
|
||||||
- **Mapping entre séries Sonarr et sources de scraping** : pour chaque série suivie dans Sonarr, on associe le titre/URL correspondant sur un site d'animes, pour que Sonarr déclenche le téléchargement dans la bonne source.
|
|
||||||
- Endpoints d'aide : recherche Sonarr, recherche d'épisodes, **suggestions automatiques de mapping**, configuration (langue, qualité, provider par défaut, activation du webhook, journalisation).
|
|
||||||
|
|
||||||
### 7. Favoris
|
|
||||||
- Sauvegarde de titres aimés avec leurs métadonnées et images, avec statistiques et pagination/navigation.
|
|
||||||
|
|
||||||
### 8. Recommandations et découvertes
|
|
||||||
- **Analyse de l'historique de téléchargement** (parsing des noms de fichiers pour extraire titres, groupes, saisons, qualité) → profil de goûts (genres, titres) → **suggestions personnalisées**.
|
|
||||||
- Sections découverte : **dernières sorties**, **sorties saisonnières**, **calendrier des sorties**, **top** actuel.
|
|
||||||
|
|
||||||
### 9. Comptes et administration
|
|
||||||
- Inscription/connexion/déconnexion avec **sessions par tokens** (token court + refresh token longue durée).
|
|
||||||
- Rôles **admin / utilisateur**, activation/désactivation de comptes.
|
|
||||||
- Interface d'**administration** : liste des utilisateurs, stats, bascule admin/actif, suppression.
|
|
||||||
|
|
||||||
### 10. Paramètres et gestion des sources
|
|
||||||
- **Activation/désactivation de chaque source** individuellement (sans redémarrage), avec état de **santé** vérifiable (test manuel + contrôle automatique périodique par le planificateur).
|
|
||||||
- Réglages de l'interface utilisateur, configuration persistée.
|
|
||||||
- Endpoints publics de santé de l'application et liste des sources disponibles.
|
|
||||||
|
|
||||||
## 🧩 Principes de conception pour la réécriture
|
|
||||||
|
|
||||||
- **Architecture à 3 niveaux de scrapers** : sites de catalogues (animes / séries) → hébergeurs vidéo → extraction générique. Chaque niveau a une classe de base et un registre ; ajouter une nouvelle source = écrire un module qui implémente le contrat et le déclarer dans le registre. *Point le plus important : les sites changent souvent, il faut pouvoir en ajouter un en une heure.*
|
|
||||||
- **Configuration de scraping externalisée** : piloter au moins certaines sources par fichier de configuration (sélecteurs), pour réparer un site cassé sans toucher au code.
|
|
||||||
- **Un seul format interne** pour transporter le contexte (`vidéo|page|titre`) du scraping jusqu'au téléchargement.
|
|
||||||
- **Sécurité non négociable** : nettoyage systématique des noms de fichiers, secrets hors fichiers de config, tokens signés.
|
|
||||||
- **Une seule source de vérité** pour les données (pas de double stockage JSON + base).
|
|
||||||
- **Pas d'erreurs silencieuses** : chaque échec de scraping/téléchargement est journalisé et exposé (statut d'échec visible dans l'UI).
|
|
||||||
- **Injection de dépendances** entre les composants (vérificateur d'épisodes ↔ gestionnaire de téléchargements) pour éviter les dépendances circulaires.
|
|
||||||
@@ -1,220 +1,408 @@
|
|||||||
# ⛩ Ohm Stream Downloader
|
# Ohm Stream Downloader
|
||||||
|
|
||||||
Application web **auto-hébergée** (homelab) : centre de contrôle unique pour découvrir,
|
**Application web complète pour télécharger des animes et fichiers depuis divers hébergeurs.**
|
||||||
regarder et télécharger des animes et séries VOSTFR/VF.
|
|
||||||
|
|
||||||
## Déploiement (Docker) — recommandé
|
Interface moderne avec recherche d'anime, métadonnées enrichies, téléchargements parallèles et streaming vidéo.
|
||||||
|
|
||||||
Le déploiement officiel passe par Docker Compose : l'image (ffmpeg inclus) est
|
## ✨ Fonctionnalités
|
||||||
hébergée sur le **registre privé du Gitea** — rien n'est publié publiquement.
|
|
||||||
|
|
||||||
### Installation guidée (recommandée)
|
### 🎬 Recherche et Téléchargement d'Animes
|
||||||
|
- **Recherche unifiée** : Recherchez sur 4 providers simultanément (Anime-Sama, Neko-Sama, Anime-Ultime, Vostfree)
|
||||||
|
- **Métadonnées riches** : Synopsis, genres, notes, année de sortie, studio, nombre d'épisodes, statut
|
||||||
|
- **Téléchargement par épisode** : Sélectionnez et téléchargez des épisodes individuels
|
||||||
|
- **Téléchargement de saison complète** : Téléchargez tous les épisodes d'un coup
|
||||||
|
- **Streaming vidéo** : Regardez vos animes directement dans le navigateur
|
||||||
|
- **Recherche floue** : Gestion des fautes de frappe et variations de noms
|
||||||
|
|
||||||
|
### 📁 Hébergeurs de Fichiers Supportés
|
||||||
|
- **1fichier** (1fichier.com, 1fichier.fr)
|
||||||
|
- **Uptobox** (uptobox.com, uptobox.fr)
|
||||||
|
- **Doodstream** (doodstream.com, dood.to, dood.lol, etc.)
|
||||||
|
- **Rapidfile** (rapidfile.net, rapidfile.com)
|
||||||
|
|
||||||
|
### 🎥 Hébergeurs Vidéo Supportés
|
||||||
|
- **VidMoly** (vidmoly.to, vidmoly.com)
|
||||||
|
- **SendVid** (sendvid.com)
|
||||||
|
|
||||||
|
### 🚀 Gestion des Téléchargements
|
||||||
|
- **Téléchargements parallèles** : Jusqu'à 3 téléchargements simultanés
|
||||||
|
- **Pause/Reprise** : Contrôle total sur vos téléchargements
|
||||||
|
- **Progression en temps réel** : Vitesse, progression, taille
|
||||||
|
- **Reprise automatique** : Support des HTTP Range pour reprendre les téléchargements interrompus
|
||||||
|
|
||||||
|
### 🌐 Interface Web
|
||||||
|
- **Design moderne** : Interface sombre avec gradients et animations
|
||||||
|
- **Responsive** : Fonctionne sur desktop et mobile
|
||||||
|
- **Mise à jour automatique** : Rafraîchissement chaque seconde
|
||||||
|
- **Métadonnées visuelles** : Affichage des informations anime avec icônes
|
||||||
|
|
||||||
|
### 🔌 API REST
|
||||||
|
- **Endpoints REST** : Intégration facile avec d'autres applications
|
||||||
|
- **Documentation automatique** : Swagger UI disponible
|
||||||
|
|
||||||
|
## 📋 Configuration Requise
|
||||||
|
|
||||||
|
- Python 3.8+
|
||||||
|
- pip
|
||||||
|
|
||||||
|
## 🚀 Installation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://git.lanro.eu/Roman/ohm_streaming.git && cd ohm_streaming
|
# Cloner le repository
|
||||||
./scripts/install.sh
|
git clone https://github.com/votre-user/Ohm_streaming.git
|
||||||
|
cd Ohm_streaming
|
||||||
|
|
||||||
|
# Créer l'environnement virtuel
|
||||||
|
python3 -m venv venv
|
||||||
|
source venv/bin/activate # Windows: venv\Scripts\activate
|
||||||
|
|
||||||
|
# Installer les dépendances
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
# Lancer le serveur de développement
|
||||||
|
uvicorn main:app --reload --host 0.0.0.0 --port 3000
|
||||||
```
|
```
|
||||||
|
|
||||||
Le script vérifie Docker, demande la **destination des épisodes** (dossier dédié
|
Accédez à l'interface : http://localhost:3000/web
|
||||||
recommandé — voir « Bibliothèque Plex/Sonarr » ci-dessous), le port, génère les
|
|
||||||
secrets, branche le montage et démarre. Non interactif aussi :
|
|
||||||
`./scripts/install.sh --dir /srv/animes --port 8777 --skip-login`.
|
|
||||||
|
|
||||||
### À la main
|
## 📖 Utilisation
|
||||||
|
|
||||||
|
### Interface Web
|
||||||
|
|
||||||
|
1. **Onglet Recherche d'Anime** :
|
||||||
|
- Entrez le nom d'un anime (ex: "Naruto", "One Piece")
|
||||||
|
- Sélectionnez la langue (VOSTFR ou VF)
|
||||||
|
- Cochez "Inclure les métadonnées" pour plus d'informations
|
||||||
|
- Cliquez sur "Rechercher"
|
||||||
|
- Sélectionnez un épisode et cliquez sur "Télécharger"
|
||||||
|
- Ou utilisez "Toute la saison" pour tout télécharger
|
||||||
|
|
||||||
|
2. **Onglet Lien Direct** :
|
||||||
|
- Collez un lien de téléchargement direct
|
||||||
|
- Cliquez sur "Télécharger"
|
||||||
|
|
||||||
|
3. **Onglet Providers** :
|
||||||
|
- Utilisez les onglets spécifiques à chaque provider
|
||||||
|
- Chaque onglet a ses propres options de recherche
|
||||||
|
|
||||||
|
### API Endpoints
|
||||||
|
|
||||||
|
#### Téléchargements
|
||||||
|
|
||||||
|
| Méthode | Endpoint | Description |
|
||||||
|
|---------|----------|-------------|
|
||||||
|
| POST | `/api/download` | Créer un nouveau téléchargement |
|
||||||
|
| GET | `/api/downloads` | Lister tous les téléchargements |
|
||||||
|
| GET | `/api/download/{task_id}` | Statut d'un téléchargement |
|
||||||
|
| POST | `/api/download/{task_id}/pause` | Mettre en pause |
|
||||||
|
| POST | `/api/download/{task_id}/resume` | Reprendre |
|
||||||
|
| DELETE | `/api/download/{task_id}` | Annuler/Supprimer |
|
||||||
|
| GET | `/api/download/{task_id}/file` | Télécharger le fichier terminé |
|
||||||
|
|
||||||
|
#### Anime
|
||||||
|
|
||||||
|
| Méthode | Endpoint | Description |
|
||||||
|
|---------|----------|-------------|
|
||||||
|
| GET | `/api/anime/search` | Rechercher un anime (paramètres: `q`, `lang`, `include_metadata`) |
|
||||||
|
| GET | `/api/anime/metadata` | Obtenir les métadonnées d'un anime (paramètre: `url`) |
|
||||||
|
| GET | `/api/anime/episodes` | Liste des épisodes d'un anime (paramètres: `url`, `lang`) |
|
||||||
|
| POST | `/api/anime/download` | Télécharger un épisode |
|
||||||
|
| POST | `/api/anime/download-season` | Télécharger toute une saison |
|
||||||
|
|
||||||
|
#### Streaming Vidéo
|
||||||
|
|
||||||
|
| Méthode | Endpoint | Description |
|
||||||
|
|---------|----------|-------------|
|
||||||
|
| GET | `/video/{task_id}` | Stream une vidéo (support Range/seeking) |
|
||||||
|
| GET | `/stream/{filename}` | Stream par nom de fichier |
|
||||||
|
| GET | `/player/{task_id}` | Lecteur vidéo pour un téléchargement |
|
||||||
|
| GET | `/watch/{filename}` | Lecteur vidéo par nom de fichier |
|
||||||
|
|
||||||
|
#### Système
|
||||||
|
|
||||||
|
| Méthode | Endpoint | Description |
|
||||||
|
|---------|----------|-------------|
|
||||||
|
| GET | `/` | Informations sur l'API |
|
||||||
|
| GET | `/api/providers` | Liste des providers supportés |
|
||||||
|
| GET | `/health` | Vérifier l'état du serveur |
|
||||||
|
| GET | `/web` | Interface web |
|
||||||
|
|
||||||
|
### Exemples API
|
||||||
|
|
||||||
|
**Rechercher un anime avec métadonnées :**
|
||||||
```bash
|
```bash
|
||||||
# 1. Récupérer le projet puis se connecter au registre privé (compte Gitea avec accès lecture)
|
curl "http://localhost:3000/api/anime/search?q=naruto&lang=vostfr&include_metadata=true"
|
||||||
git clone https://git.lanro.eu/Roman/ohm_streaming.git && cd ohm_streaming
|
|
||||||
docker login git.lanro.eu
|
|
||||||
|
|
||||||
# 2. Configuration locale
|
|
||||||
cp .env.example .env
|
|
||||||
# → OHM_SECRET_KEY (openssl rand -hex 32) et WATCHTOWER_TOKEN (openssl rand -hex 24) obligatoires
|
|
||||||
|
|
||||||
# 3. Démarrage
|
|
||||||
docker compose up -d
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Puis ouvrir http://localhost:8777 — le **premier compte créé est administrateur**.
|
**Obtenir les épisodes d'un anime :**
|
||||||
Les données (`data/`, `downloads/`) sont montées en volumes : elles survivent aux
|
|
||||||
|
|
||||||
|
|
||||||
### Bibliothèque Plex / Sonarr existante
|
|
||||||
|
|
||||||
Ohm peut déposer ses épisodes directement dans ton serveur de média :
|
|
||||||
|
|
||||||
1. **Crée un dossier dédié** (recommandé : hors de la racine Sonarr, ex.
|
|
||||||
`/plex_videos/ohm`) et monte-le à la place de `./downloads` :
|
|
||||||
`- /plex_videos/ohm:/downloads` — c'est ce que fait `scripts/install.sh`.
|
|
||||||
2. **Ajoute ce dossier à Plex** comme dossier d'une bibliothèque (ou comme
|
|
||||||
dossier supplémentaire de ta bibliothèque animes). Les épisodes arrivent
|
|
||||||
rangés par animé : `One Piece/One Piece - E1010 (VOSTFR).mp4` — le nom du
|
|
||||||
dossier sert de série, le fichier d'épisode.
|
|
||||||
3. **Sonarr n'est pas touché** : l'entrypoint du conteneur ne prend possession
|
|
||||||
(`chown`) que d'un dossier de téléchargements **vide** ; une bibliothèque
|
|
||||||
existante et ses droits restent intacts. Pour piloter Ohm depuis Sonarr ou
|
|
||||||
Prowlarr, voir l'API Torznab plus bas.
|
|
||||||
|
|
||||||
Si tu pointes `/downloads` sur un dossier **non vide**, Ohm adopte les fichiers
|
|
||||||
présents (ils apparaissent dans sa bibliothèque interne) et conserve les droits
|
|
||||||
existants — assure-toi juste que l'uid 1000 du conteneur peut y écrire.
|
|
||||||
|
|
||||||
### Mettre à jour
|
|
||||||
|
|
||||||
**Depuis l'interface** (déploiement Docker) : page **Admin → Mise à jour** —
|
|
||||||
configurer une fois le dépôt Gitea + un jeton d'accès (droit lecture), puis
|
|
||||||
« Vérifier » et « ⬆ Mettre à jour maintenant ». Watchtower tire la nouvelle
|
|
||||||
image et recrée le conteneur : quelques secondes d'indisponibilité, les pages
|
|
||||||
ouvertes se reconnectent et rechargent automatiquement.
|
|
||||||
|
|
||||||
**En ligne de commande** (toujours possible) :
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker compose pull && docker compose up -d
|
curl "http://localhost:3000/api/anime/episodes?url=https://anime-sama.si/catalogue/naruto/saison1/vostfr/&lang=vostfr"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Publier une version (mainteneur)
|
**Télécharger une saison complète :**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./scripts/release.sh 0.2.0
|
curl -X POST "http://localhost:3000/api/anime/download-season?url=https://anime-sama.si/catalogue/naruto/saison1/vostfr/&lang=vostfr"
|
||||||
```
|
```
|
||||||
|
|
||||||
Bump de version, commit + tag git, build de l'image et push vers
|
**Créer un téléchargement direct :**
|
||||||
`git.lanro.eu/roman/ohm_streaming` (tags `0.2.0` et `latest`).
|
|
||||||
|
|
||||||
## Démarrage rapide (développement)
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv sync
|
curl -X POST http://localhost:3000/api/download \
|
||||||
# ffmpeg requis pour les flux HLS (binaire statique dans ~/.local/bin, ou apt install ffmpeg)
|
-H "Content-Type: application/json" \
|
||||||
uv run uvicorn app.main:app --host 0.0.0.0 --port 8777
|
-d '{"url": "https://1fichier.com/?xxxxx"}'
|
||||||
```
|
```
|
||||||
|
|
||||||
Serveur persistant en dev : préférer Docker (voir plus haut) ; sinon
|
## 🏗️ Structure du Projet
|
||||||
`tmux new-session -d -s ohm 'uv run uvicorn app.main:app --host 0.0.0.0 --port 8777'`.
|
|
||||||
|
|
||||||
Puis ouvrir http://localhost:8777 — le **premier compte créé est administrateur**.
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Variables d'environnement (préfixe `OHM_`, voir `.env.example`) :
|
|
||||||
|
|
||||||
| Variable | Défaut | Rôle |
|
|
||||||
|---|---|---|
|
|
||||||
| `OHM_SECRET_KEY` | `change-me-in-production` | **À changer** — signature des tokens |
|
|
||||||
| `OHM_DOWNLOAD_DIR` | `./downloads` | Dossier des fichiers téléchargés |
|
|
||||||
| `OHM_DATABASE_PATH` | `./data/ohm.db` | Base SQLite |
|
|
||||||
| `OHM_MAX_PARALLEL_DOWNLOADS` | `3` | Téléchargements simultanés |
|
|
||||||
| `OHM_WATCHTOWER_URL` | *(vide)* | URL Watchtower pour la mise à jour (réglé par docker-compose) |
|
|
||||||
| `OHM_WATCHTOWER_TOKEN` | *(vide)* | Jeton partagé Watchtower (réglé par docker-compose) |
|
|
||||||
| `OHM_VERSION` | *(pyproject)* | Version affichée — cuite dans l'image Docker au build |
|
|
||||||
|
|
||||||
## Fonctionnalités
|
|
||||||
|
|
||||||
- **Recherche unifiée** sur plusieurs sources (Vostfree, French-Manga, VoirAnime pour les
|
|
||||||
animes ; French-Stream pour les séries et films VF/VOSTFR) — chaque source
|
|
||||||
est un module interchangeable activable/désactivable à chaud (page Admin), dont l'URL
|
|
||||||
est modifiable à la volée (utile si un site change de domaine).
|
|
||||||
- **Extraction en 2 niveaux** : page d'épisode → lecteurs embarqués → URL directe
|
|
||||||
(Sibnet, SendVid, VidMoly, Uqload, Vidzy, Luluvdo ; VoirAnime résout via son
|
|
||||||
endpoint « prepare » : MP4 direct ou HLS relayé par le proxy du site).
|
|
||||||
- **Préférence de contenus par compte** : sélecteur topbar ⛨ Animés / 📺 Séries /
|
|
||||||
✨ Les deux — la recherche et les nouveautés filtrent par type de média, et les
|
|
||||||
sources hors périmètre ne sont même pas interrogées.
|
|
||||||
- **Proxy vidéo intégré** (`/api/proxy`) : contourne les protections (tokens liés à l'IP, Referer/UA obligatoires), réécrit les playlists HLS.
|
|
||||||
- **Téléchargements** : file parallèle, pause/reprise (HTTP Range), retry, anti-doublons,
|
|
||||||
progression en temps réel (SSE), persistance au redémarrage.
|
|
||||||
- **Bibliothèque locale** : streaming avec range requests, reprise de lecture,
|
|
||||||
navigation épisode suivant/précédent.
|
|
||||||
- **Comptes** : JWT court + refresh token (rotation), rôles admin/utilisateur,
|
|
||||||
administration des comptes.
|
|
||||||
- **Découverte** (`/discover`) : 🎭 exploration par genre (animés via Kitsu, séries/films
|
|
||||||
via French-Stream — état dans l'URL, partageable), 🆕 nouveautés en rails séparés
|
|
||||||
(animés triés par date de sortie via Kitsu ; séries & films de French-Stream), 🔥
|
|
||||||
incontournables animés (top popularité Kitsu) et ✨ recommandations par genres déduites
|
|
||||||
des téléchargements et favoris, titres déjà possédés exclus (cache mémoire). Sans
|
|
||||||
Sonarr, les genres des séries/films téléchargés sont lus sur leur fiche source ;
|
|
||||||
sans aucun historique, une carte d'amorçage invite à télécharger un premier titre.
|
|
||||||
|
|
||||||
|
|
||||||
## Intégration Sonarr / Prowlarr (*arr)
|
|
||||||
|
|
||||||
OhmStreaming expose une **API Torznab** (indexeur) **et une API compatible
|
|
||||||
qBittorrent** (client de téléchargement) : Sonarr peut lui déléguer toute la
|
|
||||||
chaîne — recherche, téléchargement, puis **import et renommage automatiques**
|
|
||||||
dans la bibliothèque Sonarr.
|
|
||||||
|
|
||||||
### 1. OhmStreaming comme indexeur
|
|
||||||
|
|
||||||
Dans **Admin → Intégrations Sonarr / Prowlarr**, copier :
|
|
||||||
|
|
||||||
| Champ | Valeur |
|
|
||||||
|---|---|
|
|
||||||
| URL | `http://<hote-ohm>:8777/torznab/api` |
|
|
||||||
| Clé API | générée automatiquement (bouton Régénérer pour la changer) |
|
|
||||||
|
|
||||||
- **Prowlarr** : Indexers → Add Indexer → *Generic Torznab* → coller URL + clé,
|
|
||||||
puis synchroniser vers Sonarr.
|
|
||||||
- **Sonarr** (direct) : Settings → Indexers → Add → *Torznab* → coller URL + clé.
|
|
||||||
Catégories : TV (5000) / Anime (5070).
|
|
||||||
|
|
||||||
Endpoints : `t=caps`, `t=tvsearch` (q, season, ep), `t=search` — auth par
|
|
||||||
`?apikey=` ou en-tête `X-Api-Key`.
|
|
||||||
|
|
||||||
### 2. OhmStreaming comme client de téléchargement (recommandé)
|
|
||||||
|
|
||||||
Dans Sonarr : **Settings → Download Clients → Add → qBittorrent** :
|
|
||||||
|
|
||||||
| Champ | Valeur |
|
|
||||||
|---|---|
|
|
||||||
| Host | `http://<hote-ohm>:8777` |
|
|
||||||
| Username | `ohm` |
|
|
||||||
| Password | la clé API Torznab (Admin → Intégrations) |
|
|
||||||
|
|
||||||
Le flux complet devient : Sonarr grab → Ohm télécharge (progression visible
|
|
||||||
dans la file Sonarr) → Sonarr **importe, renomme et range** l'épisode dans sa
|
|
||||||
bibliothèque selon ses propres règles → le « torrent » est retiré de la file
|
|
||||||
Ohm (fichier inclu si « Remove Completed » est coché).
|
|
||||||
|
|
||||||
**Chemin d'accès** : Ohm annonce les fichiers sous `/downloads/<Animé>/…`
|
|
||||||
(chemin conteneur). Si Sonarr tourne dans Docker sans ce montage, ajouter un
|
|
||||||
*Remote Path Mapping* : hôte = `<hote-ohm>`, distant = `/downloads`, local =
|
|
||||||
le dossier hôte monté (ex. `/plex_videos/ohm`).
|
|
||||||
|
|
||||||
Variante minimale sans import : client « Torrent Blackhole » — Sonarr pose le
|
|
||||||
`.torrent` de service et l'épisode reste dans la bibliothèque OhmStreaming
|
|
||||||
seulement (pas d'import/renommage Sonarr).
|
|
||||||
|
|
||||||
### 2. « Pour toi » personnalisé par Sonarr
|
|
||||||
|
|
||||||
Toujours dans **Admin → Intégrations**, renseigner l'URL Sonarr
|
|
||||||
(`http://sonarr:8989`) et sa clé API (*Settings → General → API Key*), puis
|
|
||||||
« Enregistrer » et « Tester ». Les genres des séries téléchargées/grabées sur
|
|
||||||
Sonarr alimentent la section ✨ **Pour toi** (et ce qui y est possédé n'est pas
|
|
||||||
re-recommandé). Sonarr absent ou KO → dégradation gracieuse, rien ne casse.
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
```
|
```
|
||||||
app/
|
Ohm_streaming/
|
||||||
├── main.py # FastAPI + lifespan (DB, download manager)
|
├── main.py # Application FastAPI & endpoints API
|
||||||
├── config.py # pydantic-settings (OHM_*)
|
├── app/
|
||||||
├── db.py # SQLite (aiosqlite) — source de vérité unique
|
│ ├── models/ # Modèles Pydantic
|
||||||
├── routers/ # auth, search, discover, downloads, library, admin, torznab, pages
|
│ │ └── __init__.py # DownloadTask, AnimeMetadata, etc.
|
||||||
├── scrapers/
|
│ ├── downloaders/ # Downloaders par provider
|
||||||
│ ├── base.py # contrats SourceScraper/HosterExtractor + registres
|
│ │ ├── base.py # Classe BaseDownloader
|
||||||
│ ├── configs/ # sélecteurs YAML externalisés (réparer sans coder)
|
│ │ ├── animesama.py # Anime-Sama (avec métadonnées)
|
||||||
│ ├── sources/ # vostfree, french_manga, voiranime, french_stream
|
│ │ ├── animeultime.py # Anime-Ultime (avec métadonnées)
|
||||||
├── services/ # downloads, kitsu, discover, sonarr, torznab, settings
|
│ │ ├── nekosama.py # Neko-Sama (avec métadonnées)
|
||||||
└── templates/ + static/ # UI htmx + Alpine.js, thème sombre
|
│ │ ├── vostfree.py # Vostfree (avec métadonnées)
|
||||||
|
│ │ ├── unfichier.py # 1fichier
|
||||||
|
│ │ ├── uptobox.py # Uptobox
|
||||||
|
│ │ ├── doodstream.py # Doodstream
|
||||||
|
│ │ ├── rapidfile.py # Rapidfile
|
||||||
|
│ │ ├── vidmoly.py # VidMoly
|
||||||
|
│ │ ├── sendvid.py # SendVid
|
||||||
|
│ │ └── __init__.py # Registry des downloaders
|
||||||
|
│ ├── providers.py # Configuration des providers
|
||||||
|
│ └── download_manager.py # Gestionnaire de file d'attente
|
||||||
|
├── downloads/ # Fichiers téléchargés
|
||||||
|
├── templates/
|
||||||
|
│ ├── index.html # Interface web principale
|
||||||
|
│ └── player.html # Lecteur vidéo
|
||||||
|
├── static/ # Fichiers statiques (CSS, JS, images)
|
||||||
|
└── requirements.txt # Dépendances Python
|
||||||
```
|
```
|
||||||
|
|
||||||
**Ajouter une source** : créer `app/scrapers/sources/ma_source.py` qui implémente
|
## ⚙️ Configuration
|
||||||
`SourceScraper`, la décorer `@register_source` — c'est tout (registre auto-découvert).
|
|
||||||
|
|
||||||
## Tests
|
Modifiez ces paramètres dans `main.py` :
|
||||||
|
|
||||||
```bash
|
```python
|
||||||
uv run pytest # 132 tests
|
download_manager = DownloadManager(
|
||||||
uv run ruff check . # lint
|
download_dir="downloads", # Répertoire de stockage
|
||||||
|
max_parallel=3 # Téléchargements simultanés
|
||||||
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## 🔧 Ajouter un Provider
|
||||||
|
|
||||||
|
### Ajouter un Hébergeur de Fichiers
|
||||||
|
|
||||||
|
1. Créez `app/downloaders/myhost.py` :
|
||||||
|
```python
|
||||||
|
from .base import BaseDownloader
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
class MyHostDownloader(BaseDownloader):
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return "myhost.com" in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str) -> tuple[str, str]:
|
||||||
|
# Extraire le lien de téléchargement direct
|
||||||
|
response = await self.client.get(url)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
# ... logique d'extraction ...
|
||||||
|
return download_url, filename
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Ajoutez-le dans `app/providers.py` :
|
||||||
|
```python
|
||||||
|
FILE_HOSTS = {
|
||||||
|
# ...
|
||||||
|
"myhost": {
|
||||||
|
"name": "MyHost",
|
||||||
|
"domains": ["myhost.com"],
|
||||||
|
"icon": "📁",
|
||||||
|
"color": "#4ecdc4"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Ajouter un Provider Anime avec Métadonnées
|
||||||
|
|
||||||
|
1. Créez le downloader avec les méthodes requises :
|
||||||
|
```python
|
||||||
|
class MyAnimeDownloader(BaseDownloader):
|
||||||
|
async def search_anime(self, query: str, lang: str = "vostfr", include_metadata: bool = False):
|
||||||
|
# Implémenter la recherche
|
||||||
|
|
||||||
|
async def get_anime_metadata(self, anime_url: str) -> dict:
|
||||||
|
# Extraire: synopsis, genres, rating, release_year, studio, etc.
|
||||||
|
return {
|
||||||
|
'synopsis': '...',
|
||||||
|
'genres': ['Action', 'Aventure'],
|
||||||
|
'rating': '8.5/10',
|
||||||
|
'release_year': 2023,
|
||||||
|
'studio': 'Studio Name',
|
||||||
|
# ...
|
||||||
|
}
|
||||||
|
|
||||||
|
async def get_episodes(self, anime_url: str, lang: str = "vostfr"):
|
||||||
|
# Retourner la liste des épisodes
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Enregistrez-le dans `app/providers.py` et `main.py`
|
||||||
|
|
||||||
|
## 🗺️ Roadmap / Plans Futurs
|
||||||
|
|
||||||
|
### Version 2.2 - Système de Favoris ✅ (Terminé)
|
||||||
|
- [x] **Favoris** : Sauvegarder les animes favoris avec métadonnées complètes
|
||||||
|
- [x] **API REST complète** : 6 endpoints pour gérer les favoris
|
||||||
|
- [x] **Tri et filtrage** : Par titre, rating, année, provider, genre
|
||||||
|
- [x] **Statistiques** : Distribution par provider et genre
|
||||||
|
- [x] **Stockage persistant** : Base JSON (favorites.json)
|
||||||
|
|
||||||
|
### Version 2.3 - Base de Données & Authentification
|
||||||
|
- [ ] **SQLite avec SQLAlchemy** : Persistance complète des données
|
||||||
|
- [ ] **Système d'authentification local** :
|
||||||
|
- [ ] Inscription et connexion utilisateur
|
||||||
|
- [ ] Tokens JWT avec expiration (7 jours)
|
||||||
|
- [ ] Hachage de mot de passe bcrypt
|
||||||
|
- [ ] Préférences utilisateur personnalisables
|
||||||
|
- [ ] **Profils utilisateurs** :
|
||||||
|
- [ ] Table User : username, email, preferences, admin
|
||||||
|
- [ ] Historique de téléchargement par utilisateur
|
||||||
|
- [ ] Historique de visionnage (position, progression)
|
||||||
|
- [ ] Préférences : langue par défaut, thème, auto-download
|
||||||
|
- [ ] **Rétrocompatibilité** : Accès anonyme toujours possible
|
||||||
|
|
||||||
|
**Nouveaux endpoints :**
|
||||||
|
- `POST /api/auth/register` - Inscription
|
||||||
|
- `POST /api/auth/login` - Connexion (JWT)
|
||||||
|
- `GET /api/auth/me` - Profil utilisateur
|
||||||
|
- `PUT /api/auth/me/preferences` - Préférences
|
||||||
|
- `GET /api/auth/me/download-history` - Historique
|
||||||
|
- `GET /api/auth/me/watch-history` - Visionnage
|
||||||
|
|
||||||
|
### Version 2.4 - APIs Externes & Recommandations
|
||||||
|
- [ ] **Intégration Jikan API** (MyAnimeList) :
|
||||||
|
- [ ] Métadonnées enrichies (poster, notes, genres)
|
||||||
|
- [ ] Limitation de débit : 3 req/sec
|
||||||
|
- [ ] **Intégration AniList API** (GraphQL) :
|
||||||
|
- [ ] Recommandations basées sur l'historique
|
||||||
|
- [ ] Limitation de débit : 90 req/min
|
||||||
|
- [ ] **Système de cache** :
|
||||||
|
- [ ] Cache API dans la base de données
|
||||||
|
- [ ] TTL configurable (168h par défaut)
|
||||||
|
- [ ] Mécanisme de fallback (AniList → Jikan)
|
||||||
|
- [ ] **Enrichissement automatique** :
|
||||||
|
- [ ] Fusion des données providers + API externes
|
||||||
|
- [ [ ] Affichage des posters dans les résultats
|
||||||
|
|
||||||
|
**Nouveaux endpoints :**
|
||||||
|
- `GET /api/anime/metadata?enrich=true` - Métadonnées enrichies
|
||||||
|
- `GET /api/recommendations` - Suggestions personnalisées
|
||||||
|
|
||||||
|
### Version 2.5 - Webhooks & Automatisation ✅ (Terminé)
|
||||||
|
- [x] **Support Sonarr Webhook** :
|
||||||
|
- [x] `POST /api/webhook/sonarr` - Réception événements
|
||||||
|
- [x] Auto-téléchargement des nouveaux épisodes
|
||||||
|
- [x] Vérification HMAC SHA256 (optionnel)
|
||||||
|
- [x] Gestion des événements : Download, Rename, Delete
|
||||||
|
- [x] **Automatisations** :
|
||||||
|
- [x] Déclenchement automatique sur nouvel épisode
|
||||||
|
- [x] Analyse des infos épisodes depuis Sonarr
|
||||||
|
- [x] Mapping automatique vers les providers
|
||||||
|
- [x] Système de mapping series Sonarr → anime providers
|
||||||
|
- [x] Configuration API pour webhooks et mappings
|
||||||
|
|
||||||
|
**Nouveaux endpoints :**
|
||||||
|
- `POST /api/webhook/sonarr` - Webhook principal Sonarr
|
||||||
|
- `POST /api/webhook/test/sonarr` - Test de payload
|
||||||
|
- `GET /api/sonarr/config` - Configuration webhook
|
||||||
|
- `PUT /api/sonarr/config` - Mise à jour configuration
|
||||||
|
- `GET /api/sonarr/mappings` - Liste des mappings
|
||||||
|
- `POST /api/sonarr/mappings` - Créer mapping
|
||||||
|
- `DELETE /api/sonarr/mappings/{id}` - Supprimer mapping
|
||||||
|
- `GET /api/sonarr/search` - Rechercher anime
|
||||||
|
- `GET /api/sonarr/episodes` - Liste épisodes
|
||||||
|
- `GET /api/sonarr/suggest` - Suggestions mappings
|
||||||
|
- `POST /api/sonarr/download` - Déclencher téléchargement manuel
|
||||||
|
|
||||||
|
**Documentation :** Voir [docs/SONARR_INTEGRATION.md](docs/SONARR_INTEGRATION.md)
|
||||||
|
|
||||||
|
### Version 2.6 - Gestion de Bibliothèque Avancée
|
||||||
|
- [ ] **Bibliothèque personnelle** : Gérer sa collection d'anime téléchargés
|
||||||
|
- [ ] **Statistiques détaillées** :
|
||||||
|
- [ ] Temps de visionnage total
|
||||||
|
- [ ] Espace disque utilisé
|
||||||
|
- [ ] Animes les plus regardés
|
||||||
|
- [ ] Graphiques de statistiques
|
||||||
|
- [ ] **Marquage d'épisodes** :
|
||||||
|
- [ ] Marquer épisodes comme vus/non vus
|
||||||
|
- [ ] Système de progression automatique
|
||||||
|
- [ ] Reprendre la lecture là où on s'est arrêté
|
||||||
|
- [ ] **Listes de lecture** : Créer des playlists personnalisées
|
||||||
|
- [ ] **Notes personnelles** : Noter les animes et laisser des commentaires
|
||||||
|
|
||||||
|
### Version 2.7 - Qualité et Formats
|
||||||
|
- [ ] **Sélection de qualité** : Choisir entre 1080p, 720p, 480p
|
||||||
|
- [ ] **Conversion automatique** : Convertir en différents formats
|
||||||
|
- [ ] **Compression** : Réduire la taille des fichiers
|
||||||
|
- [ ] **Extraction de sous-titres** : Télécharger les subs automatiquement
|
||||||
|
- [ ] **Multi-audio** : Gérer les versions VF/VOSTFR
|
||||||
|
|
||||||
|
### Version 3.0 - Fonctionnalités Sociales & Mobile
|
||||||
|
- [ ] **Fonctionnalités sociales** :
|
||||||
|
- [ ] Partage de listes avec amis
|
||||||
|
- [ ] Système de commentaires et avis
|
||||||
|
- [ ] Intégration Discord/Telegram (notifications)
|
||||||
|
- [ ] **Mobile & PWA** :
|
||||||
|
- [ ] Application mobile native iOS/Android
|
||||||
|
- [ ] Progressive Web App pour offline
|
||||||
|
- [ ] Chromecast/AirPlay support
|
||||||
|
- [ ] Interface optimisée mobile
|
||||||
|
|
||||||
|
### Version 4.0 - Fonctionnalités Avancées
|
||||||
|
- [ ] **Sauvegarde cloud** : Sync avec Google Drive/Dropbox
|
||||||
|
- [ ] **Streaming distant** : Regarder partout
|
||||||
|
- [ ] **Multi-utilisateurs** : Profils et permissions
|
||||||
|
- [ ] **API publique** : API pour développeurs tiers
|
||||||
|
- [ ] **Plugins** : Système d'extensions
|
||||||
|
|
||||||
|
### Améliorations Continues
|
||||||
|
- [ ] **Performance** : Optimisation du chargement et de l'interface
|
||||||
|
- [ ] **Accessibilité** : Support lecteur d'écran, clavier
|
||||||
|
- [ ] **Tests automatisés** : Suite de tests E2E
|
||||||
|
- [ ] **Documentation** : Guides d'utilisation et API
|
||||||
|
- [ ] **Internationalisation** : Support multilingue complet
|
||||||
|
|
||||||
|
## 🤝 Contribution
|
||||||
|
|
||||||
|
Les contributions sont les bienvenues !
|
||||||
|
|
||||||
|
1. Fork le projet
|
||||||
|
2. Créez une branche (`git checkout -b feature/AmazingFeature`)
|
||||||
|
3. Commit (`git commit -m 'Add some AmazingFeature'`)
|
||||||
|
4. Push (`git push origin feature/AmazingFeature`)
|
||||||
|
5. Ouvrez une Pull Request
|
||||||
|
|
||||||
|
## 📝 Licence
|
||||||
|
|
||||||
|
Ce projet est à usage éducatif uniquement. Respectez les droits d'auteur et les lois locales.
|
||||||
|
|
||||||
|
## ⚠️ Avertissement
|
||||||
|
|
||||||
|
Ce logiciel est destiné à un usage personnel et éducatif. Les utilisateurs sont responsables de vérifier qu'ils ont le droit de télécharger du contenu protégé par des droits d'auteur dans leur juridiction.
|
||||||
|
|
||||||
|
## 📧 Support
|
||||||
|
|
||||||
|
Pour les bugs et suggestions :
|
||||||
|
- Ouvrez une issue sur GitHub
|
||||||
|
- Discutez avec la communauté
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Développé avec ❤️ pour la communauté anime**
|
||||||
|
|
||||||
|
*Version actuelle : 2.1*
|
||||||
|
*Dernière mise à jour : Janvier 2026*
|
||||||
|
|||||||
@@ -0,0 +1,175 @@
|
|||||||
|
# Restructuration des Downloaders - Résumé
|
||||||
|
|
||||||
|
## 🎯 Objectif Accompli
|
||||||
|
|
||||||
|
Restructuration complète du système de downloaders avec une distinction claire entre:
|
||||||
|
- **Sites d'anime** (catalogues avec métadonnées)
|
||||||
|
- **Players vidéo** (hébergement de fichiers)
|
||||||
|
|
||||||
|
## 📊 Nouvelle Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
app/downloaders/
|
||||||
|
├── __init__.py # Factory principal (get_downloader)
|
||||||
|
├── base.py # BaseDownloader (classe racine)
|
||||||
|
│
|
||||||
|
├── anime_sites/ # 🎌 Sites d'anime (4 downloaders)
|
||||||
|
│ ├── __init__.py # Factory: get_anime_site()
|
||||||
|
│ ├── base.py # BaseAnimeSite
|
||||||
|
│ ├── animesama.py # Anime-Sama
|
||||||
|
│ ├── nekosama.py # Neko-Sama
|
||||||
|
│ ├── animeultime.py # Anime-Ultime
|
||||||
|
│ └── vostfree.py # Vostfree
|
||||||
|
│
|
||||||
|
└── video_players/ # 🎬 Players vidéo (8 downloaders)
|
||||||
|
├── __init__.py # Factory: get_video_player()
|
||||||
|
├── base.py # BaseVideoPlayer
|
||||||
|
├── doodstream.py # Doodstream
|
||||||
|
├── sibnet.py # Sibnet
|
||||||
|
├── vidmoly.py # VidMoly (avec support M3U8 + target_filename)
|
||||||
|
├── sendvid.py # SendVid (avec target_filename)
|
||||||
|
├── lpayer.py # Lpayer
|
||||||
|
├── unfichier.py # 1fichier
|
||||||
|
├── uptobox.py # Uptobox
|
||||||
|
└── rapidfile.py # Rapidfile
|
||||||
|
```
|
||||||
|
|
||||||
|
## ✨ Changements Clés
|
||||||
|
|
||||||
|
### 1. Classes de Base Spécialisées
|
||||||
|
|
||||||
|
**BaseVideoPlayer** (`video_players/base.py`):
|
||||||
|
- Pour les hébergeurs de fichiers vidéo
|
||||||
|
- Méthode clé: `get_download_link(url, target_filename=None)`
|
||||||
|
- Supporte le paramètre optionnel `target_filename` (VidMoly, SendVid)
|
||||||
|
- Gère l'extraction d'URL de téléchargement direct
|
||||||
|
|
||||||
|
**BaseAnimeSite** (`anime_sites/base.py`):
|
||||||
|
- Pour les sites de streaming anime
|
||||||
|
- Méthodes clés:
|
||||||
|
- `search_anime(query, lang)` - Recherche dans le catalogue
|
||||||
|
- `get_episodes(anime_url, lang)` - Liste des épisodes
|
||||||
|
- `get_anime_metadata(anime_url)` - Métadonnées riches
|
||||||
|
- `get_download_link(url)` - URL du player vidéo
|
||||||
|
|
||||||
|
### 2. Preservation des Spécificités
|
||||||
|
|
||||||
|
✅ **VidMoly**: Toutes ses spécificités préservées
|
||||||
|
- Support M3U8 → MP4 conversion
|
||||||
|
- Playwright network interception
|
||||||
|
- Multi-domaines (.biz, .to, .org)
|
||||||
|
- Paramètre `target_filename`
|
||||||
|
|
||||||
|
✅ **SendVid**: Paramètre `target_filename` préservé
|
||||||
|
|
||||||
|
✅ **Tous les autres**: Aucune modification de fonctionnalité
|
||||||
|
|
||||||
|
### 3. Factory Pattern
|
||||||
|
|
||||||
|
**Nouveau `get_downloader()` dans `__init__.py`**:
|
||||||
|
```python
|
||||||
|
def get_downloader(url: str):
|
||||||
|
# Essaye les sites anime d'abord
|
||||||
|
anime_site = get_anime_site(url)
|
||||||
|
if anime_site:
|
||||||
|
return anime_site
|
||||||
|
|
||||||
|
# Puis les players vidéo
|
||||||
|
video_player = get_video_player(url)
|
||||||
|
if video_player:
|
||||||
|
return video_player
|
||||||
|
|
||||||
|
# Fallback générique
|
||||||
|
return GenericDownloader()
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🧪 Tests
|
||||||
|
|
||||||
|
✅ **23/23 tests passants** dans `tests/test_downloaders.py`
|
||||||
|
✅ **Imports mis à jour** pour utiliser la nouvelle structure
|
||||||
|
✅ **URL routing correct** pour tous les types
|
||||||
|
|
||||||
|
## 📈 Avantages
|
||||||
|
|
||||||
|
1. **Organisation claire**: Distinction évidente entre catalogues et hébergeurs
|
||||||
|
2. **Maintenabilité**: Ajouter un nouveau player ou site est plus intuitif
|
||||||
|
3. **Type safety**: Héritage spécifique avec méthodes appropriées
|
||||||
|
4. **Flexibilité**: Support des cas particuliers (VidMoly, SendVid)
|
||||||
|
5. **Backward compatibility**: L'API principale `get_downloader()` fonctionne toujours
|
||||||
|
|
||||||
|
## 🚀 Comment Ajouter un Nouveau Downloader
|
||||||
|
|
||||||
|
### Nouveau Player Vidéo:
|
||||||
|
```python
|
||||||
|
# app/downloaders/video_players/myplayer.py
|
||||||
|
from .base import BaseVideoPlayer
|
||||||
|
|
||||||
|
class MyPlayerDownloader(BaseVideoPlayer):
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return "myplayer.com" in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: str = None):
|
||||||
|
# ... extraction logic ...
|
||||||
|
return download_url, filename
|
||||||
|
```
|
||||||
|
|
||||||
|
### Nouveau Site Anime:
|
||||||
|
```python
|
||||||
|
# app/downloaders/anime_sites/mysite.py
|
||||||
|
from .base import BaseAnimeSite
|
||||||
|
|
||||||
|
class MyAnimeSiteDownloader(BaseAnimeSite):
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return "myanime.site" in url.lower()
|
||||||
|
|
||||||
|
async def search_anime(self, query: str, lang: str = "vostfr"):
|
||||||
|
# ... search logic ...
|
||||||
|
return anime_list
|
||||||
|
|
||||||
|
async def get_episodes(self, anime_url: str, lang: str = "vostfr"):
|
||||||
|
# ... episode listing logic ...
|
||||||
|
return episode_list
|
||||||
|
|
||||||
|
async def get_anime_metadata(self, anime_url: str):
|
||||||
|
# ... metadata extraction ...
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str):
|
||||||
|
# ... extract video player URL ...
|
||||||
|
return player_url, title
|
||||||
|
```
|
||||||
|
|
||||||
|
## ✅ Validation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Tests
|
||||||
|
pytest tests/test_downloaders.py -v # 23/23 passed ✅
|
||||||
|
|
||||||
|
# Imports
|
||||||
|
from app.downloaders import get_downloader # ✅
|
||||||
|
from app.downloaders.video_players import BaseVideoPlayer # ✅
|
||||||
|
from app.downloaders.anime_sites import BaseAnimeSite # ✅
|
||||||
|
|
||||||
|
# Routing
|
||||||
|
get_downloader('https://doodstream.com/e/abc') # → DoodStreamDownloader ✅
|
||||||
|
get_downloader('https://anime-sama.si/naruto') # → AnimeSamaDownloader ✅
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📝 Fichiers Modifiés
|
||||||
|
|
||||||
|
**Nouveaux**: 18 fichiers
|
||||||
|
- 2 classes de base (base.py)
|
||||||
|
- 2 __init__.py avec factories
|
||||||
|
- 12 downloaders migrés
|
||||||
|
- 2 dossiers (anime_sites/, video_players/)
|
||||||
|
|
||||||
|
**Supprimés**: 12 anciens fichiers dans `app/downloaders/`
|
||||||
|
|
||||||
|
**Mis à jour**:
|
||||||
|
- `app/downloaders/__init__.py` (factory principal)
|
||||||
|
- `tests/test_downloaders.py` (imports)
|
||||||
|
|
||||||
|
---
|
||||||
|
**Date**: 2026-01-24
|
||||||
|
**Statut**: ✅ Terminé et testé
|
||||||
|
**Impact**: Aucune rupture de fonctionnalité
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
# Ohm Stream Downloader Package
|
||||||
|
|||||||
+153
-127
@@ -1,161 +1,187 @@
|
|||||||
"""Authentification : JWT court + refresh token longue durée, rôles admin/user."""
|
"""User authentication and management system"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Optional, Dict
|
||||||
|
from passlib.context import CryptContext
|
||||||
import logging
|
import logging
|
||||||
import secrets
|
from fastapi import HTTPException
|
||||||
from dataclasses import dataclass
|
from fastapi.security import HTTPAuthorizationCredentials
|
||||||
from datetime import UTC, datetime, timedelta
|
|
||||||
|
|
||||||
import jwt
|
|
||||||
from pwdlib import PasswordHash
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.db import db
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
password_hash = PasswordHash.recommended()
|
# Password hashing context
|
||||||
|
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||||
|
|
||||||
|
# JWT Secret key - SHOULD BE CONFIGURED VIA ENV
|
||||||
|
SECRET_KEY = os.getenv("JWT_SECRET_KEY", "dev-secret-change-in-production")
|
||||||
ALGORITHM = "HS256"
|
ALGORITHM = "HS256"
|
||||||
|
ACCESS_TOKEN_EXPIRE_MINUTES = 60 * 24 * 7 # 7 days
|
||||||
|
|
||||||
|
# Users database file
|
||||||
|
USERS_DB_FILE = "config/users.json"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
class UserManager:
|
||||||
class User:
|
"""Manages user storage and authentication"""
|
||||||
id: int
|
|
||||||
username: str
|
|
||||||
is_admin: bool
|
|
||||||
is_active: bool
|
|
||||||
content_preference: str = "both" # anime | serie | both
|
|
||||||
|
|
||||||
|
def __init__(self, db_file: str = USERS_DB_FILE):
|
||||||
|
self.db_file = db_file
|
||||||
|
self.users: Dict[str, dict] = {}
|
||||||
|
self._load_users()
|
||||||
|
|
||||||
VALID_CONTENT_PREFERENCES = ("anime", "serie", "both")
|
def _load_users(self):
|
||||||
|
"""Load users from JSON file"""
|
||||||
|
try:
|
||||||
|
if os.path.exists(self.db_file):
|
||||||
|
with open(self.db_file, 'r', encoding='utf-8') as f:
|
||||||
|
self.users = json.load(f)
|
||||||
|
logger.info(f"Loaded {len(self.users)} users from database")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error loading users: {e}")
|
||||||
|
self.users = {}
|
||||||
|
|
||||||
|
def _save_users(self):
|
||||||
|
try:
|
||||||
|
os.makedirs(os.path.dirname(self.db_file), exist_ok=True)
|
||||||
|
temp_file = f"{self.db_file}.tmp"
|
||||||
|
with open(temp_file, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(self.users, f, indent=2, ensure_ascii=False, default=str)
|
||||||
|
os.replace(temp_file, self.db_file)
|
||||||
|
logger.info(f"Saved {len(self.users)} users to database")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error saving users: {e}")
|
||||||
|
|
||||||
# ---------------------------------------------------------------- mots de passe
|
def get_user(self, username: str) -> Optional[dict]:
|
||||||
|
"""Get user by username"""
|
||||||
|
return self.users.get(username)
|
||||||
|
|
||||||
|
def get_user_by_id(self, user_id: str) -> Optional[dict]:
|
||||||
def hash_password(password: str) -> str:
|
"""Get user by ID"""
|
||||||
return password_hash.hash(password)
|
for user in self.users.values():
|
||||||
|
if user.get('id') == user_id:
|
||||||
|
return user
|
||||||
def verify_password(password: str, hashed: str) -> bool:
|
|
||||||
return password_hash.verify(password, hashed)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- utilisateurs
|
|
||||||
|
|
||||||
|
|
||||||
async def create_user(username: str, password: str) -> User:
|
|
||||||
"""Crée un compte ; le tout premier utilisateur devient admin."""
|
|
||||||
row = await db.fetchone("SELECT COUNT(*) AS n FROM users")
|
|
||||||
is_first = row["n"] == 0
|
|
||||||
cursor = await db.execute(
|
|
||||||
"INSERT INTO users (username, password_hash, is_admin) VALUES (?, ?, ?)",
|
|
||||||
(username.strip(), hash_password(password), int(is_first)),
|
|
||||||
)
|
|
||||||
user = User(id=cursor.lastrowid, username=username.strip(), is_admin=is_first, is_active=True)
|
|
||||||
logger.info("Utilisateur créé : %s (admin=%s)", user.username, user.is_admin)
|
|
||||||
return user
|
|
||||||
|
|
||||||
|
|
||||||
async def authenticate(username: str, password: str) -> User | None:
|
|
||||||
row = await db.fetchone(
|
|
||||||
"SELECT id, username, password_hash, is_admin, is_active, content_preference "
|
|
||||||
"FROM users WHERE username = ?",
|
|
||||||
(username.strip(),),
|
|
||||||
)
|
|
||||||
if row is None or not verify_password(password, row["password_hash"]):
|
|
||||||
logger.warning("Échec d'authentification pour %r", username)
|
|
||||||
return None
|
return None
|
||||||
if not row["is_active"]:
|
|
||||||
logger.warning("Compte désactivé : %r", username)
|
def create_user(self, username: str, password: str, email: str = None, full_name: str = None) -> dict:
|
||||||
return None
|
"""Create a new user"""
|
||||||
return User(
|
if username in self.users:
|
||||||
id=row["id"],
|
raise ValueError(f"Username '{username}' already exists")
|
||||||
username=row["username"],
|
|
||||||
is_admin=bool(row["is_admin"]),
|
# Truncate password to 72 bytes if necessary (bcrypt limitation)
|
||||||
is_active=True,
|
password_bytes = password.encode('utf-8')
|
||||||
content_preference=row["content_preference"],
|
if len(password_bytes) > 72:
|
||||||
)
|
password = password_bytes[:72].decode('utf-8', errors='ignore')
|
||||||
|
|
||||||
|
# Hash password
|
||||||
|
hashed_password = pwd_context.hash(password)
|
||||||
|
|
||||||
|
# Create user
|
||||||
|
user = {
|
||||||
|
"id": hashlib.sha256(username.encode()).hexdigest()[:32],
|
||||||
|
"username": username,
|
||||||
|
"email": email,
|
||||||
|
"full_name": full_name,
|
||||||
|
"hashed_password": hashed_password,
|
||||||
|
"is_active": True,
|
||||||
|
"created_at": datetime.now().isoformat(),
|
||||||
|
"last_login": None
|
||||||
|
}
|
||||||
|
|
||||||
|
self.users[username] = user
|
||||||
|
self._save_users()
|
||||||
|
|
||||||
|
logger.info(f"Created user: {username}")
|
||||||
|
return user
|
||||||
|
|
||||||
|
def authenticate_user(self, username: str, password: str) -> Optional[dict]:
|
||||||
|
"""Authenticate user with username and password"""
|
||||||
|
user = self.get_user(username)
|
||||||
|
if not user:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not pwd_context.verify(password, user["hashed_password"]):
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Update last login
|
||||||
|
user["last_login"] = datetime.now().isoformat()
|
||||||
|
self._save_users()
|
||||||
|
|
||||||
|
return user
|
||||||
|
|
||||||
|
def update_last_login(self, username: str):
|
||||||
|
"""Update user's last login time"""
|
||||||
|
user = self.get_user(username)
|
||||||
|
if user:
|
||||||
|
user["last_login"] = datetime.now().isoformat()
|
||||||
|
self._save_users()
|
||||||
|
|
||||||
|
|
||||||
async def get_user(user_id: int) -> User | None:
|
# Global user manager instance
|
||||||
row = await db.fetchone(
|
user_manager = UserManager()
|
||||||
"SELECT id, username, is_admin, is_active, content_preference FROM users WHERE id = ?",
|
|
||||||
(user_id,),
|
|
||||||
)
|
|
||||||
if row is None or not row["is_active"]:
|
|
||||||
return None
|
|
||||||
return User(
|
|
||||||
id=row["id"],
|
|
||||||
username=row["username"],
|
|
||||||
is_admin=bool(row["is_admin"]),
|
|
||||||
is_active=True,
|
|
||||||
content_preference=row["content_preference"],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def set_content_preference(user_id: int, preference: str) -> None:
|
def verify_password(plain_password: str, hashed_password: str) -> bool:
|
||||||
if preference not in VALID_CONTENT_PREFERENCES:
|
"""Verify a password against a hash"""
|
||||||
raise ValueError(f"Préférence invalide : {preference!r}")
|
return pwd_context.verify(plain_password, hashed_password)
|
||||||
await db.execute(
|
|
||||||
"UPDATE users SET content_preference = ? WHERE id = ?", (preference, user_id)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- tokens
|
def get_password_hash(password: str) -> str:
|
||||||
|
"""Hash a password for storage"""
|
||||||
|
return pwd_context.hash(password)
|
||||||
|
|
||||||
|
|
||||||
def create_access_token(user: User) -> str:
|
def create_access_token(data: dict, expires_delta: timedelta = None) -> str:
|
||||||
settings = get_settings()
|
"""Create JWT access token"""
|
||||||
payload = {
|
from jose import jwt
|
||||||
"sub": str(user.id),
|
|
||||||
"username": user.username,
|
to_encode = data.copy()
|
||||||
"admin": user.is_admin,
|
|
||||||
"exp": datetime.now(UTC) + timedelta(minutes=settings.access_token_ttl_minutes),
|
if expires_delta:
|
||||||
}
|
expire = datetime.utcnow() + expires_delta
|
||||||
return jwt.encode(payload, settings.secret_key, algorithm=ALGORITHM)
|
else:
|
||||||
|
expire = datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||||
|
|
||||||
|
to_encode.update({"exp": expire})
|
||||||
|
encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
|
||||||
|
|
||||||
|
return encoded_jwt
|
||||||
|
|
||||||
|
|
||||||
def decode_access_token(token: str) -> dict | None:
|
def verify_token(token: str) -> Optional[str]:
|
||||||
|
"""Verify JWT token and return username"""
|
||||||
|
from jose import jwt
|
||||||
|
from jose.exceptions import JWTError
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return jwt.decode(token, get_settings().secret_key, algorithms=[ALGORITHM])
|
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
|
||||||
except jwt.PyJWTError as exc:
|
username: str = payload.get("sub")
|
||||||
logger.debug("Token d'accès invalide : %s", exc)
|
if username is None:
|
||||||
|
return None
|
||||||
|
return username
|
||||||
|
except JWTError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _hash_token(token: str) -> str:
|
# Alias for backward compatibility
|
||||||
return hashlib.sha256(token.encode()).hexdigest()
|
get_user_from_token = verify_token
|
||||||
|
|
||||||
|
|
||||||
async def create_refresh_token(user_id: int) -> str:
|
def get_current_user(credentials: HTTPAuthorizationCredentials) -> dict:
|
||||||
"""Refresh token opaque ; seul son hash est stocké en DB (révocable)."""
|
|
||||||
token = secrets.token_urlsafe(48)
|
|
||||||
expires = datetime.now(UTC) + timedelta(days=get_settings().refresh_token_ttl_days)
|
|
||||||
await db.execute(
|
|
||||||
"INSERT INTO refresh_tokens (user_id, token_hash, expires_at) VALUES (?, ?, ?)",
|
|
||||||
(user_id, _hash_token(token), expires.strftime("%Y-%m-%d %H:%M:%S")),
|
|
||||||
)
|
|
||||||
return token
|
|
||||||
|
|
||||||
|
|
||||||
async def use_refresh_token(token: str) -> User | None:
|
|
||||||
"""Valide un refresh token, le révoque (rotation) et retourne l'utilisateur."""
|
|
||||||
row = await db.fetchone(
|
|
||||||
"SELECT id, user_id FROM refresh_tokens "
|
|
||||||
"WHERE token_hash = ? AND revoked = 0 AND expires_at > datetime('now')",
|
|
||||||
(_hash_token(token),),
|
|
||||||
)
|
|
||||||
if row is None:
|
|
||||||
logger.warning("Refresh token invalide ou expiré")
|
|
||||||
return None
|
return None
|
||||||
await db.execute("UPDATE refresh_tokens SET revoked = 1 WHERE id = ?", (row["id"],))
|
|
||||||
user = await get_user(row["user_id"])
|
|
||||||
if user is None:
|
|
||||||
logger.warning("Refresh token d'un compte supprimé/désactivé (user_id=%s)", row["user_id"])
|
|
||||||
return user
|
|
||||||
|
|
||||||
|
|
||||||
async def revoke_all_refresh_tokens(user_id: int) -> None:
|
def get_current_user(credentials: HTTPAuthorizationCredentials) -> dict:
|
||||||
await db.execute("UPDATE refresh_tokens SET revoked = 1 WHERE user_id = ?", (user_id,))
|
"""Get current user from JWT token"""
|
||||||
|
token = credentials.credentials
|
||||||
|
username = verify_token(token)
|
||||||
|
if username:
|
||||||
|
user = user_manager.get_user(username)
|
||||||
|
if not user:
|
||||||
|
raise HTTPException(status_code=401, detail="User not found")
|
||||||
|
if not user.get("is_active", True):
|
||||||
|
raise HTTPException(status_code=401, detail="Inactive user")
|
||||||
|
return user
|
||||||
|
raise HTTPException(status_code=401, detail="Invalid authentication credentials")
|
||||||
|
|||||||
@@ -0,0 +1,154 @@
|
|||||||
|
"""Scheduler for automatic episode checking and downloading"""
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||||
|
from apscheduler.triggers.interval import IntervalTrigger
|
||||||
|
|
||||||
|
from app.watchlist import watchlist_manager, WatchlistManager
|
||||||
|
from app.episode_checker import EpisodeChecker, episode_checker
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class AutoDownloadScheduler:
|
||||||
|
"""Manages automatic episode checking and downloading on a schedule"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
wlm: Optional[WatchlistManager] = None,
|
||||||
|
checker: Optional[EpisodeChecker] = None
|
||||||
|
):
|
||||||
|
self.wlm = wlm or watchlist_manager
|
||||||
|
self.checker = checker or episode_checker
|
||||||
|
self.scheduler: Optional[AsyncIOScheduler] = None
|
||||||
|
self._running = False
|
||||||
|
|
||||||
|
async def _check_job(self):
|
||||||
|
"""Job function that runs periodically to check for new episodes"""
|
||||||
|
try:
|
||||||
|
logger.info("Running scheduled episode check...")
|
||||||
|
results = await self.checker.check_all_due()
|
||||||
|
|
||||||
|
# Log summary
|
||||||
|
for result in results:
|
||||||
|
if result.new_episodes_found > 0:
|
||||||
|
logger.info(
|
||||||
|
f"✓ {result.anime_title}: "
|
||||||
|
f"{result.new_episodes_found} new, "
|
||||||
|
f"{len(result.episodes_downloaded)} downloaded"
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"Scheduled check complete: processed {len(results)} items")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error in scheduled check job: {e}", exc_info=True)
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
"""Start the scheduler"""
|
||||||
|
if self._running:
|
||||||
|
logger.warning("Scheduler already running")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.scheduler = AsyncIOScheduler()
|
||||||
|
|
||||||
|
# Get initial check interval from settings
|
||||||
|
settings = self.wlm.get_settings()
|
||||||
|
interval_hours = settings.check_interval_hours
|
||||||
|
|
||||||
|
# Add the job
|
||||||
|
self.scheduler.add_job(
|
||||||
|
self._check_job,
|
||||||
|
trigger=IntervalTrigger(hours=interval_hours),
|
||||||
|
id='episode_check',
|
||||||
|
name='Check for new episodes',
|
||||||
|
replace_existing=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# Start the scheduler
|
||||||
|
self.scheduler.start()
|
||||||
|
self._running = True
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"Auto-download scheduler started (checking every {interval_hours}h)"
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error starting scheduler: {e}", exc_info=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""Stop the scheduler"""
|
||||||
|
if not self._running:
|
||||||
|
logger.warning("Scheduler not running")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
if self.scheduler:
|
||||||
|
self.scheduler.shutdown(wait=False)
|
||||||
|
self.scheduler = None
|
||||||
|
|
||||||
|
self._running = False
|
||||||
|
logger.info("Auto-download scheduler stopped")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error stopping scheduler: {e}", exc_info=True)
|
||||||
|
|
||||||
|
def restart(self):
|
||||||
|
"""Restart the scheduler with updated settings"""
|
||||||
|
logger.info("Restarting scheduler with new settings...")
|
||||||
|
self.stop()
|
||||||
|
self.start()
|
||||||
|
|
||||||
|
def update_interval(self, hours: int):
|
||||||
|
"""Update the check interval"""
|
||||||
|
if not self._running:
|
||||||
|
logger.warning("Scheduler not running, interval will be applied on start")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
settings = self.wlm.get_settings()
|
||||||
|
settings.check_interval_hours = hours
|
||||||
|
self.wlm.update_settings(settings)
|
||||||
|
|
||||||
|
# Restart to apply new interval
|
||||||
|
self.restart()
|
||||||
|
|
||||||
|
logger.info(f"Updated check interval to {hours}h")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error updating interval: {e}", exc_info=True)
|
||||||
|
|
||||||
|
def get_next_run_time(self) -> Optional[datetime]:
|
||||||
|
"""Get the next scheduled run time"""
|
||||||
|
if not self._running or not self.scheduler:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
job = self.scheduler.get_job('episode_check')
|
||||||
|
if job:
|
||||||
|
return job.next_run_time
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting next run time: {e}")
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def is_running(self) -> bool:
|
||||||
|
"""Check if scheduler is running"""
|
||||||
|
return self._running
|
||||||
|
|
||||||
|
async def trigger_check_now(self):
|
||||||
|
"""Manually trigger an episode check now"""
|
||||||
|
logger.info("Manually triggering episode check...")
|
||||||
|
try:
|
||||||
|
await self._check_job()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error in manual check: {e}", exc_info=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
# Global scheduler instance
|
||||||
|
auto_download_scheduler = AutoDownloadScheduler()
|
||||||
+43
-38
@@ -1,53 +1,58 @@
|
|||||||
from functools import lru_cache
|
"""Application configuration using environment variables"""
|
||||||
from pathlib import Path
|
from pydantic_settings import BaseSettings
|
||||||
|
from typing import List
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
import os
|
||||||
|
|
||||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
|
||||||
|
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
model_config = SettingsConfigDict(env_prefix="OHM_", env_file=".env", extra="ignore")
|
"""Application settings loaded from environment variables"""
|
||||||
|
|
||||||
|
# Application
|
||||||
app_name: str = "Ohm Stream Downloader"
|
app_name: str = "Ohm Stream Downloader"
|
||||||
|
app_version: str = "2.2"
|
||||||
debug: bool = False
|
debug: bool = False
|
||||||
|
|
||||||
# Données
|
# Server
|
||||||
data_dir: Path = BASE_DIR / "data"
|
host: str = "0.0.0.0"
|
||||||
download_dir: Path = BASE_DIR / "downloads"
|
port: int = 3000
|
||||||
database_path: Path = BASE_DIR / "data" / "ohm.db"
|
reload: bool = True
|
||||||
|
|
||||||
# Sécurité — à surcharger via OHM_SECRET_KEY en production
|
# Downloads
|
||||||
secret_key: str = "change-me-in-production"
|
download_dir: str = "downloads"
|
||||||
access_token_ttl_minutes: int = 15
|
|
||||||
refresh_token_ttl_days: int = 30
|
|
||||||
|
|
||||||
# Scraping
|
|
||||||
http_timeout: float = 20.0
|
|
||||||
user_agent: str = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
|
|
||||||
scrapers_config_dir: Path = BASE_DIR / "app" / "scrapers" / "configs"
|
|
||||||
|
|
||||||
# Téléchargements
|
|
||||||
max_parallel_downloads: int = 3
|
max_parallel_downloads: int = 3
|
||||||
|
chunk_size: int = 1024 * 1024 # 1MB chunks
|
||||||
|
|
||||||
# Kitsu
|
# CORS
|
||||||
kitsu_base_url: str = "https://kitsu.io/api/edge"
|
cors_origins: List[str] = [
|
||||||
metadata_cache_ttl_hours: int = 72
|
"http://localhost:3000",
|
||||||
|
"http://127.0.0.1:3000",
|
||||||
|
"http://192.168.1.204:3000",
|
||||||
|
"http://192.168.1.204"
|
||||||
|
]
|
||||||
|
|
||||||
# Mise à jour (dépôt public — lecture anonyme de l'API Gitea)
|
# Storage
|
||||||
gitea_url: str = "https://git.lanro.eu"
|
favorites_storage_path: str = "favorites.json"
|
||||||
gitea_repo: str = "Roman/ohm_streaming"
|
|
||||||
# Déploiement Docker — Watchtower compagnon
|
|
||||||
watchtower_url: str = ""
|
|
||||||
watchtower_token: str = ""
|
|
||||||
|
|
||||||
def ensure_dirs(self) -> None:
|
# Sonarr
|
||||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
sonarr_config_path: str = "config/sonarr.json"
|
||||||
self.download_dir.mkdir(parents=True, exist_ok=True)
|
sonarr_mappings_path: str = "config/sonarr_mappings.json"
|
||||||
|
|
||||||
|
# API Timeouts
|
||||||
|
http_timeout: float = 10.0
|
||||||
|
download_timeout: int = 300 # 5 minutes
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
log_level: str = "INFO"
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
env_file = ".env"
|
||||||
|
env_file_encoding = "utf-8"
|
||||||
|
case_sensitive = False
|
||||||
|
|
||||||
|
|
||||||
|
# Global settings instance
|
||||||
|
settings = Settings()
|
||||||
|
|
||||||
|
|
||||||
@lru_cache
|
|
||||||
def get_settings() -> Settings:
|
def get_settings() -> Settings:
|
||||||
settings = Settings()
|
"""Get the global settings instance"""
|
||||||
settings.ensure_dirs()
|
|
||||||
return settings
|
return settings
|
||||||
|
|||||||
@@ -1,135 +0,0 @@
|
|||||||
"""Couche d'accès SQLite (aiosqlite) — unique source de vérité de l'application."""
|
|
||||||
|
|
||||||
import aiosqlite
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
|
|
||||||
SCHEMA = """
|
|
||||||
CREATE TABLE IF NOT EXISTS users (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
username TEXT NOT NULL UNIQUE COLLATE NOCASE,
|
|
||||||
password_hash TEXT NOT NULL,
|
|
||||||
is_admin INTEGER NOT NULL DEFAULT 0,
|
|
||||||
is_active INTEGER NOT NULL DEFAULT 1,
|
|
||||||
content_preference TEXT NOT NULL DEFAULT 'both'
|
|
||||||
CHECK (content_preference IN ('anime','serie','both')),
|
|
||||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS refresh_tokens (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
|
||||||
token_hash TEXT NOT NULL UNIQUE,
|
|
||||||
expires_at TEXT NOT NULL,
|
|
||||||
revoked INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS downloads (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
source_key TEXT NOT NULL, -- clé de déduplication (video_url normalisée)
|
|
||||||
video_url TEXT NOT NULL,
|
|
||||||
page_url TEXT,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
file_path TEXT, -- relatif au dossier de téléchargement
|
|
||||||
status TEXT NOT NULL DEFAULT 'pending'
|
|
||||||
CHECK (status IN ('pending','downloading','paused','done','failed','cancelled')),
|
|
||||||
total_bytes INTEGER,
|
|
||||||
downloaded_bytes INTEGER NOT NULL DEFAULT 0,
|
|
||||||
error TEXT,
|
|
||||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
||||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
||||||
);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_downloads_status ON downloads(status);
|
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_downloads_active_source
|
|
||||||
ON downloads(source_key) WHERE status IN ('pending','downloading','paused');
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS metadata_cache (
|
|
||||||
cache_key TEXT PRIMARY KEY,
|
|
||||||
payload TEXT NOT NULL, -- JSON
|
|
||||||
fetched_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS settings (
|
|
||||||
key TEXT PRIMARY KEY,
|
|
||||||
value TEXT NOT NULL -- JSON
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS favorites (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT NOT NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
image_url TEXT,
|
|
||||||
payload TEXT, -- JSON métadonnées
|
|
||||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
||||||
UNIQUE (user_id, source, source_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS watch_progress (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
|
||||||
download_id INTEGER NOT NULL REFERENCES downloads(id) ON DELETE CASCADE,
|
|
||||||
position_seconds REAL NOT NULL DEFAULT 0,
|
|
||||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
||||||
UNIQUE (user_id, download_id)
|
|
||||||
);
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class Database:
|
|
||||||
"""Connexion SQLite partagée, initialisée au démarrage de l'app."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self._conn: aiosqlite.Connection | None = None
|
|
||||||
|
|
||||||
async def connect(self) -> None:
|
|
||||||
path = get_settings().database_path
|
|
||||||
self._conn = await aiosqlite.connect(path)
|
|
||||||
self._conn.row_factory = aiosqlite.Row
|
|
||||||
await self._conn.execute("PRAGMA journal_mode=WAL")
|
|
||||||
await self._conn.execute("PRAGMA foreign_keys=ON")
|
|
||||||
await self._conn.executescript(SCHEMA)
|
|
||||||
await self._migrate()
|
|
||||||
await self._conn.commit()
|
|
||||||
|
|
||||||
async def _migrate(self) -> None:
|
|
||||||
"""Migrations légères : colonnes ajoutées après coup pour les bases existantes."""
|
|
||||||
cursor = await self._conn.execute("PRAGMA table_info(users)")
|
|
||||||
columns = {row["name"] for row in await cursor.fetchall()}
|
|
||||||
if "content_preference" not in columns:
|
|
||||||
await self._conn.execute(
|
|
||||||
"ALTER TABLE users ADD COLUMN content_preference TEXT NOT NULL DEFAULT 'both'"
|
|
||||||
)
|
|
||||||
|
|
||||||
async def close(self) -> None:
|
|
||||||
if self._conn is not None:
|
|
||||||
await self._conn.close()
|
|
||||||
self._conn = None
|
|
||||||
|
|
||||||
@property
|
|
||||||
def conn(self) -> aiosqlite.Connection:
|
|
||||||
if self._conn is None:
|
|
||||||
raise RuntimeError("Database.connect() n'a pas été appelé")
|
|
||||||
return self._conn
|
|
||||||
|
|
||||||
async def execute(self, sql: str, params: tuple = ()) -> aiosqlite.Cursor:
|
|
||||||
cursor = await self.conn.execute(sql, params)
|
|
||||||
await self.conn.commit()
|
|
||||||
return cursor
|
|
||||||
|
|
||||||
async def fetchone(self, sql: str, params: tuple = ()) -> aiosqlite.Row | None:
|
|
||||||
cursor = await self.conn.execute(sql, params)
|
|
||||||
row = await cursor.fetchone()
|
|
||||||
await cursor.close()
|
|
||||||
return row
|
|
||||||
|
|
||||||
async def fetchall(self, sql: str, params: tuple = ()) -> list[aiosqlite.Row]:
|
|
||||||
cursor = await self.conn.execute(sql, params)
|
|
||||||
rows = await cursor.fetchall()
|
|
||||||
await cursor.close()
|
|
||||||
return rows
|
|
||||||
|
|
||||||
|
|
||||||
db = Database()
|
|
||||||
@@ -0,0 +1,281 @@
|
|||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, Optional
|
||||||
|
import httpx
|
||||||
|
from app.models import DownloadTask, DownloadStatus, DownloadRequest
|
||||||
|
from app.downloaders import get_downloader
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadManager:
|
||||||
|
"""Manages multiple downloads with queue and progress tracking"""
|
||||||
|
|
||||||
|
def __init__(self, download_dir: str = "downloads", max_parallel: int = 3):
|
||||||
|
self.download_dir = Path(download_dir)
|
||||||
|
self.download_dir.mkdir(exist_ok=True)
|
||||||
|
self.max_parallel = max_parallel
|
||||||
|
self.tasks: Dict[str, DownloadTask] = {}
|
||||||
|
self.active_downloads: Dict[str, asyncio.Task] = {}
|
||||||
|
self._semaphore = asyncio.Semaphore(max_parallel)
|
||||||
|
|
||||||
|
def get_task(self, task_id: str) -> Optional[DownloadTask]:
|
||||||
|
return self.tasks.get(task_id)
|
||||||
|
|
||||||
|
def get_all_tasks(self) -> list[DownloadTask]:
|
||||||
|
return list(self.tasks.values())
|
||||||
|
|
||||||
|
def create_task(self, request: DownloadRequest) -> DownloadTask:
|
||||||
|
# Check for existing tasks with the same URL
|
||||||
|
# Extract actual URL from pipe-separated format
|
||||||
|
url_to_check = request.url.split('|')[0] if '|' in request.url else request.url
|
||||||
|
|
||||||
|
# Look for existing non-failed tasks with the same URL
|
||||||
|
for existing_task in self.tasks.values():
|
||||||
|
existing_url = existing_task.url.split('|')[0] if '|' in existing_task.url else existing_task.url
|
||||||
|
|
||||||
|
# If same URL and task is not failed/cancelled/completed
|
||||||
|
if existing_url == url_to_check and existing_task.status not in [
|
||||||
|
DownloadStatus.FAILED,
|
||||||
|
DownloadStatus.CANCELLED,
|
||||||
|
DownloadStatus.COMPLETED
|
||||||
|
]:
|
||||||
|
logger.info(f"Duplicate download detected: {url_to_check[:80]}...")
|
||||||
|
logger.info(f"Returning existing task: {existing_task.id}")
|
||||||
|
return existing_task
|
||||||
|
|
||||||
|
# No duplicate found, create new task
|
||||||
|
task_id = str(uuid.uuid4())
|
||||||
|
task = DownloadTask(
|
||||||
|
id=task_id,
|
||||||
|
url=request.url,
|
||||||
|
filename=request.filename or "download",
|
||||||
|
host="other",
|
||||||
|
status=DownloadStatus.PENDING,
|
||||||
|
created_at=datetime.now()
|
||||||
|
)
|
||||||
|
self.tasks[task_id] = task
|
||||||
|
return task
|
||||||
|
|
||||||
|
async def start_download(self, task_id: str):
|
||||||
|
task = self.tasks.get(task_id)
|
||||||
|
if not task:
|
||||||
|
raise ValueError(f"Task {task_id} not found")
|
||||||
|
|
||||||
|
if task.status == DownloadStatus.DOWNLOADING:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Cancel any existing download task
|
||||||
|
if task_id in self.active_downloads:
|
||||||
|
self.active_downloads[task_id].cancel()
|
||||||
|
|
||||||
|
# Start new download
|
||||||
|
download_task = asyncio.create_task(self._download(task))
|
||||||
|
self.active_downloads[task_id] = download_task
|
||||||
|
|
||||||
|
async def pause_download(self, task_id: str):
|
||||||
|
task = self.tasks.get(task_id)
|
||||||
|
if task and task.status == DownloadStatus.DOWNLOADING:
|
||||||
|
task.status = DownloadStatus.PAUSED
|
||||||
|
if task_id in self.active_downloads:
|
||||||
|
self.active_downloads[task_id].cancel()
|
||||||
|
del self.active_downloads[task_id]
|
||||||
|
|
||||||
|
async def cancel_download(self, task_id: str):
|
||||||
|
task = self.tasks.get(task_id)
|
||||||
|
if task:
|
||||||
|
task.status = DownloadStatus.CANCELLED
|
||||||
|
if task_id in self.active_downloads:
|
||||||
|
self.active_downloads[task_id].cancel()
|
||||||
|
del self.active_downloads[task_id]
|
||||||
|
|
||||||
|
# Delete partial file
|
||||||
|
if task.file_path and os.path.exists(task.file_path):
|
||||||
|
os.remove(task.file_path)
|
||||||
|
|
||||||
|
async def delete_task(self, task_id: str):
|
||||||
|
"""Completely remove a task from the task list (keeps completed files)"""
|
||||||
|
task = self.tasks.get(task_id)
|
||||||
|
if task:
|
||||||
|
# Cancel if downloading
|
||||||
|
if task_id in self.active_downloads:
|
||||||
|
self.active_downloads[task_id].cancel()
|
||||||
|
del self.active_downloads[task_id]
|
||||||
|
|
||||||
|
# Delete partial file ONLY if download is not completed
|
||||||
|
if task.status != DownloadStatus.COMPLETED:
|
||||||
|
if task.file_path and os.path.exists(task.file_path):
|
||||||
|
os.remove(task.file_path)
|
||||||
|
|
||||||
|
# Remove from tasks dict
|
||||||
|
del self.tasks[task_id]
|
||||||
|
|
||||||
|
async def _download(self, task: DownloadTask):
|
||||||
|
async with self._semaphore:
|
||||||
|
try:
|
||||||
|
task.status = DownloadStatus.DOWNLOADING
|
||||||
|
task.started_at = datetime.now()
|
||||||
|
|
||||||
|
# Get downloader and extract link
|
||||||
|
downloader = get_downloader(task.url)
|
||||||
|
|
||||||
|
# Extract episode title from pipe-separated URL if present
|
||||||
|
# Format: video_url|anime_page_url|episode_title
|
||||||
|
target_filename = None
|
||||||
|
if '|' in task.url:
|
||||||
|
parts = task.url.split('|')
|
||||||
|
if len(parts) >= 3:
|
||||||
|
target_filename = parts[2].strip()
|
||||||
|
logger.debug(f"Extracted target filename from pipe: {target_filename}")
|
||||||
|
|
||||||
|
download_url, filename = await downloader.get_download_link(task.url, target_filename)
|
||||||
|
|
||||||
|
logger.info(f"Download URL: {download_url[:100] if len(download_url) > 100 else download_url}")
|
||||||
|
logger.debug(f"Downloader filename: {filename}")
|
||||||
|
logger.debug(f"Task filename before: {task.filename}")
|
||||||
|
|
||||||
|
if not task.filename or task.filename == "download":
|
||||||
|
task.filename = filename
|
||||||
|
logger.debug(f"Task filename updated to: {task.filename}")
|
||||||
|
else:
|
||||||
|
logger.debug(f"Task filename kept as: {task.filename}")
|
||||||
|
|
||||||
|
task.file_path = str(self.download_dir / task.filename)
|
||||||
|
|
||||||
|
# Check if download_url is a local file path (VidMoly M3U8 pre-download)
|
||||||
|
if os.path.exists(download_url):
|
||||||
|
logger.info(f"VidMoly already downloaded file to: {download_url}")
|
||||||
|
# Move file to expected location if different
|
||||||
|
import shutil
|
||||||
|
if download_url != task.file_path:
|
||||||
|
shutil.move(download_url, task.file_path)
|
||||||
|
logger.debug(f"Moved file to: {task.file_path}")
|
||||||
|
|
||||||
|
# Mark as complete
|
||||||
|
file_size = os.path.getsize(task.file_path)
|
||||||
|
logger.info(f"File size: {file_size / (1024*1024):.2f} MB")
|
||||||
|
task.status = DownloadStatus.COMPLETED
|
||||||
|
task.progress = 100.0
|
||||||
|
task.downloaded_bytes = file_size
|
||||||
|
task.total_bytes = file_size
|
||||||
|
task.completed_at = datetime.now()
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check if file already exists and is complete (for VidMoly which downloads directly)
|
||||||
|
if os.path.exists(task.file_path):
|
||||||
|
file_size = os.path.getsize(task.file_path)
|
||||||
|
if file_size > 1024: # More than 1KB - assume complete
|
||||||
|
logger.info(f"File already exists: {task.filename} ({file_size / (1024*1024):.2f} MB)")
|
||||||
|
task.status = DownloadStatus.COMPLETED
|
||||||
|
task.progress = 100.0
|
||||||
|
task.downloaded_bytes = file_size
|
||||||
|
task.total_bytes = file_size
|
||||||
|
task.completed_at = datetime.now()
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check for partial download (resume)
|
||||||
|
downloaded_bytes = 0
|
||||||
|
if os.path.exists(task.file_path):
|
||||||
|
downloaded_bytes = os.path.getsize(task.file_path)
|
||||||
|
|
||||||
|
headers = {}
|
||||||
|
# Add SendVid-specific headers to avoid 403 errors
|
||||||
|
if 'sendvid.com' in download_url:
|
||||||
|
headers.update({
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
|
||||||
|
'Referer': 'https://sendvid.com/',
|
||||||
|
})
|
||||||
|
# Add Sibnet-specific headers to avoid 403 errors
|
||||||
|
elif 'sibnet.ru' in download_url:
|
||||||
|
headers.update({
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||||
|
'Referer': 'https://video.sibnet.ru/',
|
||||||
|
'Accept': '*/*',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
|
})
|
||||||
|
if downloaded_bytes > 0:
|
||||||
|
headers['Range'] = f'bytes={downloaded_bytes}-'
|
||||||
|
|
||||||
|
# Download with streaming
|
||||||
|
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||||||
|
# First attempt with Range header if resuming
|
||||||
|
try:
|
||||||
|
async with client.stream('GET', download_url, headers=headers) as response:
|
||||||
|
response.raise_for_status()
|
||||||
|
# Process download (same code for both cases)
|
||||||
|
await self._process_download(response, task, downloaded_bytes)
|
||||||
|
except httpx.HTTPStatusError as e:
|
||||||
|
# If server doesn't support Range (416 error), restart from beginning
|
||||||
|
if e.response.status_code == 416 and downloaded_bytes > 0:
|
||||||
|
logger.info(f" Server doesn't support Range, restarting download: {task.filename}")
|
||||||
|
# Remove partial file and restart without Range header
|
||||||
|
if os.path.exists(task.file_path):
|
||||||
|
os.remove(task.file_path)
|
||||||
|
downloaded_bytes = 0
|
||||||
|
headers = {}
|
||||||
|
async with client.stream('GET', download_url, headers=headers) as response:
|
||||||
|
response.raise_for_status()
|
||||||
|
await self._process_download(response, task, downloaded_bytes)
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
task.status = DownloadStatus.FAILED
|
||||||
|
task.error = str(e)
|
||||||
|
finally:
|
||||||
|
if task.id in self.active_downloads:
|
||||||
|
del self.active_downloads[task.id]
|
||||||
|
|
||||||
|
async def _process_download(self, response, task: DownloadTask, downloaded_bytes: int):
|
||||||
|
"""Process the download response stream"""
|
||||||
|
# Log response info
|
||||||
|
logger.info(f" Response status: {response.status_code}")
|
||||||
|
logger.info(f" Response headers: {dict(response.headers)}")
|
||||||
|
|
||||||
|
# Get total size
|
||||||
|
if 'content-range' in response.headers:
|
||||||
|
# Resume mode
|
||||||
|
total_size = int(response.headers['content-range'].split('/')[-1])
|
||||||
|
else:
|
||||||
|
# New download
|
||||||
|
total_size = int(response.headers.get('content-length', 0))
|
||||||
|
downloaded_bytes = 0
|
||||||
|
|
||||||
|
task.total_bytes = total_size
|
||||||
|
|
||||||
|
# Write file
|
||||||
|
mode = 'ab' if downloaded_bytes > 0 else 'wb'
|
||||||
|
with open(task.file_path, mode) as f:
|
||||||
|
start_time = asyncio.get_event_loop().time()
|
||||||
|
|
||||||
|
async for chunk in response.aiter_bytes(chunk_size=1024 * 1024):
|
||||||
|
if task.status == DownloadStatus.CANCELLED:
|
||||||
|
return
|
||||||
|
|
||||||
|
if task.status == DownloadStatus.PAUSED:
|
||||||
|
return
|
||||||
|
|
||||||
|
f.write(chunk)
|
||||||
|
downloaded_bytes += len(chunk)
|
||||||
|
task.downloaded_bytes = downloaded_bytes
|
||||||
|
|
||||||
|
# Calculate progress
|
||||||
|
if total_size > 0:
|
||||||
|
task.progress = (downloaded_bytes / total_size) * 100
|
||||||
|
|
||||||
|
# Calculate speed
|
||||||
|
elapsed = asyncio.get_event_loop().time() - start_time
|
||||||
|
if elapsed > 0:
|
||||||
|
task.speed = downloaded_bytes / elapsed
|
||||||
|
|
||||||
|
task.status = DownloadStatus.COMPLETED
|
||||||
|
task.completed_at = datetime.now()
|
||||||
|
task.progress = 100.0
|
||||||
|
|
||||||
|
# Log completion info
|
||||||
|
final_size = os.path.getsize(task.file_path) if os.path.exists(task.file_path) else 0
|
||||||
|
logger.info(f" ✅ Completed: {task.filename} ({final_size / (1024*1024):.2f} MB)")
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
from .base import BaseDownloader
|
||||||
|
|
||||||
|
# Import from new organized structure
|
||||||
|
from .video_players import (
|
||||||
|
BaseVideoPlayer,
|
||||||
|
get_video_player,
|
||||||
|
DoodStreamDownloader,
|
||||||
|
SibnetDownloader,
|
||||||
|
VidMolyDownloader,
|
||||||
|
SendVidDownloader,
|
||||||
|
LpayerDownloader,
|
||||||
|
UnFichierDownloader,
|
||||||
|
UptoboxDownloader,
|
||||||
|
RapidFileDownloader
|
||||||
|
)
|
||||||
|
from .anime_sites import (
|
||||||
|
BaseAnimeSite,
|
||||||
|
get_anime_site,
|
||||||
|
AnimeSamaDownloader,
|
||||||
|
NekoSamaDownloader,
|
||||||
|
AnimeUltimeDownloader,
|
||||||
|
VostfreeDownloader
|
||||||
|
)
|
||||||
|
from .series_sites import (
|
||||||
|
BaseSeriesSite,
|
||||||
|
get_series_site,
|
||||||
|
FS7Downloader
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_downloader(url: str) -> BaseDownloader:
|
||||||
|
"""
|
||||||
|
Factory function to get the appropriate downloader for a URL.
|
||||||
|
|
||||||
|
This function now uses the organized structure:
|
||||||
|
- Checks anime sites first (for catalogs/search)
|
||||||
|
- Then checks series sites (for catalogs/search)
|
||||||
|
- Then checks video players (for direct download links)
|
||||||
|
- Falls back to generic downloader if no match
|
||||||
|
"""
|
||||||
|
# Try anime sites first
|
||||||
|
anime_site = get_anime_site(url)
|
||||||
|
if anime_site:
|
||||||
|
return anime_site
|
||||||
|
|
||||||
|
# Then try series sites
|
||||||
|
series_site = get_series_site(url)
|
||||||
|
if series_site:
|
||||||
|
return series_site
|
||||||
|
|
||||||
|
# Then try video players
|
||||||
|
video_player = get_video_player(url)
|
||||||
|
if video_player:
|
||||||
|
return video_player
|
||||||
|
|
||||||
|
# Return generic downloader if no match
|
||||||
|
return GenericDownloader()
|
||||||
|
|
||||||
|
|
||||||
|
class GenericDownloader(BaseDownloader):
|
||||||
|
"""Generic downloader for unhandled hosts"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: str = None) -> tuple[str, str]:
|
||||||
|
# Just return the URL as-is
|
||||||
|
filename = target_filename or url.split('/')[-1] or "download"
|
||||||
|
return url, filename
|
||||||
|
# Just return the URL as-is
|
||||||
|
filename = url.split('/')[-1] or "download"
|
||||||
|
return url, filename
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""Anime streaming sites (catalogs) downloaders"""
|
||||||
|
from .base import BaseAnimeSite
|
||||||
|
# Import all anime site downloaders
|
||||||
|
from .animesama import AnimeSamaDownloader
|
||||||
|
from .nekosama import NekoSamaDownloader
|
||||||
|
from .animeultime import AnimeUltimeDownloader
|
||||||
|
from .vostfree import VostfreeDownloader
|
||||||
|
from .frenchmanga import FrenchMangaDownloader
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseAnimeSite",
|
||||||
|
"AnimeSamaDownloader",
|
||||||
|
"NekoSamaDownloader",
|
||||||
|
"AnimeUltimeDownloader",
|
||||||
|
"VostfreeDownloader",
|
||||||
|
"FrenchMangaDownloader",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_anime_site(url: str) -> BaseAnimeSite:
|
||||||
|
"""Factory function to get the appropriate anime site for a URL"""
|
||||||
|
sites = [
|
||||||
|
AnimeSamaDownloader(),
|
||||||
|
AnimeUltimeDownloader(),
|
||||||
|
NekoSamaDownloader(),
|
||||||
|
VostfreeDownloader(),
|
||||||
|
FrenchMangaDownloader(),
|
||||||
|
]
|
||||||
|
|
||||||
|
for site in sites:
|
||||||
|
if site.can_handle(url):
|
||||||
|
return site
|
||||||
|
|
||||||
|
# Return None if no match (should not happen in normal flow)
|
||||||
|
return None
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,435 @@
|
|||||||
|
from .base import BaseAnimeSite
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
import httpx
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
|
||||||
|
class AnimeUltimeDownloader(BaseAnimeSite):
|
||||||
|
"""Downloader for anime-ultime.net"""
|
||||||
|
|
||||||
|
BASE_DOMAINS = ["anime-ultime.com", "anime-ultime.net", "www.anime-ultime.net"]
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return any(domain in url.lower() for domain in self.BASE_DOMAINS)
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract download link from anime-ultime URL
|
||||||
|
Anime-Ultime stores video links in og:video meta tags
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Follow redirects
|
||||||
|
response = await self.client.get(url, follow_redirects=True)
|
||||||
|
final_url = str(response.url)
|
||||||
|
|
||||||
|
# Parse the page
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Method 0: Look for og:video meta tag (most reliable for anime-ultime)
|
||||||
|
og_video = soup.find('meta', property='og:video')
|
||||||
|
if og_video and og_video.get('content'):
|
||||||
|
video_url = og_video['content']
|
||||||
|
if video_url.endswith('.mp4'):
|
||||||
|
filename = self._generate_filename(final_url)
|
||||||
|
print(f"[ANIME-ULTIME] Found og:video link: {video_url}")
|
||||||
|
return video_url, filename
|
||||||
|
|
||||||
|
# Method 1: Look for direct download links (DDL)
|
||||||
|
# Anime-Ultime often uses links to file hosts
|
||||||
|
download_links = soup.find_all('a', href=True)
|
||||||
|
for link in download_links:
|
||||||
|
href = link['href']
|
||||||
|
text = link.get_text().lower()
|
||||||
|
|
||||||
|
# Look for download buttons/links
|
||||||
|
if any(keyword in text for keyword in ['télécharger', 'download', 'ddl', 'mega', 'google', 'drive']):
|
||||||
|
# Check if it's a direct link or to a file host
|
||||||
|
if any(host in href.lower() for host in ['mega.nz', 'drive.google.com', 'uptobox.com', '1fichier.com']):
|
||||||
|
filename = self._generate_filename(final_url)
|
||||||
|
return href, filename
|
||||||
|
|
||||||
|
# Method 2: Look for iframe with video player
|
||||||
|
iframes = soup.find_all('iframe')
|
||||||
|
for iframe in iframes:
|
||||||
|
src = iframe.get('src', '')
|
||||||
|
if src and any(provider in src for provider in ['video', 'player', 'stream', 'play']):
|
||||||
|
if src.startswith('http'):
|
||||||
|
filename = self._generate_filename(final_url)
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
# Method 3: Look for video tags
|
||||||
|
videos = soup.find_all('video')
|
||||||
|
for video in videos:
|
||||||
|
src = video.get('src', '')
|
||||||
|
if src:
|
||||||
|
filename = self._generate_filename(final_url)
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
# Check source tags
|
||||||
|
sources = video.find_all('source')
|
||||||
|
for source in sources:
|
||||||
|
src = source.get('src', '')
|
||||||
|
if src:
|
||||||
|
filename = self._generate_filename(final_url)
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
# Method 4: Look in scripts for video URLs
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
# Look for common video patterns
|
||||||
|
patterns = [
|
||||||
|
r'(https?://[^"\'>\s]+\.(?:mp4|m3u8|mkv)(?:\?[^"\'>\s]*)?)',
|
||||||
|
r'"url":"([^"]+)"',
|
||||||
|
r'"video":"([^"]+)"',
|
||||||
|
r'"file":"([^"]+)"',
|
||||||
|
r'file:\s*"([^"]+)"',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, script.string)
|
||||||
|
for match in matches:
|
||||||
|
# Clean up escaped characters
|
||||||
|
match = match.replace('\\/', '/').replace('\\', '')
|
||||||
|
if any(ext in match for ext in ['mp4', 'm3u8', 'mkv']):
|
||||||
|
filename = self._generate_filename(final_url)
|
||||||
|
return match, filename
|
||||||
|
|
||||||
|
# Look for anime-ultime specific patterns
|
||||||
|
# They sometimes store links in JavaScript variables
|
||||||
|
ddl_match = re.search(r'ddl["\']?\s*:\s*["\']([^"\']+)["\']', script.string)
|
||||||
|
if ddl_match:
|
||||||
|
ddl_url = ddl_match.group(1)
|
||||||
|
if ddl_url.startswith('http'):
|
||||||
|
filename = self._generate_filename(final_url)
|
||||||
|
return ddl_url, filename
|
||||||
|
|
||||||
|
# Method 5: Look for links with specific classes or IDs
|
||||||
|
# Anime-Ultime might use specific class names for download links
|
||||||
|
potential_links = soup.find_all('a', class_=re.compile(r'download|ddl|episode', re.I))
|
||||||
|
for link in potential_links:
|
||||||
|
href = link.get('href', '')
|
||||||
|
if href and href.startswith('http'):
|
||||||
|
filename = self._generate_filename(final_url)
|
||||||
|
return href, filename
|
||||||
|
|
||||||
|
# If nothing found, raise error
|
||||||
|
raise Exception("Could not find download link on page")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Anime-Ultime link: {str(e)}")
|
||||||
|
|
||||||
|
def _generate_filename(self, url: str) -> str:
|
||||||
|
"""Generate filename from URL"""
|
||||||
|
# Extract anime name and episode from URL
|
||||||
|
# URL formats:
|
||||||
|
# - info-0-1/30200
|
||||||
|
# - info-0-1/30200/Naruto-OAV-01-vostfr
|
||||||
|
# - file-0-1/2991-Naruto-OAV
|
||||||
|
|
||||||
|
anime_name = "Anime"
|
||||||
|
episode = "01"
|
||||||
|
|
||||||
|
# Format: info-0-1/EPISODE_ID or info-0-1/EPISODE_ID/NAME-EP-vostfr
|
||||||
|
if 'info-0-1/' in url:
|
||||||
|
# Extract episode ID
|
||||||
|
ep_match = re.search(r'info-0-1/(\d+)', url)
|
||||||
|
if ep_match:
|
||||||
|
ep_id = ep_match.group(1)
|
||||||
|
|
||||||
|
# Try to get anime name from URL path
|
||||||
|
name_match = re.search(r'info-0-1/\d+/([^/]+)', url)
|
||||||
|
if name_match:
|
||||||
|
raw_name = name_match.group(1)
|
||||||
|
# Extract episode number
|
||||||
|
ep_num_match = re.search(r'-(\d+)-vostfr$', raw_name, re.I)
|
||||||
|
if ep_num_match:
|
||||||
|
episode = ep_num_match.group(1).zfill(2)
|
||||||
|
# Remove episode number and suffix from name
|
||||||
|
anime_name = re.sub(r'-\d+-vostfr$', '', raw_name, flags=re.I).replace('-', ' ')
|
||||||
|
else:
|
||||||
|
# Just use the ID
|
||||||
|
anime_name = f"Episode {ep_id}"
|
||||||
|
else:
|
||||||
|
anime_name = f"Episode {ep_id}"
|
||||||
|
|
||||||
|
elif 'file-0-1/' in url:
|
||||||
|
# Extract from file-0-1/ID-NAME format
|
||||||
|
file_match = re.search(r'file-0-1/\d+-(.+)$', url)
|
||||||
|
if file_match:
|
||||||
|
anime_name = file_match.group(1).replace('-', ' ')
|
||||||
|
|
||||||
|
# Sanitize filename
|
||||||
|
anime_name = anime_name.replace('/', ' ').strip()
|
||||||
|
filename = f"{anime_name} - Episode {episode}.mp4"
|
||||||
|
return filename.title()
|
||||||
|
|
||||||
|
async def get_anime_metadata(self, anime_url: str) -> dict:
|
||||||
|
"""
|
||||||
|
Extract rich metadata from anime page
|
||||||
|
Returns synopsis, genres, rating, release year, studio, etc.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
print(f"[ANIME-ULTIME] Extracting metadata from: {anime_url}")
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
metadata = {
|
||||||
|
'synopsis': None,
|
||||||
|
'genres': [],
|
||||||
|
'rating': None,
|
||||||
|
'release_year': None,
|
||||||
|
'studio': None,
|
||||||
|
'poster_image': None,
|
||||||
|
'banner_image': None,
|
||||||
|
'total_episodes': None,
|
||||||
|
'status': None,
|
||||||
|
'alternative_titles': []
|
||||||
|
}
|
||||||
|
|
||||||
|
# Extract synopsis
|
||||||
|
synopsis_selectors = [
|
||||||
|
'div.synopsis',
|
||||||
|
'div.description',
|
||||||
|
'div[class*="synopsis"]',
|
||||||
|
'div[class*="synopsis"]',
|
||||||
|
'p.synopsis',
|
||||||
|
'.info',
|
||||||
|
'div.texte'
|
||||||
|
]
|
||||||
|
|
||||||
|
for selector in synopsis_selectors:
|
||||||
|
synopsis_elem = soup.select_one(selector)
|
||||||
|
if synopsis_elem:
|
||||||
|
synopsis = synopsis_elem.get_text(strip=True)
|
||||||
|
if len(synopsis) > 50:
|
||||||
|
metadata['synopsis'] = synopsis
|
||||||
|
break
|
||||||
|
|
||||||
|
# Extract genres from meta tags and page content
|
||||||
|
page_text = soup.get_text()
|
||||||
|
|
||||||
|
# Look for genre in meta tags
|
||||||
|
genre_meta = soup.find('meta', property='genre') or soup.find('meta', attrs={'name': 'genre'})
|
||||||
|
if genre_meta:
|
||||||
|
genres_text = genre_meta.get('content', '')
|
||||||
|
if genres_text:
|
||||||
|
metadata['genres'] = [g.strip() for g in genres_text.split(',')]
|
||||||
|
|
||||||
|
# Try to find genre links
|
||||||
|
genre_links = soup.find_all('a', href=re.compile(r'genre|tag|type|cat', re.I))
|
||||||
|
if genre_links:
|
||||||
|
for link in genre_links[:5]:
|
||||||
|
genre = link.get_text(strip=True)
|
||||||
|
if genre and genre not in metadata['genres']:
|
||||||
|
metadata['genres'].append(genre)
|
||||||
|
|
||||||
|
# Extract rating
|
||||||
|
rating_selectors = [
|
||||||
|
'span.rating',
|
||||||
|
'div.rating',
|
||||||
|
'span.score',
|
||||||
|
'div.note',
|
||||||
|
'.rating'
|
||||||
|
]
|
||||||
|
|
||||||
|
for selector in rating_selectors:
|
||||||
|
rating_elem = soup.select_one(selector)
|
||||||
|
if rating_elem:
|
||||||
|
rating_text = rating_elem.get_text(strip=True)
|
||||||
|
rating_match = re.search(r'(\d+\.?\d*)\s*/\s*10', rating_text)
|
||||||
|
if rating_match:
|
||||||
|
metadata['rating'] = f"{rating_match.group(1)}/10"
|
||||||
|
break
|
||||||
|
rating_match = re.search(r'(\d+\.?\d*)\s*/\s*5', rating_text)
|
||||||
|
if rating_match:
|
||||||
|
rating_val = float(rating_match.group(1)) * 2
|
||||||
|
metadata['rating'] = f"{rating_val:.1f}/10"
|
||||||
|
break
|
||||||
|
|
||||||
|
# Extract release year
|
||||||
|
year_match = re.search(r'\b(19\d{2}|20\d{2})\b', page_text)
|
||||||
|
if year_match:
|
||||||
|
import datetime
|
||||||
|
current_year = datetime.datetime.now().year + 2
|
||||||
|
year = int(year_match.group(1))
|
||||||
|
if 1950 <= year <= current_year:
|
||||||
|
metadata['release_year'] = year
|
||||||
|
|
||||||
|
# Extract poster image from og:image
|
||||||
|
og_image = soup.find('meta', property='og:image')
|
||||||
|
if og_image:
|
||||||
|
metadata['poster_image'] = og_image.get('content')
|
||||||
|
|
||||||
|
# Extract total episodes
|
||||||
|
episodes_count = len(await self.get_episodes(anime_url))
|
||||||
|
if episodes_count > 0:
|
||||||
|
metadata['total_episodes'] = episodes_count
|
||||||
|
|
||||||
|
print(f"[ANIME-ULTIME] Extracted metadata: {metadata}")
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ANIME-ULTIME] Error extracting metadata: {e}")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
async def search_anime(self, query: str, lang: str = "vostfr", include_metadata: bool = False) -> list[dict]:
|
||||||
|
"""
|
||||||
|
Search for anime on anime-ultime
|
||||||
|
Returns list of anime with title, url, and cover image
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query string
|
||||||
|
lang: Language preference (vostfr, vf)
|
||||||
|
include_metadata: Whether to fetch full metadata for each result (slower)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import time
|
||||||
|
start = time.time()
|
||||||
|
print(f"[ANIME-ULTIME] Searching for '{query}' ({lang})...")
|
||||||
|
|
||||||
|
# Anime-Ultime uses POST for search
|
||||||
|
search_url = "https://www.anime-ultime.net/search-0-1"
|
||||||
|
|
||||||
|
response = await self.client.post(search_url, data={'search': query})
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
elapsed = time.time() - start
|
||||||
|
print(f"[ANIME-ULTIME] Got response {response.status_code} in {elapsed:.2f}s")
|
||||||
|
|
||||||
|
results = []
|
||||||
|
|
||||||
|
# Look for search result links - better parsing
|
||||||
|
# Search results use file-0-1/ pattern, not info-
|
||||||
|
search_results = soup.find_all('a', href=re.compile(r'file-0-1/'))
|
||||||
|
|
||||||
|
seen_urls = set()
|
||||||
|
for result in search_results[:10]: # Limit to 10 results
|
||||||
|
href = result.get('href', '')
|
||||||
|
raw_title = result.get_text().strip()
|
||||||
|
|
||||||
|
# Skip if no href
|
||||||
|
if not href:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Skip duplicates
|
||||||
|
if href in seen_urls:
|
||||||
|
continue
|
||||||
|
seen_urls.add(href)
|
||||||
|
|
||||||
|
# Extract better title from URL or parent elements
|
||||||
|
better_title = raw_title
|
||||||
|
|
||||||
|
# If raw_title is just "Télécharger" or similar, try to find better title
|
||||||
|
if len(raw_title) < 5 or raw_title.lower() in ['télécharger', 'download', 'ddl']:
|
||||||
|
# Try to extract from URL (file-0-1/ID-Title format)
|
||||||
|
url_match = re.search(r'file-0-1/\d+-(.+)$', href)
|
||||||
|
if url_match:
|
||||||
|
better_title = url_match.group(1).replace('-', ' ').title()
|
||||||
|
|
||||||
|
# If still no good title, look at parent/row elements
|
||||||
|
if len(better_title) < 5:
|
||||||
|
# Check parent row (table structure)
|
||||||
|
row = result.find_parent(['tr', 'td', 'div'])
|
||||||
|
if row:
|
||||||
|
# Look for text in the row that's not the link text
|
||||||
|
row_text = row.get_text().strip()
|
||||||
|
# Remove the link text from row text
|
||||||
|
if raw_title in row_text:
|
||||||
|
row_text = row_text.replace(raw_title, '').strip()
|
||||||
|
if len(row_text) > 5 and len(row_text) < 100:
|
||||||
|
better_title = row_text
|
||||||
|
|
||||||
|
# Make URL absolute
|
||||||
|
if not href.startswith('http'):
|
||||||
|
href = urljoin("https://www.anime-ultime.net/", href)
|
||||||
|
|
||||||
|
result_item = {
|
||||||
|
'title': better_title,
|
||||||
|
'url': href,
|
||||||
|
'type': 'search_result',
|
||||||
|
'metadata': None
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fetch metadata if requested
|
||||||
|
if include_metadata:
|
||||||
|
metadata = await self.get_anime_metadata(href)
|
||||||
|
result_item['metadata'] = metadata
|
||||||
|
|
||||||
|
results.append(result_item)
|
||||||
|
|
||||||
|
print(f"[ANIME-ULTIME] Found {len(results)} results")
|
||||||
|
return results
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ANIME-ULTIME] Error: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_episodes(self, anime_url: str, lang: str = "vostfr") -> list[dict]:
|
||||||
|
"""
|
||||||
|
Get list of episodes for an anime
|
||||||
|
Returns list of episode numbers and their URLs
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
episodes = []
|
||||||
|
|
||||||
|
# Look for episode links - anime-ultime uses info-XXXXX-Name-XX-vostfr format
|
||||||
|
# The URL pattern is info-0-1/ID-Anime-Name-XX-vostfr where XX is episode number
|
||||||
|
episode_links = soup.find_all('a', href=re.compile(r'info-0-1/\d+'))
|
||||||
|
|
||||||
|
for link in episode_links:
|
||||||
|
href = link.get('href', '')
|
||||||
|
text = link.get_text().strip()
|
||||||
|
|
||||||
|
# Extract episode number from URL pattern
|
||||||
|
# Matches: info-0-1/30200/Naruto-OAV-01-vostfr
|
||||||
|
match = re.search(r'-(\d+)-vostfr$', href, re.I)
|
||||||
|
if not match:
|
||||||
|
# Try other patterns
|
||||||
|
match = re.search(r'Episode[-\s]?(\d+)', href, re.I)
|
||||||
|
if not match:
|
||||||
|
# Try to extract from text
|
||||||
|
match = re.search(r'(\d+)', text)
|
||||||
|
|
||||||
|
if match:
|
||||||
|
episode_num = match.group(1).zfill(2) # Pad with zero
|
||||||
|
|
||||||
|
# Extract the episode ID from href and build correct URL
|
||||||
|
# href might be "info-0-1/30200" or "info-0-1/30200/..."
|
||||||
|
# We need: https://www.anime-ultime.net/info-0-1/30200
|
||||||
|
ep_id_match = re.search(r'info-0-1/(\d+)', href)
|
||||||
|
if ep_id_match:
|
||||||
|
ep_id = ep_id_match.group(1)
|
||||||
|
# Build the correct episode URL
|
||||||
|
episode_url = f"https://www.anime-ultime.net/info-0-1/{ep_id}"
|
||||||
|
else:
|
||||||
|
# Fallback to making URL absolute
|
||||||
|
if not href.startswith('http'):
|
||||||
|
href = urljoin(anime_url, href)
|
||||||
|
episode_url = href
|
||||||
|
|
||||||
|
episodes.append({
|
||||||
|
'episode': episode_num,
|
||||||
|
'url': episode_url,
|
||||||
|
'title': text
|
||||||
|
})
|
||||||
|
|
||||||
|
# Remove duplicates and sort
|
||||||
|
seen = set()
|
||||||
|
unique_episodes = []
|
||||||
|
for ep in episodes:
|
||||||
|
if ep['episode'] not in seen:
|
||||||
|
seen.add(ep['episode'])
|
||||||
|
unique_episodes.append(ep)
|
||||||
|
|
||||||
|
unique_episodes.sort(key=lambda x: int(x['episode']))
|
||||||
|
|
||||||
|
return unique_episodes
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error getting episodes: {e}")
|
||||||
|
return []
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
"""Base class for anime streaming sites (catalogs)"""
|
||||||
|
from abc import abstractmethod
|
||||||
|
from typing import List, Dict, Any, Optional, Tuple
|
||||||
|
import logging
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class BaseAnimeSite:
|
||||||
|
"""
|
||||||
|
Base class for anime streaming sites.
|
||||||
|
|
||||||
|
Anime sites provide catalogs, metadata, and episode listings.
|
||||||
|
They typically link to video players for actual file hosting.
|
||||||
|
|
||||||
|
Examples: Anime-Sama, Neko-Sama, Anime-Ultime, Vostfree, etc.
|
||||||
|
|
||||||
|
KEY FEATURE: Provides rich metadata and episode management
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# Realistic browser headers to avoid blocking by video hosts
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9,fr;q=0.8",
|
||||||
|
"Referer": "https://anime-sama.tv/",
|
||||||
|
}
|
||||||
|
# Initialize HTTP client with browser headers
|
||||||
|
self.client = httpx.AsyncClient(timeout=10.0, follow_redirects=True, headers=headers)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this anime site can handle the given URL"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def search_anime(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
lang: str = "vostfr"
|
||||||
|
) -> List[Dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Search for anime on this site.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query (anime title)
|
||||||
|
lang: Language preference (vostfr, vf)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of anime with keys:
|
||||||
|
- title: Anime title
|
||||||
|
- url: Anime page URL
|
||||||
|
- cover_image: Optional cover image URL
|
||||||
|
- lang: Available languages
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_episodes(
|
||||||
|
self,
|
||||||
|
anime_url: str,
|
||||||
|
lang: str = "vostfr"
|
||||||
|
) -> List[Dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Get list of episodes for an anime.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_url: URL of the anime page
|
||||||
|
lang: Language preference
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of episodes with keys:
|
||||||
|
- episode_number: Episode number
|
||||||
|
- url: Episode page URL
|
||||||
|
- title: Optional episode title
|
||||||
|
- host: Video player hosting the file
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_anime_metadata(self, anime_url: str) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Get detailed metadata for an anime.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_url: URL of the anime page
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with metadata:
|
||||||
|
- title: Anime title
|
||||||
|
- synopsis: Plot summary
|
||||||
|
- genres: List of genres
|
||||||
|
- rating: Rating (e.g., "8.5/10")
|
||||||
|
- release_year: Release year
|
||||||
|
- studio: Animation studio
|
||||||
|
- poster_image: Poster URL
|
||||||
|
- total_episodes: Total episode count
|
||||||
|
- status: Airing status (ongoing, completed)
|
||||||
|
- languages: Available languages
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_download_link(self, url: str) -> Tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Get download link for a specific episode.
|
||||||
|
|
||||||
|
For anime sites, this extracts the video player URL from an episode page.
|
||||||
|
Note: Returns video player URL, NOT direct download link!
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (video_player_url, episode_title)
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Common methods for all anime sites
|
||||||
|
async def close(self):
|
||||||
|
"""Close HTTP client"""
|
||||||
|
await self.client.aclose()
|
||||||
|
|
||||||
|
async def _fetch_page(self, url: str) -> str:
|
||||||
|
"""Fetch HTML page content"""
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.text
|
||||||
|
|
||||||
|
def _parse_html(self, html: str) -> BeautifulSoup:
|
||||||
|
"""Parse HTML with BeautifulSoup"""
|
||||||
|
return BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
def _extract_season_number(self, title: str) -> Optional[int]:
|
||||||
|
"""Extract season number from title (e.g., 'Saison 2' -> 2)"""
|
||||||
|
import re
|
||||||
|
match = re.search(r'saison\s*(\d+)', title.lower())
|
||||||
|
return int(match.group(1)) if match else None
|
||||||
@@ -0,0 +1,299 @@
|
|||||||
|
"""French-Manga.net anime streaming site downloader"""
|
||||||
|
from .base import BaseAnimeSite
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class FrenchMangaDownloader(BaseAnimeSite):
|
||||||
|
"""Downloader for french-manga.net anime streaming site"""
|
||||||
|
|
||||||
|
# Known domains for French-Manga
|
||||||
|
BASE_DOMAINS = [
|
||||||
|
"french-manga.net",
|
||||||
|
"w16.french-manga.net",
|
||||||
|
"w15.french-manga.net",
|
||||||
|
"www.french-manga.net"
|
||||||
|
]
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.base_url = "https://w16.french-manga.net"
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this downloader can handle the given URL"""
|
||||||
|
return any(domain in url.lower() for domain in self.BASE_DOMAINS)
|
||||||
|
|
||||||
|
async def search_anime(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
lang: str = "vostfr"
|
||||||
|
) -> List[Dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Search for anime on French-Manga.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query (anime title)
|
||||||
|
lang: Language preference (vostfr, vf)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of anime with title, url, cover_image
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# French-Manga uses a search endpoint
|
||||||
|
search_url = f"{self.base_url}/index.php?do=search"
|
||||||
|
params = {
|
||||||
|
'do': 'search',
|
||||||
|
'subaction': 'search',
|
||||||
|
'story': query,
|
||||||
|
'x': '0',
|
||||||
|
'y': '0'
|
||||||
|
}
|
||||||
|
|
||||||
|
response = await self.client.post(search_url, data=params)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
results = []
|
||||||
|
|
||||||
|
# Look for search results in article or story classes
|
||||||
|
for item in soup.find_all('article', class_=lambda x: x and 'story' in x.lower()):
|
||||||
|
title_elem = item.find(['h2', 'h3', 'h4'])
|
||||||
|
link_elem = item.find('a', href=True)
|
||||||
|
img_elem = item.find('img')
|
||||||
|
|
||||||
|
if title_elem and link_elem:
|
||||||
|
title = title_elem.get_text(strip=True)
|
||||||
|
url = link_elem['href']
|
||||||
|
|
||||||
|
# Ensure absolute URL
|
||||||
|
if url.startswith('/'):
|
||||||
|
url = self.base_url + url
|
||||||
|
|
||||||
|
cover_image = ""
|
||||||
|
if img_elem and img_elem.get('src'):
|
||||||
|
cover_image = img_elem['src']
|
||||||
|
if cover_image.startswith('/'):
|
||||||
|
cover_image = self.base_url + cover_image
|
||||||
|
|
||||||
|
results.append({
|
||||||
|
'title': title,
|
||||||
|
'url': url,
|
||||||
|
'cover_image': cover_image,
|
||||||
|
'lang': lang
|
||||||
|
})
|
||||||
|
|
||||||
|
logger.info(f"Found {len(results)} anime results for query: {query}")
|
||||||
|
return results
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error searching anime: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_episodes(
|
||||||
|
self,
|
||||||
|
anime_url: str,
|
||||||
|
lang: str = "vostfr"
|
||||||
|
) -> List[Dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Get episode list for an anime.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_url: URL of the anime page
|
||||||
|
lang: Language preference
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of episodes with episode_number, url, title
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
episodes = []
|
||||||
|
|
||||||
|
# Look for episode links (typically in a list or table)
|
||||||
|
# French-Manga usually has episode links in <a> tags with episode numbers
|
||||||
|
for link in soup.find_all('a', href=True):
|
||||||
|
href = link['href']
|
||||||
|
text = link.get_text(strip=True)
|
||||||
|
|
||||||
|
# Pattern: Episode links usually contain "episode" or numbers
|
||||||
|
if re.search(r'episode?\s*\d+', text.lower()):
|
||||||
|
episode_num = re.search(r'(\d+)', text)
|
||||||
|
if episode_num:
|
||||||
|
episode_number = int(episode_num.group(1))
|
||||||
|
|
||||||
|
# Ensure absolute URL
|
||||||
|
if href.startswith('/'):
|
||||||
|
href = self.base_url + href
|
||||||
|
|
||||||
|
episodes.append({
|
||||||
|
'episode_number': episode_number,
|
||||||
|
'url': href,
|
||||||
|
'title': text,
|
||||||
|
'host': 'french-manga'
|
||||||
|
})
|
||||||
|
|
||||||
|
# Sort by episode number
|
||||||
|
episodes.sort(key=lambda x: x['episode_number'])
|
||||||
|
|
||||||
|
logger.info(f"Found {len(episodes)} episodes for {anime_url}")
|
||||||
|
return episodes
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting episodes: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_anime_metadata(self, anime_url: str) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Get detailed metadata for an anime.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_url: URL of the anime page
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with metadata (synopsis, genres, rating, etc.)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
# Extract title
|
||||||
|
title = ""
|
||||||
|
title_elem = soup.find('h1') or soup.find('h2', class_='title')
|
||||||
|
if title_elem:
|
||||||
|
title = title_elem.get_text(strip=True)
|
||||||
|
|
||||||
|
# Extract synopsis
|
||||||
|
synopsis = ""
|
||||||
|
synopsis_elem = soup.find('div', class_=lambda x: x and 'story' in x.lower())
|
||||||
|
if synopsis_elem:
|
||||||
|
synopsis = synopsis_elem.get_text(strip=True)
|
||||||
|
|
||||||
|
# Extract cover image
|
||||||
|
poster_image = ""
|
||||||
|
img_elem = soup.find('img', class_=lambda x: x and 'poster' in x.lower())
|
||||||
|
if img_elem and img_elem.get('src'):
|
||||||
|
poster_image = img_elem['src']
|
||||||
|
if poster_image.startswith('/'):
|
||||||
|
poster_image = self.base_url + poster_image
|
||||||
|
|
||||||
|
# Extract genres
|
||||||
|
genres = []
|
||||||
|
genre_links = soup.find_all('a', href=re.compile(r'/xfsearch/.*genre/'))
|
||||||
|
for link in genre_links[:10]: # Limit to 10 genres
|
||||||
|
genre = link.get_text(strip=True)
|
||||||
|
if genre:
|
||||||
|
genres.append(genre)
|
||||||
|
|
||||||
|
# Extract rating (if available)
|
||||||
|
rating = ""
|
||||||
|
rating_elem = soup.find(['span', 'div'], class_=lambda x: x and 'rating' in x.lower())
|
||||||
|
if rating_elem:
|
||||||
|
rating = rating_elem.get_text(strip=True)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'title': title,
|
||||||
|
'synopsis': synopsis,
|
||||||
|
'genres': genres,
|
||||||
|
'rating': rating,
|
||||||
|
'release_year': '',
|
||||||
|
'studio': '',
|
||||||
|
'poster_image': poster_image,
|
||||||
|
'total_episodes': len(await self.get_episodes(anime_url)),
|
||||||
|
'status': '',
|
||||||
|
'languages': ['vf', 'vostfr']
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting anime metadata: {e}")
|
||||||
|
return {
|
||||||
|
'title': '',
|
||||||
|
'synopsis': '',
|
||||||
|
'genres': [],
|
||||||
|
'rating': '',
|
||||||
|
'release_year': '',
|
||||||
|
'studio': '',
|
||||||
|
'poster_image': '',
|
||||||
|
'total_episodes': 0,
|
||||||
|
'status': '',
|
||||||
|
'languages': ['vf', 'vostfr']
|
||||||
|
}
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Get download link from episode page.
|
||||||
|
|
||||||
|
For French-Manga, this returns the video player URL.
|
||||||
|
The actual video extraction will be handled by the video player downloaders.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: Episode page URL
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (video_player_url, episode_title)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
# Look for iframe or video player
|
||||||
|
iframe = soup.find('iframe', src=True)
|
||||||
|
if iframe:
|
||||||
|
video_url = iframe['src']
|
||||||
|
else:
|
||||||
|
# Look for video tag directly
|
||||||
|
video = soup.find('video', src=True)
|
||||||
|
if video:
|
||||||
|
video_url = video['src']
|
||||||
|
else:
|
||||||
|
# Try to find in script tags
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
# Look for iframe or video URLs in JavaScript
|
||||||
|
patterns = [
|
||||||
|
r'iframe.*?src=["\']([^"\']+)["\']',
|
||||||
|
r'video.*?src=["\']([^"\']+)["\']',
|
||||||
|
]
|
||||||
|
for pattern in patterns:
|
||||||
|
match = re.search(pattern, script.string, re.IGNORECASE)
|
||||||
|
if match:
|
||||||
|
video_url = match.group(1)
|
||||||
|
break
|
||||||
|
if 'video_url' in locals():
|
||||||
|
break
|
||||||
|
|
||||||
|
if 'video_url' not in locals():
|
||||||
|
raise ValueError("Could not find video player URL")
|
||||||
|
|
||||||
|
# Ensure absolute URL
|
||||||
|
if video_url.startswith('//'):
|
||||||
|
video_url = 'https:' + video_url
|
||||||
|
elif video_url.startswith('/'):
|
||||||
|
video_url = self.base_url + video_url
|
||||||
|
|
||||||
|
# Extract episode title
|
||||||
|
title_elem = soup.find('h1') or soup.find('h2')
|
||||||
|
episode_title = title_elem.get_text(strip=True) if title_elem else "Episode"
|
||||||
|
episode_title = sanitize_filename(episode_title)
|
||||||
|
|
||||||
|
logger.info(f"Extracted video player URL: {video_url[:60]}...")
|
||||||
|
return video_url, episode_title
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting download link: {e}")
|
||||||
|
raise ValueError(f"Failed to extract download link: {str(e)}")
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
from .base import BaseAnimeSite
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
from typing import Optional
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
|
||||||
|
class NekoSamaDownloader(BaseAnimeSite):
|
||||||
|
"""Downloader for neko-sama.org (anime streaming via Gupy)
|
||||||
|
|
||||||
|
NOTE: neko-sama.org now redirects to Gupy, which is a legal streaming search engine.
|
||||||
|
It does NOT host video content - it provides metadata about where to watch legally.
|
||||||
|
This provider can search and get metadata but cannot provide direct download links.
|
||||||
|
"""
|
||||||
|
|
||||||
|
BASE_DOMAINS = ["neko-sama.org", "www.neko-sama.org", "neko-sama.fr", "nekosama.fr", "www.gupy.fr", "gupy.fr"]
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return any(domain in url.lower() for domain in self.BASE_DOMAINS)
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: Optional[str] = None) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract download link from neko-sama URL.
|
||||||
|
|
||||||
|
NOTE: neko-sama.org/Gupy is a legal streaming search engine, NOT a video host.
|
||||||
|
This returns streaming platform information instead of direct video links.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Check if this is a Gupy URL
|
||||||
|
if 'gupy.fr' in url or 'neko-sama.org' in url:
|
||||||
|
response = await self.client.get(url, follow_redirects=True)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Look for streaming platform links
|
||||||
|
streaming_links = []
|
||||||
|
for link in soup.find_all('a', href=True):
|
||||||
|
href = link.get('href', '')
|
||||||
|
if '/out/' in href:
|
||||||
|
text = link.get_text(strip=True)
|
||||||
|
if text and 'Regarder' in text:
|
||||||
|
streaming_links.append(f"{text}: {href}")
|
||||||
|
|
||||||
|
if streaming_links:
|
||||||
|
title_elem = soup.find('h1') or soup.find('title')
|
||||||
|
title = title_elem.get_text(strip=True).split('|')[0].strip() if title_elem else "Unknown"
|
||||||
|
info = "Available streaming platforms:\n" + "\n".join(streaming_links[:5])
|
||||||
|
filename = target_filename or f"{title}_streaming_info.txt"
|
||||||
|
return info, filename
|
||||||
|
|
||||||
|
raise Exception("No streaming links found - Gupy is a legal streaming search, not a video host")
|
||||||
|
|
||||||
|
# Legacy: try original method for other URLs
|
||||||
|
response = await self.client.get(url, follow_redirects=True)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Method 1: Look for iframes with video
|
||||||
|
iframes = soup.find_all('iframe')
|
||||||
|
for iframe in iframes:
|
||||||
|
src = iframe.get('src', '')
|
||||||
|
if src and any(p in src for p in ['video', 'player', 'stream']):
|
||||||
|
if not src.startswith('http'):
|
||||||
|
src = urljoin(str(response.url), src)
|
||||||
|
filename = self._generate_filename(str(response.url))
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
# Method 2: Look for video tags
|
||||||
|
videos = soup.find_all('video')
|
||||||
|
for video in videos:
|
||||||
|
src = video.get('src') or video.get('data-src')
|
||||||
|
if src:
|
||||||
|
filename = self._generate_filename(str(response.url))
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
sources = video.find_all('source')
|
||||||
|
for source in sources:
|
||||||
|
src = source.get('src', '')
|
||||||
|
if src:
|
||||||
|
filename = self._generate_filename(str(response.url))
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
# Method 3: Look in scripts
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
patterns = [
|
||||||
|
r'(https?://[^"\'>\s]+\.(?:mp4|m3u8)(?:\?[^"\'>\s]*)?)',
|
||||||
|
r'"url":"([^"]+)"',
|
||||||
|
r'"video":"([^"]+)"',
|
||||||
|
]
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, script.string)
|
||||||
|
for match in matches:
|
||||||
|
match = match.replace('\\/', '/')
|
||||||
|
if any(ext in match for ext in ['mp4', 'm3u8']):
|
||||||
|
filename = self._generate_filename(str(response.url))
|
||||||
|
return match, filename
|
||||||
|
|
||||||
|
raise Exception("Could not find video link - Neko-Sama/Gupy does not host video content")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting NekoSama link: {str(e)}")
|
||||||
|
|
||||||
|
def _generate_filename(self, url: str) -> str:
|
||||||
|
parts = url.split('/')
|
||||||
|
anime_name = "anime"
|
||||||
|
episode = "1"
|
||||||
|
|
||||||
|
for i, part in enumerate(parts):
|
||||||
|
if 'episode' in part.lower():
|
||||||
|
match = re.search(r'episode[-\s]*(\d+)', part, re.I)
|
||||||
|
if match:
|
||||||
|
episode = match.group(1)
|
||||||
|
|
||||||
|
filename = f"{anime_name} - Episode {episode}.mp4"
|
||||||
|
return filename.title()
|
||||||
|
|
||||||
|
async def get_episodes(self, anime_url: str, lang: str = "vostfr") -> list[dict]:
|
||||||
|
"""Get list of episodes for an anime."""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
episodes = []
|
||||||
|
# Try to find episode links
|
||||||
|
episode_links = soup.find_all('a', href=re.compile(r'episode'))
|
||||||
|
|
||||||
|
for link in episode_links:
|
||||||
|
href = link.get('href', '')
|
||||||
|
match = re.search(r'episode[-\s]*(\d+)', href, re.I)
|
||||||
|
if match:
|
||||||
|
episode_num = match.group(1)
|
||||||
|
if not href.startswith('http'):
|
||||||
|
href = urljoin(anime_url, href)
|
||||||
|
|
||||||
|
episodes.append({'episode': episode_num, 'url': href})
|
||||||
|
|
||||||
|
# Deduplicate and sort
|
||||||
|
seen = set()
|
||||||
|
unique_episodes = []
|
||||||
|
for ep in episodes:
|
||||||
|
if ep['episode'] not in seen:
|
||||||
|
seen.add(ep['episode'])
|
||||||
|
unique_episodes.append(ep)
|
||||||
|
|
||||||
|
unique_episodes.sort(key=lambda x: int(x['episode']))
|
||||||
|
return unique_episodes
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_anime_metadata(self, anime_url: str) -> dict:
|
||||||
|
"""Extract rich metadata from anime page."""
|
||||||
|
try:
|
||||||
|
print(f"[NEKO-SAMA] Extracting metadata from: {anime_url}")
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
metadata = {
|
||||||
|
'synopsis': None,
|
||||||
|
'genres': [],
|
||||||
|
'rating': None,
|
||||||
|
'release_year': None,
|
||||||
|
'studio': None,
|
||||||
|
'poster_image': None,
|
||||||
|
'banner_image': None,
|
||||||
|
'total_episodes': None,
|
||||||
|
'status': None,
|
||||||
|
'alternative_titles': []
|
||||||
|
}
|
||||||
|
|
||||||
|
# Extract title and year from h1
|
||||||
|
title_elem = soup.find('h1')
|
||||||
|
if title_elem:
|
||||||
|
title_text = title_elem.get_text(strip=True)
|
||||||
|
# Extract year from title like "Naruto (2002)"
|
||||||
|
year_match = re.search(r'\((\d{4})\)', title_text)
|
||||||
|
if year_match:
|
||||||
|
metadata['release_year'] = int(year_match.group(1))
|
||||||
|
|
||||||
|
# Extract synopsis - Gupy shows it as paragraphs
|
||||||
|
synopsis_elem = soup.find('p')
|
||||||
|
if synopsis_elem:
|
||||||
|
text = synopsis_elem.get_text(strip=True)
|
||||||
|
if len(text) > 50:
|
||||||
|
metadata['synopsis'] = text
|
||||||
|
|
||||||
|
# Extract genres from meta tags or links
|
||||||
|
genre_links = soup.find_all('a', href=re.compile(r'serie-|genre|tag'))
|
||||||
|
if genre_links:
|
||||||
|
genres = []
|
||||||
|
for link in genre_links[:5]:
|
||||||
|
text = link.get_text(strip=True)
|
||||||
|
if text and '/' not in text and len(text) < 30:
|
||||||
|
genres.append(text)
|
||||||
|
metadata['genres'] = genres
|
||||||
|
|
||||||
|
# Extract rating from percentage
|
||||||
|
rating_elem = soup.find(string=re.compile(r'\d+(\.\d+)?%'))
|
||||||
|
if rating_elem:
|
||||||
|
match = re.search(r'(\d+(\.\d+)?)%', rating_elem)
|
||||||
|
if match:
|
||||||
|
rating = float(match.group(1)) / 10
|
||||||
|
metadata['rating'] = f"{rating:.1f}/10"
|
||||||
|
|
||||||
|
# Extract poster image
|
||||||
|
poster_elem = soup.find('img', src=re.compile(r'poster|poster'))
|
||||||
|
if poster_elem:
|
||||||
|
metadata['poster_image'] = poster_elem.get('src')
|
||||||
|
|
||||||
|
# Extract episode count from page text
|
||||||
|
page_text = soup.get_text()
|
||||||
|
ep_match = re.search(r'(\d+)\s*episodes?', page_text, re.I)
|
||||||
|
if ep_match:
|
||||||
|
metadata['total_episodes'] = int(ep_match.group(1))
|
||||||
|
|
||||||
|
# Extract studio/director
|
||||||
|
director_elem = soup.find('a', href=re.compile(r'person|réalisé'))
|
||||||
|
if director_elem:
|
||||||
|
metadata['studio'] = director_elem.get_text(strip=True)
|
||||||
|
|
||||||
|
print(f"[NEKO-SAMA] Extracted metadata: {metadata}")
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[NEKO-SAMA] Error extracting metadata: {e}")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
async def search_anime(self, query: str, lang: str = "vostfr", include_metadata: bool = False) -> list[dict]:
|
||||||
|
"""Search for anime on neko-sama (uses Gupy backend)."""
|
||||||
|
try:
|
||||||
|
import time
|
||||||
|
from html import unescape
|
||||||
|
start = time.time()
|
||||||
|
print(f"[NEKO-SAMA] Searching for '{query}' ({lang})...")
|
||||||
|
|
||||||
|
# Neko-Sama now uses Gupy - try the direct URL pattern
|
||||||
|
search_slug = query.lower().replace(' ', '-')
|
||||||
|
search_urls = [
|
||||||
|
f"https://www.gupy.fr/series/{search_slug}/",
|
||||||
|
f"https://neko-sama.org/series/{search_slug}/",
|
||||||
|
]
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for search_url in search_urls:
|
||||||
|
response = await self.client.get(search_url, follow_redirects=True)
|
||||||
|
print(f"[NEKO-SAMA] Tried {search_url} -> {response.status_code}")
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
final_url = str(response.url)
|
||||||
|
print(f"[NEKO-SAMA] Found anime at {final_url}")
|
||||||
|
|
||||||
|
# Extract title from page
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
title_elem = soup.find('h1') or soup.find('title')
|
||||||
|
title = unescape(title_elem.get_text(strip=True)) if title_elem else query
|
||||||
|
# Clean up title
|
||||||
|
title = title.split('|')[0].split('-')[0].strip()
|
||||||
|
|
||||||
|
result = {
|
||||||
|
'title': title,
|
||||||
|
'url': final_url,
|
||||||
|
'cover_image': None,
|
||||||
|
'type': 'direct',
|
||||||
|
'metadata': None
|
||||||
|
}
|
||||||
|
|
||||||
|
# Try to get poster
|
||||||
|
poster = soup.find('img', src=re.compile(r'poster'))
|
||||||
|
if poster:
|
||||||
|
result['cover_image'] = poster.get('src')
|
||||||
|
|
||||||
|
if include_metadata:
|
||||||
|
metadata = await self.get_anime_metadata(final_url)
|
||||||
|
result['metadata'] = metadata
|
||||||
|
|
||||||
|
results.append(result)
|
||||||
|
break
|
||||||
|
|
||||||
|
elapsed = time.time() - start
|
||||||
|
print(f"[NEKO-SAMA] Search completed in {elapsed:.2f}s, found {len(results)} results")
|
||||||
|
return results
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[NEKO-SAMA] Error: {str(e)}")
|
||||||
|
return []
|
||||||
@@ -0,0 +1,253 @@
|
|||||||
|
from .base import BaseAnimeSite
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
|
||||||
|
class VostfreeDownloader(BaseAnimeSite):
|
||||||
|
"""Downloader for vostfree.tv"""
|
||||||
|
|
||||||
|
BASE_DOMAINS = ["vostfree.tv", "www.vostfree.tv"]
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return any(domain in url.lower() for domain in self.BASE_DOMAINS)
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str) -> tuple[str, str]:
|
||||||
|
"""Extract download link from vostfree URL"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(url, follow_redirects=True)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Method 1: Look for iframe players
|
||||||
|
iframes = soup.find_all('iframe')
|
||||||
|
for iframe in iframes:
|
||||||
|
src = iframe.get('src', '')
|
||||||
|
if src and any(p in src for p in ['player', 'video', 'stream']):
|
||||||
|
if not src.startswith('http'):
|
||||||
|
src = urljoin(str(response.url), src)
|
||||||
|
filename = self._generate_filename(str(response.url))
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
# Method 2: Look for video tags
|
||||||
|
videos = soup.find_all('video')
|
||||||
|
for video in videos:
|
||||||
|
src = video.get('src')
|
||||||
|
if src:
|
||||||
|
filename = self._generate_filename(str(response.url))
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
sources = video.find_all('source')
|
||||||
|
for source in sources:
|
||||||
|
src = source.get('src', '')
|
||||||
|
if src and any(ext in src for ext in ['mp4', 'm3u8']):
|
||||||
|
filename = self._generate_filename(str(response.url))
|
||||||
|
return src, filename
|
||||||
|
|
||||||
|
# Method 3: Look in scripts
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
patterns = [
|
||||||
|
r'(https?://[^"\'>\s]+\.(?:mp4|m3u8)(?:\?[^"\'>\s]*)?)',
|
||||||
|
r'"url":"([^"]+)"',
|
||||||
|
r'"file":"([^"]+)"',
|
||||||
|
r'"video":"([^"]+)"',
|
||||||
|
]
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, script.string)
|
||||||
|
for match in matches:
|
||||||
|
match = match.replace('\\/', '/')
|
||||||
|
if any(ext in match for ext in ['mp4', 'm3u8']):
|
||||||
|
filename = self._generate_filename(str(response.url))
|
||||||
|
return match, filename
|
||||||
|
|
||||||
|
raise Exception("Could not find video link")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Vostfree link: {str(e)}")
|
||||||
|
|
||||||
|
def _generate_filename(self, url: str) -> str:
|
||||||
|
parts = url.split('/')
|
||||||
|
anime_name = "anime"
|
||||||
|
episode = "1"
|
||||||
|
|
||||||
|
for part in parts:
|
||||||
|
match = re.search(r'episode[-\s]*(\d+)', part, re.I)
|
||||||
|
if match:
|
||||||
|
episode = match.group(1)
|
||||||
|
|
||||||
|
filename = f"{anime_name} - Episode {episode}.mp4"
|
||||||
|
return filename.title()
|
||||||
|
|
||||||
|
async def get_episodes(self, anime_url: str, lang: str = "vostfr") -> list[dict]:
|
||||||
|
try:
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
episodes = []
|
||||||
|
episode_links = soup.find_all('a', href=re.compile(r'episode', re.I))
|
||||||
|
|
||||||
|
for link in episode_links:
|
||||||
|
href = link.get('href', '')
|
||||||
|
match = re.search(r'episode[-\s]*(\d+)', href, re.I)
|
||||||
|
if match:
|
||||||
|
episode_num = match.group(1)
|
||||||
|
if not href.startswith('http'):
|
||||||
|
href = urljoin(anime_url, href)
|
||||||
|
|
||||||
|
episodes.append({'episode': episode_num, 'url': href})
|
||||||
|
|
||||||
|
# Deduplicate and sort
|
||||||
|
seen = set()
|
||||||
|
unique_episodes = []
|
||||||
|
for ep in episodes:
|
||||||
|
if ep['episode'] not in seen:
|
||||||
|
seen.add(ep['episode'])
|
||||||
|
unique_episodes.append(ep)
|
||||||
|
|
||||||
|
unique_episodes.sort(key=lambda x: int(x['episode']))
|
||||||
|
return unique_episodes
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_anime_metadata(self, anime_url: str) -> dict:
|
||||||
|
"""
|
||||||
|
Extract rich metadata from anime page
|
||||||
|
Returns synopsis, genres, rating, release year, studio, etc.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
print(f"[VOSTFREE] Extracting metadata from: {anime_url}")
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
metadata = {
|
||||||
|
'synopsis': None,
|
||||||
|
'genres': [],
|
||||||
|
'rating': None,
|
||||||
|
'release_year': None,
|
||||||
|
'studio': None,
|
||||||
|
'poster_image': None,
|
||||||
|
'banner_image': None,
|
||||||
|
'total_episodes': None,
|
||||||
|
'status': None,
|
||||||
|
'alternative_titles': []
|
||||||
|
}
|
||||||
|
|
||||||
|
# Extract synopsis
|
||||||
|
synopsis_selectors = [
|
||||||
|
'div.synopsis',
|
||||||
|
'div.description',
|
||||||
|
'div[class*="synopsis"]',
|
||||||
|
'div[class*="desc"]',
|
||||||
|
'p.synopsis',
|
||||||
|
'.anime-synopsis'
|
||||||
|
]
|
||||||
|
|
||||||
|
for selector in synopsis_selectors:
|
||||||
|
synopsis_elem = soup.select_one(selector)
|
||||||
|
if synopsis_elem:
|
||||||
|
synopsis = synopsis_elem.get_text(strip=True)
|
||||||
|
if len(synopsis) > 50:
|
||||||
|
metadata['synopsis'] = synopsis
|
||||||
|
break
|
||||||
|
|
||||||
|
# Extract genres
|
||||||
|
genre_links = soup.find_all('a', href=re.compile(r'genre|tag|type', re.I))
|
||||||
|
if genre_links:
|
||||||
|
metadata['genres'] = [link.get_text(strip=True) for link in genre_links[:5]]
|
||||||
|
|
||||||
|
# Extract rating
|
||||||
|
rating_selectors = [
|
||||||
|
'span.rating',
|
||||||
|
'div.rating',
|
||||||
|
'span.score',
|
||||||
|
'div[class*="rating"]',
|
||||||
|
'div[class*="score"]'
|
||||||
|
]
|
||||||
|
|
||||||
|
for selector in rating_selectors:
|
||||||
|
rating_elem = soup.select_one(selector)
|
||||||
|
if rating_elem:
|
||||||
|
rating_text = rating_elem.get_text(strip=True)
|
||||||
|
rating_match = re.search(r'(\d+\.?\d*)\s*/\s*10', rating_text)
|
||||||
|
if rating_match:
|
||||||
|
metadata['rating'] = f"{rating_match.group(1)}/10"
|
||||||
|
break
|
||||||
|
|
||||||
|
# Extract release year
|
||||||
|
page_text = soup.get_text()
|
||||||
|
year_matches = re.findall(r'\b(19\d{2}|20\d{2})\b', page_text)
|
||||||
|
if year_matches:
|
||||||
|
import datetime
|
||||||
|
current_year = datetime.datetime.now().year + 2
|
||||||
|
valid_years = [int(y) for y in year_matches if 1950 <= int(y) <= current_year]
|
||||||
|
if valid_years:
|
||||||
|
from collections import Counter
|
||||||
|
metadata['release_year'] = Counter(valid_years).most_common(1)[0][0]
|
||||||
|
|
||||||
|
# Extract poster image
|
||||||
|
poster_elem = soup.select_one('img.poster, img.cover, .anime-poster img')
|
||||||
|
if poster_elem:
|
||||||
|
metadata['poster_image'] = poster_elem.get('src') or poster_elem.get('data-src')
|
||||||
|
|
||||||
|
# Extract poster from og:image
|
||||||
|
og_image = soup.find('meta', property='og:image')
|
||||||
|
if og_image and not metadata['poster_image']:
|
||||||
|
metadata['poster_image'] = og_image.get('content')
|
||||||
|
|
||||||
|
# Extract total episodes
|
||||||
|
episodes_count = len(await self.get_episodes(anime_url))
|
||||||
|
if episodes_count > 0:
|
||||||
|
metadata['total_episodes'] = episodes_count
|
||||||
|
|
||||||
|
print(f"[VOSTFREE] Extracted metadata: {metadata}")
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[VOSTFREE] Error extracting metadata: {e}")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
async def search_anime(self, query: str, lang: str = "vostfr", include_metadata: bool = False) -> list[dict]:
|
||||||
|
"""
|
||||||
|
Search for anime on vostfree
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query string
|
||||||
|
lang: Language preference (vostfr, vf)
|
||||||
|
include_metadata: Whether to fetch full metadata for each result (slower)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import time
|
||||||
|
start = time.time()
|
||||||
|
print(f"[VOSTFREE] Searching for '{query}' ({lang})...")
|
||||||
|
|
||||||
|
# Vostfree URL pattern
|
||||||
|
search_url = f"https://vostfree.tv/anime/{query.lower().replace(' ', '-')}"
|
||||||
|
|
||||||
|
response = await self.client.get(search_url)
|
||||||
|
|
||||||
|
elapsed = time.time() - start
|
||||||
|
print(f"[VOSTFREE] Got response {response.status_code} in {elapsed:.2f}s")
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
print(f"[VOSTFREE] Found anime at {str(response.url)}")
|
||||||
|
result = {
|
||||||
|
'title': query,
|
||||||
|
'url': str(response.url),
|
||||||
|
'type': 'direct',
|
||||||
|
'metadata': None
|
||||||
|
}
|
||||||
|
|
||||||
|
if include_metadata:
|
||||||
|
metadata = await self.get_anime_metadata(str(response.url))
|
||||||
|
result['metadata'] = metadata
|
||||||
|
|
||||||
|
return [result]
|
||||||
|
|
||||||
|
print(f"[VOSTFREE] No anime found")
|
||||||
|
return []
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[VOSTFREE] Error: {str(e)}")
|
||||||
|
return []
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from typing import Optional, Tuple
|
||||||
|
import httpx
|
||||||
|
import re
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
|
||||||
|
class BaseDownloader(ABC):
|
||||||
|
"""Base class for all host downloaders"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.client = httpx.AsyncClient(timeout=10.0, follow_redirects=True)
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_download_link(self, url: str) -> Tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract direct download link and filename from host URL
|
||||||
|
Returns: (download_url, filename)
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this downloader can handle the given URL"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
await self.client.aclose()
|
||||||
|
|
||||||
|
async def _fetch_page(self, url: str) -> str:
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.text
|
||||||
|
|
||||||
|
def _extract_filename_from_headers(self, headers: dict) -> Optional[str]:
|
||||||
|
content_disposition = headers.get("content-disposition", "")
|
||||||
|
if "filename=" in content_disposition:
|
||||||
|
filename = content_disposition.split("filename=")[-1].strip('"')
|
||||||
|
return filename
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def search_anime(self, query: str, lang: str = "vostfr") -> list[dict]:
|
||||||
|
"""
|
||||||
|
Search for anime on this provider
|
||||||
|
Returns list of anime with title, url, and optional cover image
|
||||||
|
"""
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_episodes(self, anime_url: str, lang: str = "vostfr") -> list[dict]:
|
||||||
|
"""
|
||||||
|
Get list of episodes for an anime
|
||||||
|
Returns list of episode numbers and their URLs
|
||||||
|
"""
|
||||||
|
return []
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
"""Series streaming sites (catalogs) downloaders"""
|
||||||
|
from .base import BaseSeriesSite
|
||||||
|
# Import all series site downloaders
|
||||||
|
from .fs7 import FS7Downloader
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseSeriesSite",
|
||||||
|
"FS7Downloader",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_series_site(url: str) -> BaseSeriesSite:
|
||||||
|
"""Factory function to get the appropriate series site for a URL"""
|
||||||
|
sites = [
|
||||||
|
FS7Downloader(),
|
||||||
|
]
|
||||||
|
|
||||||
|
for site in sites:
|
||||||
|
if site.can_handle(url):
|
||||||
|
return site
|
||||||
|
|
||||||
|
# Return None if no match (should not happen in normal flow)
|
||||||
|
return None
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
"""Base class for series streaming sites (catalogs)"""
|
||||||
|
from abc import abstractmethod
|
||||||
|
from typing import List, Dict, Any, Optional, Tuple
|
||||||
|
import logging
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class BaseSeriesSite:
|
||||||
|
"""
|
||||||
|
Base class for series streaming sites.
|
||||||
|
|
||||||
|
Series sites provide catalogs, metadata, and episode listings.
|
||||||
|
They typically link to video players for actual file hosting.
|
||||||
|
|
||||||
|
Examples: FS7 (French Stream), etc.
|
||||||
|
|
||||||
|
KEY FEATURE: Provides rich metadata and episode management for TV series
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# Initialize HTTP client directly
|
||||||
|
self.client = httpx.AsyncClient(timeout=10.0, follow_redirects=True)
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this series site can handle the given URL"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def search_anime(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
lang: str = "vf"
|
||||||
|
) -> List[Dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Search for series on this site.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query (series title)
|
||||||
|
lang: Language preference (vf, vostfr)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of series with keys:
|
||||||
|
- title: Series title
|
||||||
|
- url: Series page URL
|
||||||
|
- cover_image: Optional cover image URL
|
||||||
|
- lang: Available languages
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_episodes(
|
||||||
|
self,
|
||||||
|
anime_url: str,
|
||||||
|
lang: str = "vf"
|
||||||
|
) -> List[Dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Get list of episodes for a series.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_url: URL of the series page
|
||||||
|
lang: Language preference
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of episodes with keys:
|
||||||
|
- episode_number: Episode number
|
||||||
|
- url: Episode page URL
|
||||||
|
- title: Optional episode title
|
||||||
|
- host: Video player hosting the file
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_anime_metadata(self, anime_url: str) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Get detailed metadata for a series.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_url: URL of the series page
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with metadata:
|
||||||
|
- title: Series title
|
||||||
|
- synopsis: Plot summary
|
||||||
|
- genres: List of genres
|
||||||
|
- rating: Rating (e.g., "8.5/10")
|
||||||
|
- release_year: Release year
|
||||||
|
- studio: Production studio
|
||||||
|
- poster_image: Poster URL
|
||||||
|
- total_episodes: Total episode count
|
||||||
|
- status: Airing status (ongoing, completed)
|
||||||
|
- languages: Available languages
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_download_link(self, url: str) -> Tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Get download link for a specific episode.
|
||||||
|
|
||||||
|
For series sites, this extracts the video player URL from an episode page.
|
||||||
|
Note: Returns video player URL, NOT direct download link!
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (video_player_url, episode_title)
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Common methods for all series sites
|
||||||
|
async def close(self):
|
||||||
|
"""Close HTTP client"""
|
||||||
|
await self.client.aclose()
|
||||||
|
|
||||||
|
async def _fetch_page(self, url: str) -> str:
|
||||||
|
"""Fetch HTML page content"""
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.text
|
||||||
|
|
||||||
|
def _parse_html(self, html: str) -> BeautifulSoup:
|
||||||
|
"""Parse HTML with BeautifulSoup"""
|
||||||
|
return BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
def _extract_season_number(self, title: str) -> Optional[int]:
|
||||||
|
"""Extract season number from title (e.g., 'Saison 2' -> 2)"""
|
||||||
|
import re
|
||||||
|
match = re.search(r'saison\s*(\d+)', title.lower())
|
||||||
|
return int(match.group(1)) if match else None
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
"""FS7 (French Stream) series site downloader"""
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from typing import List, Dict, Any, Optional
|
||||||
|
from urllib.parse import urljoin, urlparse
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
from .base import BaseSeriesSite
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class FS7Downloader(BaseSeriesSite):
|
||||||
|
"""
|
||||||
|
Downloader for FS7 (French Stream) series site.
|
||||||
|
|
||||||
|
FS7 is a French streaming site for TV series and films.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.base_url = "https://fs7.lol"
|
||||||
|
self.search_url = f"{self.base_url}/"
|
||||||
|
# Update client headers to mimic browser
|
||||||
|
self.client.headers.update({
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||||
|
'Accept-Language': 'fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||||
|
'Accept-Encoding': 'gzip, deflate',
|
||||||
|
'Connection': 'keep-alive',
|
||||||
|
'Upgrade-Insecure-Requests': '1'
|
||||||
|
})
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this downloader can handle the given URL"""
|
||||||
|
return "fs7.lol" in url.lower() or "french-stream" in url.lower()
|
||||||
|
|
||||||
|
async def search_anime(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
lang: str = "vf"
|
||||||
|
) -> List[Dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Search for series on FS7.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query
|
||||||
|
lang: Language preference (vf, vostfr)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of series with title, url, cover_image
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
logger.info(f"Searching FS7 for: {query}")
|
||||||
|
|
||||||
|
# FS7 uses GET request with query parameters for search
|
||||||
|
response = await self.client.get(
|
||||||
|
self.search_url,
|
||||||
|
params={
|
||||||
|
"do": "search",
|
||||||
|
"subaction": "search",
|
||||||
|
"story": query
|
||||||
|
}
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
results = []
|
||||||
|
|
||||||
|
# Look for series items (FS7 has both films and series in search results)
|
||||||
|
# We filter for /s-tv/ URLs ending with .html (actual series/season pages)
|
||||||
|
items = soup.find_all('a', href=re.compile(r'/s-tv/\d+-.+\.html'))
|
||||||
|
|
||||||
|
for item in items[:20]: # Limit to 20 results
|
||||||
|
url = item.get('href', '')
|
||||||
|
if not url.startswith('http'):
|
||||||
|
url = urljoin(self.base_url, url)
|
||||||
|
|
||||||
|
# Extract title from the item
|
||||||
|
title_elem = item.find('img', alt=True)
|
||||||
|
if title_elem:
|
||||||
|
title = title_elem.get('alt', '').strip()
|
||||||
|
else:
|
||||||
|
# Get text content and clean it
|
||||||
|
text = item.get_text(strip=True)
|
||||||
|
# Skip if it's just a category name
|
||||||
|
if any(cat in text.lower() for cat in ['séries', 'series', 'vf', 'vostfr', 'vo', 'netflix', 'disney', 'amazon', 'apple']):
|
||||||
|
continue
|
||||||
|
title = text
|
||||||
|
|
||||||
|
# Clean up title: remove "affiche" suffix and clean extra whitespace
|
||||||
|
title = re.sub(r'\s+affiche$', '', title, flags=re.IGNORECASE).strip()
|
||||||
|
title = re.sub(r'\s+', ' ', title) # Normalize whitespace
|
||||||
|
|
||||||
|
# Extract cover image
|
||||||
|
img = item.find('img')
|
||||||
|
cover_image = img.get('src', '') if img else ''
|
||||||
|
|
||||||
|
# Only add if we have a title and it's not empty
|
||||||
|
if title and len(title) > 5:
|
||||||
|
# Avoid duplicates
|
||||||
|
if not any(r['url'] == url for r in results):
|
||||||
|
results.append({
|
||||||
|
'title': title,
|
||||||
|
'url': url,
|
||||||
|
'cover_image': cover_image
|
||||||
|
})
|
||||||
|
|
||||||
|
logger.info(f"Found {len(results)} series on FS7")
|
||||||
|
return results
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error searching FS7: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_episodes(
|
||||||
|
self,
|
||||||
|
anime_url: str,
|
||||||
|
lang: str = "vf"
|
||||||
|
) -> List[Dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Get episode list for a series.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_url: URL of the series page
|
||||||
|
lang: Language preference
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of episodes with episode number and url
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
logger.info(f"Fetching episodes from: {anime_url}")
|
||||||
|
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
episodes = []
|
||||||
|
|
||||||
|
# Get series title for episode naming
|
||||||
|
title_elem = soup.find('h1')
|
||||||
|
series_title = title_elem.get_text(strip=True) if title_elem else "Series"
|
||||||
|
# Clean up title: remove "affiche" suffix
|
||||||
|
series_title = re.sub(r'\s+affiche$', '', series_title, flags=re.IGNORECASE).strip()
|
||||||
|
|
||||||
|
# FS7 stores episode data in JavaScript div elements
|
||||||
|
# Format: <div data-ep="1" data-vidzy="..." data-uqload="..." data-netu="..." data-voe="..."></div>
|
||||||
|
episode_divs = soup.find_all('div', attrs={'data-ep': True})
|
||||||
|
|
||||||
|
for div in episode_divs:
|
||||||
|
ep_num = div.get('data-ep', '').strip()
|
||||||
|
|
||||||
|
# Try different video players in order of preference
|
||||||
|
video_url = None
|
||||||
|
host_name = None
|
||||||
|
for player in ['data-vidzy', 'data-uqload', 'data-voe', 'data-netu']:
|
||||||
|
player_url = div.get(player, '').strip()
|
||||||
|
if player_url:
|
||||||
|
video_url = player_url
|
||||||
|
# Extract host name from attribute name
|
||||||
|
host_name = player.replace('data-', '').title()
|
||||||
|
logger.debug(f"Found episode {ep_num} on {host_name}")
|
||||||
|
break
|
||||||
|
|
||||||
|
if video_url and ep_num:
|
||||||
|
# Create episode title for filename
|
||||||
|
episode_title = f"{series_title} - Episode {ep_num}"
|
||||||
|
|
||||||
|
# Use pipe-separated format: video_url|anime_url|episode_title
|
||||||
|
combined_url = f"{video_url}|{anime_url}|{episode_title}"
|
||||||
|
|
||||||
|
episodes.append({
|
||||||
|
'episode': ep_num,
|
||||||
|
'url': combined_url,
|
||||||
|
'title': episode_title,
|
||||||
|
'host': host_name or 'Unknown'
|
||||||
|
})
|
||||||
|
|
||||||
|
# Sort by episode number
|
||||||
|
episodes.sort(key=lambda x: int(x['episode']) if x['episode'].isdigit() else 0)
|
||||||
|
|
||||||
|
logger.info(f"Found {len(episodes)} episodes")
|
||||||
|
return episodes
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting episodes from FS7: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_anime_metadata(
|
||||||
|
self,
|
||||||
|
anime_url: str
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Get metadata for a series.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_url: URL of the series page
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary with metadata
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
logger.info(f"Fetching metadata from: {anime_url}")
|
||||||
|
|
||||||
|
response = await self.client.get(anime_url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
# Extract title
|
||||||
|
title = soup.find('h1')
|
||||||
|
title = title.get_text(strip=True) if title else "Unknown"
|
||||||
|
|
||||||
|
# Clean up title: remove "affiche" suffix
|
||||||
|
title = re.sub(r'\s+affiche$', '', title, flags=re.IGNORECASE).strip()
|
||||||
|
|
||||||
|
# Extract description/synopsis
|
||||||
|
description_elem = soup.find('div', class_='full-text')
|
||||||
|
description = description_elem.get_text(strip=True) if description_elem else ""
|
||||||
|
|
||||||
|
# Extract cover image
|
||||||
|
img = soup.find('img', class_='poster')
|
||||||
|
poster_image = img.get('src', '') if img else ''
|
||||||
|
|
||||||
|
# Try to get poster from meta tag if not found
|
||||||
|
if not poster_image:
|
||||||
|
meta_img = soup.find('meta', property='og:image')
|
||||||
|
poster_image = meta_img.get('content', '') if meta_img else ''
|
||||||
|
|
||||||
|
# Extract year
|
||||||
|
year_match = re.search(r'\b(19|20)\d{2}\b', description)
|
||||||
|
release_year = int(year_match.group()) if year_match else None
|
||||||
|
|
||||||
|
return {
|
||||||
|
'title': title,
|
||||||
|
'synopsis': description,
|
||||||
|
'poster_image': poster_image,
|
||||||
|
'release_year': release_year,
|
||||||
|
'genres': [],
|
||||||
|
'rating': None,
|
||||||
|
'studio': None,
|
||||||
|
'total_episodes': None,
|
||||||
|
'status': None
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting metadata from FS7: {e}")
|
||||||
|
return {
|
||||||
|
'title': "Unknown",
|
||||||
|
'synopsis': "",
|
||||||
|
'poster_image': '',
|
||||||
|
'genres': [],
|
||||||
|
'rating': None,
|
||||||
|
'release_year': None,
|
||||||
|
'studio': None,
|
||||||
|
'total_episodes': None,
|
||||||
|
'status': None
|
||||||
|
}
|
||||||
|
|
||||||
|
async def get_download_link(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
target_filename: Optional[str] = None
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract download link from video player URL.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: Video player URL
|
||||||
|
target_filename: Optional filename override
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (download_url, filename)
|
||||||
|
"""
|
||||||
|
# FS7 uses embedded video players
|
||||||
|
# Delegate to the appropriate video player downloader
|
||||||
|
from app.downloaders.video_players import get_video_player
|
||||||
|
|
||||||
|
player = get_video_player(url)
|
||||||
|
if player:
|
||||||
|
return await player.get_download_link(url, target_filename)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"No video player found for URL: {url}")
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"""Video hosting services (players) downloaders"""
|
||||||
|
from .base import BaseVideoPlayer
|
||||||
|
# Import all video player downloaders
|
||||||
|
from .doodstream import DoodStreamDownloader
|
||||||
|
from .sibnet import SibnetDownloader
|
||||||
|
from .vidmoly import VidMolyDownloader
|
||||||
|
from .sendvid import SendVidDownloader
|
||||||
|
from .lpayer import LpayerDownloader
|
||||||
|
from .unfichier import UnFichierDownloader
|
||||||
|
from .uptobox import UptoboxDownloader
|
||||||
|
from .rapidfile import RapidFileDownloader
|
||||||
|
from .vidzy import VidzyDownloader
|
||||||
|
from .luluv import LuLuvidDownloader
|
||||||
|
from .uqload import UqloadDownloader
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseVideoPlayer",
|
||||||
|
"DoodStreamDownloader",
|
||||||
|
"SibnetDownloader",
|
||||||
|
"VidMolyDownloader",
|
||||||
|
"SendVidDownloader",
|
||||||
|
"LpayerDownloader",
|
||||||
|
"UnFichierDownloader",
|
||||||
|
"UptoboxDownloader",
|
||||||
|
"RapidFileDownloader",
|
||||||
|
"VidzyDownloader",
|
||||||
|
"LuLuvidDownloader",
|
||||||
|
"UqloadDownloader",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_video_player(url: str) -> BaseVideoPlayer:
|
||||||
|
"""Factory function to get the appropriate video player for a URL"""
|
||||||
|
players = [
|
||||||
|
DoodStreamDownloader(),
|
||||||
|
SibnetDownloader(),
|
||||||
|
VidMolyDownloader(),
|
||||||
|
SendVidDownloader(),
|
||||||
|
LpayerDownloader(),
|
||||||
|
UnFichierDownloader(),
|
||||||
|
UptoboxDownloader(),
|
||||||
|
RapidFileDownloader(),
|
||||||
|
VidzyDownloader(),
|
||||||
|
LuLuvidDownloader(),
|
||||||
|
UqloadDownloader(),
|
||||||
|
]
|
||||||
|
|
||||||
|
for player in players:
|
||||||
|
if player.can_handle(url):
|
||||||
|
return player
|
||||||
|
|
||||||
|
# Return None if no match (should not happen in normal flow)
|
||||||
|
return None
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
"""Base class for video hosting services (players)"""
|
||||||
|
from abc import abstractmethod
|
||||||
|
from typing import Optional, Tuple
|
||||||
|
import logging
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class BaseVideoPlayer:
|
||||||
|
"""
|
||||||
|
Base class for video hosting services.
|
||||||
|
|
||||||
|
Video players host actual video files and provide direct download links.
|
||||||
|
They extract URLs from embedded players and handle file downloads.
|
||||||
|
|
||||||
|
Examples: Doodstream, Sibnet, VidMoly, SendVid, Lpayer, 1fichier, etc.
|
||||||
|
|
||||||
|
KEY FEATURE: Flexible get_download_link() signature to support:
|
||||||
|
- Standard: get_download_link(url)
|
||||||
|
- With target_filename: get_download_link(url, target_filename="...") (VidMoly, SendVid)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# Realistic browser headers to avoid blocking by video hosts
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9,fr;q=0.8",
|
||||||
|
"Referer": "https://anime-sama.tv/",
|
||||||
|
}
|
||||||
|
# Initialize HTTP client with browser headers
|
||||||
|
self.client = httpx.AsyncClient(timeout=10.0, follow_redirects=True, headers=headers)
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this player can handle the given URL"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_download_link(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
target_filename: Optional[str] = None
|
||||||
|
) -> Tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract direct download link and filename from video player URL.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: The video player URL
|
||||||
|
target_filename: Optional filename override (used by VidMoly, SendVid)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (download_url, filename)
|
||||||
|
|
||||||
|
Note:
|
||||||
|
- Always use sanitize_filename() on extracted filenames!
|
||||||
|
- target_filename parameter is optional but MUST be supported
|
||||||
|
for compatibility with VidMoly and SendVid
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Common methods for all video players
|
||||||
|
async def close(self):
|
||||||
|
"""Close HTTP client"""
|
||||||
|
await self.client.aclose()
|
||||||
|
|
||||||
|
async def _fetch_page(self, url: str) -> str:
|
||||||
|
"""Fetch HTML page content"""
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.text
|
||||||
|
|
||||||
|
def _parse_html(self, html: str) -> BeautifulSoup:
|
||||||
|
"""Parse HTML with BeautifulSoup"""
|
||||||
|
return BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
def _extract_filename_from_headers(self, headers: dict) -> Optional[str]:
|
||||||
|
"""Extract filename from Content-Disposition header"""
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
|
||||||
|
content_disposition = headers.get("content-disposition", "")
|
||||||
|
if "filename=" in content_disposition:
|
||||||
|
filename = content_disposition.split("filename=")[-1].strip('"')
|
||||||
|
return sanitize_filename(filename) # Security!
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _sanitize(self, filename: str) -> str:
|
||||||
|
"""Convenience method for filename sanitization"""
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
return sanitize_filename(filename)
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
|
||||||
|
class DoodStreamDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for doodstream.com"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return any(domain in url.lower() for domain in ["doodstream.com", "dood.stream", "dood.to", "dood.lol", "dood.cx", "dood.so", "dood.watch", "dood.sh"])
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: str = None) -> tuple[str, str]:
|
||||||
|
try:
|
||||||
|
# Get the page
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Doodstream usually has the video URL in a script with '$(function)'
|
||||||
|
# or in a token-based system
|
||||||
|
download_url = None
|
||||||
|
filename = "doodstream_video.mp4"
|
||||||
|
|
||||||
|
# Method 1: Look for /pass_md5 or similar patterns
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
# Look for token patterns
|
||||||
|
match = re.search(r'https?://[^\"\']+\.(?:mp4|mkv|avi)', script.string)
|
||||||
|
if match:
|
||||||
|
download_url = match.group(0)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Look for doodstream CDN patterns
|
||||||
|
match = re.search(r'(https?://[^\s\"\'<>]+/download/[^\s\"\'<>]+)', script.string)
|
||||||
|
if match:
|
||||||
|
download_url = match.group(0)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Method 2: Try to construct download URL from page
|
||||||
|
if not download_url:
|
||||||
|
# Extract video ID from URL
|
||||||
|
# Format: https://doodstream.com/e/VIDEO_ID or /d/VIDEO_ID
|
||||||
|
video_id_match = re.search(r'/[ed]/([a-zA-Z0-9]+)', url)
|
||||||
|
if video_id_match:
|
||||||
|
video_id = video_id_match.group(1)
|
||||||
|
# Try direct download pattern
|
||||||
|
download_url = f"https://dood.stream/e/{video_id}"
|
||||||
|
|
||||||
|
# Method 3: Look for any MP4 source in iframes or video tags
|
||||||
|
if not download_url:
|
||||||
|
video = soup.find('video')
|
||||||
|
if video and video.get('src'):
|
||||||
|
download_url = video['src']
|
||||||
|
else:
|
||||||
|
sources = soup.find_all('source')
|
||||||
|
for source in sources:
|
||||||
|
if source.get('src'):
|
||||||
|
download_url = source['src']
|
||||||
|
filename = source.get('src', '').split('/')[-1]
|
||||||
|
break
|
||||||
|
|
||||||
|
if download_url:
|
||||||
|
# Try to get real filename from HEAD request
|
||||||
|
try:
|
||||||
|
head_resp = await self.client.head(download_url, timeout=5.0)
|
||||||
|
fname = self._extract_filename_from_headers(head_resp.headers)
|
||||||
|
if fname:
|
||||||
|
filename = fname
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return download_url, filename
|
||||||
|
|
||||||
|
raise Exception("Could not extract download link from Doodstream page")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Doodstream link: {str(e)}")
|
||||||
@@ -0,0 +1,471 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
import asyncio
|
||||||
|
from typing import Optional
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
|
||||||
|
class LpayerDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for lpayer.embed4me.com video player"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return 'lpayer.embed4me.com' in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: Optional[str] = None) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract download link from Lpayer video page.
|
||||||
|
Uses Playwright for JavaScript rendering, falls back to HTML parsing.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
print(f"[LPAYER] Extracting link from: {url}")
|
||||||
|
|
||||||
|
# Try Playwright first (handles JavaScript-rendered pages)
|
||||||
|
video_url = await self._extract_with_playwright(url)
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
# Fallback to HTML parsing
|
||||||
|
print("[LPAYER] Playwright failed, trying HTML parsing fallback...")
|
||||||
|
video_url = await self._extract_with_http(url)
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
raise Exception("Could not find video URL in Lpayer page")
|
||||||
|
|
||||||
|
print(f"[LPAYER] Found video URL: {video_url[:80]}...")
|
||||||
|
|
||||||
|
# Use target_filename if provided, otherwise generate default
|
||||||
|
if target_filename:
|
||||||
|
filename = target_filename
|
||||||
|
else:
|
||||||
|
filename = "lpayer_video.mp4"
|
||||||
|
|
||||||
|
# Ensure .mp4 extension if direct MP4
|
||||||
|
if video_url.endswith('.mp4') and not filename.endswith('.mp4'):
|
||||||
|
filename += '.mp4'
|
||||||
|
|
||||||
|
return video_url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Lpayer link: {str(e)}")
|
||||||
|
|
||||||
|
async def _extract_with_playwright(self, url: str) -> Optional[str]:
|
||||||
|
"""Extract video URL using Playwright to render JavaScript"""
|
||||||
|
browser = None
|
||||||
|
try:
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
print("[LPAYER] Launching Playwright browser...")
|
||||||
|
video_urls = []
|
||||||
|
|
||||||
|
async with async_playwright() as p:
|
||||||
|
browser = await p.chromium.launch(
|
||||||
|
headless=True,
|
||||||
|
args=[
|
||||||
|
'--no-sandbox',
|
||||||
|
'--disable-setuid-sandbox',
|
||||||
|
'--disable-dev-shm-usage',
|
||||||
|
'--disable-blink-features=AutomationControlled',
|
||||||
|
'--disable-features=IsolateOrigins,site-per-process',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
context = await browser.new_context(
|
||||||
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||||
|
viewport={'width': 1920, 'height': 1080}
|
||||||
|
)
|
||||||
|
|
||||||
|
page = await context.new_page()
|
||||||
|
|
||||||
|
# Set up request interception to capture video requests
|
||||||
|
async def handle_request(route):
|
||||||
|
req_url = route.request.url
|
||||||
|
if any(ext in req_url.lower() for ext in ['.m3u8', '.mp4', '.mkv']):
|
||||||
|
if 'lpayer' not in req_url.lower():
|
||||||
|
print(f"[LPAYER] 🎥 Captured video URL: {req_url[:100]}...")
|
||||||
|
video_urls.append(req_url)
|
||||||
|
await route.continue_()
|
||||||
|
|
||||||
|
await page.route('**', handle_request)
|
||||||
|
|
||||||
|
# Navigate to URL with timeout
|
||||||
|
print("[LPAYER] Navigating to page...")
|
||||||
|
try:
|
||||||
|
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] Navigation warning: {e}")
|
||||||
|
|
||||||
|
# Wait for JavaScript to execute
|
||||||
|
print("[LPAYER] Waiting for video player to load...")
|
||||||
|
await asyncio.sleep(5)
|
||||||
|
|
||||||
|
# Try to interact with player to trigger video load
|
||||||
|
try:
|
||||||
|
await page.mouse.click(640, 360)
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Try JavaScript extraction to find video URLs in DOM
|
||||||
|
try:
|
||||||
|
js_result = await page.evaluate("""
|
||||||
|
() => {
|
||||||
|
// Check all video elements
|
||||||
|
const videos = document.querySelectorAll('video');
|
||||||
|
for (let v of videos) {
|
||||||
|
if (v.src && (v.src.includes('.m3u8') || v.src.includes('.mp4'))) {
|
||||||
|
console.log('Found video src:', v.src);
|
||||||
|
return v.src;
|
||||||
|
}
|
||||||
|
const sources = v.querySelectorAll('source');
|
||||||
|
for (let s of sources) {
|
||||||
|
if (s.src && (s.src.includes('.m3u8') || s.src.includes('.mp4'))) {
|
||||||
|
console.log('Found source src:', s.src);
|
||||||
|
return s.src;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for jwplayer
|
||||||
|
if (window.jwplayer) {
|
||||||
|
try {
|
||||||
|
const player = jwplayer();
|
||||||
|
const playlist = player.getPlaylist();
|
||||||
|
if (playlist && playlist[0] && playlist[0].sources) {
|
||||||
|
const src = playlist[0].sources[0].file;
|
||||||
|
console.log('Found jwplayer source:', src);
|
||||||
|
return src;
|
||||||
|
}
|
||||||
|
} catch(e) {
|
||||||
|
console.log('jwplayer error:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for VidStack player
|
||||||
|
const player = document.querySelector('media-player');
|
||||||
|
if (player && player.provider) {
|
||||||
|
const provider = player.provider;
|
||||||
|
// Try to get source from provider
|
||||||
|
if (provider.src) return provider.src;
|
||||||
|
if (provider.currentSrc) return provider.currentSrc;
|
||||||
|
if (provider.url) return provider.url;
|
||||||
|
if (provider.videoUrl) return provider.videoUrl;
|
||||||
|
// Check internal properties
|
||||||
|
for (let key in provider) {
|
||||||
|
try {
|
||||||
|
const val = provider[key];
|
||||||
|
if (typeof val === 'string' && (val.includes('.m3u8') || val.includes('.mp4')) && val.startsWith('http')) {
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
} catch(e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for video URLs in window object
|
||||||
|
for (let key in window) {
|
||||||
|
if (typeof window[key] === 'string') {
|
||||||
|
const str = window[key];
|
||||||
|
if ((str.includes('.m3u8') || str.includes('.mp4')) && str.startsWith('http')) {
|
||||||
|
console.log('Found in window:', str);
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
""")
|
||||||
|
|
||||||
|
if js_result and ('.m3u8' in js_result or '.mp4' in js_result):
|
||||||
|
print(f"[LPAYER] Found video URL via JavaScript")
|
||||||
|
video_urls.append(js_result)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] JS extraction error: {e}")
|
||||||
|
|
||||||
|
# Final check: parse rendered page HTML
|
||||||
|
try:
|
||||||
|
content = await page.content()
|
||||||
|
patterns = [
|
||||||
|
r'"file"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r"'file'\s*:\s*'([^']+\.m3u8[^']*)'",
|
||||||
|
r"'file'\s*:\s*'([^']+\.mp4[^']*)'",
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, content)
|
||||||
|
for match in matches:
|
||||||
|
match = match.replace('\\', '').replace('\\/', '/')
|
||||||
|
if 'http' in match and 'lpayer' not in match.lower():
|
||||||
|
print(f"[LPAYER] Found in HTML: {match[:100]}...")
|
||||||
|
video_urls.append(match)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] HTML parsing error: {e}")
|
||||||
|
|
||||||
|
await browser.close()
|
||||||
|
browser = None
|
||||||
|
|
||||||
|
# Return first valid video URL
|
||||||
|
if video_urls:
|
||||||
|
seen = set()
|
||||||
|
unique_urls = []
|
||||||
|
for url in video_urls:
|
||||||
|
if url not in seen:
|
||||||
|
seen.add(url)
|
||||||
|
unique_urls.append(url)
|
||||||
|
|
||||||
|
if unique_urls:
|
||||||
|
print(f"[LPAYER] ✅ Found {len(unique_urls)} video URL(s)")
|
||||||
|
return unique_urls[0]
|
||||||
|
|
||||||
|
print("[LPAYER] ❌ No video URLs found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
print("[LPAYER] Playwright not installed")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] Playwright error: {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
return None
|
||||||
|
finally:
|
||||||
|
# Ensure browser is always closed
|
||||||
|
if browser:
|
||||||
|
try:
|
||||||
|
await browser.close()
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
"""Extract video URL using Playwright to render JavaScript"""
|
||||||
|
try:
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
print("[LPAYER] Launching Playwright browser...")
|
||||||
|
video_urls = []
|
||||||
|
|
||||||
|
async with async_playwright() as p:
|
||||||
|
browser = await p.chromium.launch(
|
||||||
|
headless=True,
|
||||||
|
args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
|
||||||
|
)
|
||||||
|
|
||||||
|
context = await browser.new_context(
|
||||||
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||||
|
viewport={'width': 1920, 'height': 1080}
|
||||||
|
)
|
||||||
|
|
||||||
|
page = await context.new_page()
|
||||||
|
|
||||||
|
# Set up request interception to capture video requests
|
||||||
|
async def handle_request(route):
|
||||||
|
req_url = route.request.url
|
||||||
|
if any(ext in req_url.lower() for ext in ['.m3u8', '.mp4', '.mkv']):
|
||||||
|
if 'lpayer' not in req_url.lower():
|
||||||
|
print(f"[LPAYER] 🎥 Captured video URL: {req_url[:100]}...")
|
||||||
|
video_urls.append(req_url)
|
||||||
|
await route.continue_()
|
||||||
|
|
||||||
|
await page.route('**', handle_request)
|
||||||
|
|
||||||
|
# Navigate to URL with timeout
|
||||||
|
print("[LPAYER] Navigating to page...")
|
||||||
|
try:
|
||||||
|
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] Navigation warning: {e}")
|
||||||
|
|
||||||
|
# Wait for JavaScript to execute and video to load
|
||||||
|
print("[LPAYER] Waiting for video player to load...")
|
||||||
|
await asyncio.sleep(5)
|
||||||
|
|
||||||
|
# Try JavaScript extraction to find video URLs in DOM
|
||||||
|
try:
|
||||||
|
js_result = await page.evaluate("""
|
||||||
|
() => {
|
||||||
|
// Check all video elements
|
||||||
|
const videos = document.querySelectorAll('video');
|
||||||
|
for (let v of videos) {
|
||||||
|
if (v.src && (v.src.includes('.m3u8') || v.src.includes('.mp4'))) {
|
||||||
|
console.log('Found video src:', v.src);
|
||||||
|
return v.src;
|
||||||
|
}
|
||||||
|
const sources = v.querySelectorAll('source');
|
||||||
|
for (let s of sources) {
|
||||||
|
if (s.src && (s.src.includes('.m3u8') || s.src.includes('.mp4'))) {
|
||||||
|
console.log('Found source src:', s.src);
|
||||||
|
return s.src;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for jwplayer
|
||||||
|
if (window.jwplayer) {
|
||||||
|
try {
|
||||||
|
const player = jwplayer();
|
||||||
|
const playlist = player.getPlaylist();
|
||||||
|
if (playlist && playlist[0] && playlist[0].sources) {
|
||||||
|
const src = playlist[0].sources[0].file;
|
||||||
|
console.log('Found jwplayer source:', src);
|
||||||
|
return src;
|
||||||
|
}
|
||||||
|
} catch(e) {
|
||||||
|
console.log('jwplayer error:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for video URLs in window object
|
||||||
|
for (let key in window) {
|
||||||
|
if (typeof window[key] === 'string') {
|
||||||
|
const str = window[key];
|
||||||
|
if ((str.includes('.m3u8') || str.includes('.mp4')) && str.startsWith('http')) {
|
||||||
|
console.log('Found in window:', str);
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
""")
|
||||||
|
|
||||||
|
if js_result and ('.m3u8' in js_result or '.mp4' in js_result):
|
||||||
|
print(f"[LPAYER] Found video URL via JavaScript")
|
||||||
|
video_urls.append(js_result)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] JS extraction error: {e}")
|
||||||
|
|
||||||
|
# Final check: parse rendered page HTML
|
||||||
|
try:
|
||||||
|
content = await page.content()
|
||||||
|
patterns = [
|
||||||
|
r'"file"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r"'file'\s*:\s*'([^']+\.m3u8[^']*)'",
|
||||||
|
r"'file'\s*:\s*'([^']+\.mp4[^']*)'",
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, content)
|
||||||
|
for match in matches:
|
||||||
|
match = match.replace('\\', '').replace('\\/', '/')
|
||||||
|
if 'http' in match and 'lpayer' not in match.lower():
|
||||||
|
print(f"[LPAYER] Found in HTML: {match[:100]}...")
|
||||||
|
video_urls.append(match)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] HTML parsing error: {e}")
|
||||||
|
|
||||||
|
await browser.close()
|
||||||
|
|
||||||
|
# Return first valid video URL
|
||||||
|
if video_urls:
|
||||||
|
seen = set()
|
||||||
|
unique_urls = []
|
||||||
|
for url in video_urls:
|
||||||
|
if url not in seen:
|
||||||
|
seen.add(url)
|
||||||
|
unique_urls.append(url)
|
||||||
|
|
||||||
|
if unique_urls:
|
||||||
|
print(f"[LPAYER] ✅ Found {len(unique_urls)} video URL(s)")
|
||||||
|
return unique_urls[0]
|
||||||
|
|
||||||
|
print("[LPAYER] ❌ No video URLs found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
print("[LPAYER] Playwright not installed")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] Playwright error: {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _extract_with_http(self, url: str) -> Optional[str]:
|
||||||
|
"""Fallback: Extract video source using pure HTTP requests"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html_content = response.text
|
||||||
|
return self._extract_video_from_html(html_content)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] HTTP extraction error: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _extract_video_from_html(self, html_content: str) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Extract video URL from HTML using BeautifulSoup parsing
|
||||||
|
|
||||||
|
Looks for video URLs in this priority:
|
||||||
|
1. <video src="URL"> tags
|
||||||
|
2. <source src="URL"> tags
|
||||||
|
3. Direct URLs in page content with video extensions (.mp4, .m3u8)
|
||||||
|
|
||||||
|
Returns first valid URL found, or None if not found
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
soup = BeautifulSoup(html_content, 'lxml')
|
||||||
|
|
||||||
|
# Priority 1: Look for <video src="..."> tags
|
||||||
|
video_tags = soup.find_all('video')
|
||||||
|
for video in video_tags:
|
||||||
|
src = video.get('src')
|
||||||
|
if src and self._is_valid_video_url(src):
|
||||||
|
print(f"[LPAYER] Found video in <video> tag: {src[:80]}...")
|
||||||
|
return src
|
||||||
|
|
||||||
|
# Priority 2: Look for <source src="..."> tags
|
||||||
|
source_tags = soup.find_all('source')
|
||||||
|
for source in source_tags:
|
||||||
|
src = source.get('src')
|
||||||
|
if src and self._is_valid_video_url(src):
|
||||||
|
print(f"[LPAYER] Found video in <source> tag: {src[:80]}...")
|
||||||
|
return src
|
||||||
|
|
||||||
|
# Priority 3: Look for direct URLs in page content
|
||||||
|
patterns = [
|
||||||
|
r'"file"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, html_content)
|
||||||
|
for match in matches:
|
||||||
|
match = match.replace('\\', '').replace(r'\/', '/')
|
||||||
|
if self._is_valid_video_url(match):
|
||||||
|
print(f"[LPAYER] Found video in content: {match[:80]}...")
|
||||||
|
return match
|
||||||
|
|
||||||
|
print("[LPAYER] No video URL found in HTML")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LPAYER] HTML parsing error: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _is_valid_video_url(self, url: str) -> bool:
|
||||||
|
"""
|
||||||
|
Check if URL is a valid video URL
|
||||||
|
|
||||||
|
Valid if:
|
||||||
|
- Starts with http:// or https://
|
||||||
|
- Contains .mp4 or .m3u8 extension
|
||||||
|
"""
|
||||||
|
if not url:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Must be http(s) URL
|
||||||
|
if not url.startswith('http'):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Must contain video extension
|
||||||
|
url_lower = url.lower()
|
||||||
|
if '.mp4' not in url_lower and '.m3u8' not in url_lower:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
"""LuLuvid video hosting service downloader"""
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class LuLuvidDownloader(BaseVideoPlayer):
|
||||||
|
"""
|
||||||
|
Downloader for LuLuvid video hosting service.
|
||||||
|
|
||||||
|
LuLuvid is a video hosting platform used by various anime streaming sites.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this downloader can handle the given URL"""
|
||||||
|
return "luluv" in url.lower() or "luluvid" in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
target_filename: Optional[str] = None
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract direct download link and filename from LuLuvid URL.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: The LuLuvid video player URL
|
||||||
|
target_filename: Optional filename override
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (download_url, filename)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
logger.info(f"Fetching LuLuvid URL: {url}")
|
||||||
|
|
||||||
|
# Fetch the page
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
# Method 1: Look for video source in <video> tag
|
||||||
|
video_tag = soup.find('video')
|
||||||
|
if video_tag and video_tag.get('src'):
|
||||||
|
download_url = video_tag['src']
|
||||||
|
logger.info(f"Found video source from <video> tag")
|
||||||
|
else:
|
||||||
|
# Method 2: Look for source in <source> tag
|
||||||
|
source_tag = soup.find('source')
|
||||||
|
if source_tag and source_tag.get('src'):
|
||||||
|
download_url = source_tag['src']
|
||||||
|
logger.info(f"Found video source from <source> tag")
|
||||||
|
else:
|
||||||
|
# Method 3: Look for video URL in JavaScript
|
||||||
|
# LuLuvid often stores the video URL in a JavaScript variable
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
# Look for patterns like 'file:"URL"' or 'source:"URL"'
|
||||||
|
import re
|
||||||
|
patterns = [
|
||||||
|
r'file\s*:\s*["\']([^"\']+\.mp4[^"\']*)["\']',
|
||||||
|
r'source\s*:\s*["\']([^"\']+\.mp4[^"\']*)["\']',
|
||||||
|
r'videoUrl\s*:\s*["\']([^"\']+)["\']',
|
||||||
|
r'"url"\s*:\s*["\']([^"\']+\.mp4[^"\']*)["\']',
|
||||||
|
r'["\']src["\']\s*:\s*["\']([^"\']+\.mp4[^"\']*)["\']',
|
||||||
|
]
|
||||||
|
for pattern in patterns:
|
||||||
|
match = re.search(pattern, script.string)
|
||||||
|
if match:
|
||||||
|
download_url = match.group(1)
|
||||||
|
logger.info(f"Found video source from JavaScript")
|
||||||
|
break
|
||||||
|
if 'download_url' in locals():
|
||||||
|
break
|
||||||
|
|
||||||
|
if 'download_url' not in locals():
|
||||||
|
raise ValueError("Could not find video URL in page")
|
||||||
|
|
||||||
|
# Ensure URL is absolute
|
||||||
|
if not download_url.startswith('http'):
|
||||||
|
if download_url.startswith('//'):
|
||||||
|
download_url = 'https:' + download_url
|
||||||
|
else:
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
download_url = urljoin(url, download_url)
|
||||||
|
|
||||||
|
# Generate filename
|
||||||
|
if target_filename:
|
||||||
|
filename = sanitize_filename(target_filename)
|
||||||
|
else:
|
||||||
|
# Try to extract filename from URL
|
||||||
|
filename = download_url.split('/')[-1].split('?')[0]
|
||||||
|
if not filename or len(filename) < 5:
|
||||||
|
filename = "luluv_video.mp4"
|
||||||
|
filename = sanitize_filename(filename)
|
||||||
|
|
||||||
|
# Ensure .mp4 extension
|
||||||
|
if not filename.endswith('.mp4'):
|
||||||
|
filename += '.mp4'
|
||||||
|
|
||||||
|
logger.info(f"Successfully extracted LuLuvid download link: {filename}")
|
||||||
|
return download_url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error extracting LuLuvid download link: {e}")
|
||||||
|
raise ValueError(f"Failed to extract download link from LuLuvid: {str(e)}")
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
import asyncio
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
class OneuploadDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for oneupload.to video player"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return 'oneupload.to' in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: Optional[str] = None) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract download link from Oneupload video page
|
||||||
|
Oneupload uses a custom video player with dynamic loading
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: The Oneupload video page URL
|
||||||
|
target_filename: Optional filename override
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (direct_video_url, filename)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
print(f"[ONEUPLOAD] Extracting link from: {url}")
|
||||||
|
|
||||||
|
# Try using Playwright first (more reliable for dynamic content)
|
||||||
|
video_url = await self._extract_with_playwright(url)
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
# Fallback to HTTP extraction
|
||||||
|
video_url = await self._extract_with_http(url)
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
raise Exception("Could not find video URL in Oneupload page")
|
||||||
|
|
||||||
|
print(f"[ONEUPLOAD] Found video URL: {video_url[:80]}...")
|
||||||
|
|
||||||
|
# Generate filename
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
if target_filename:
|
||||||
|
filename = sanitize_filename(target_filename)
|
||||||
|
else:
|
||||||
|
# Try to extract filename from URL
|
||||||
|
filename = "oneupload_video.mp4"
|
||||||
|
|
||||||
|
return video_url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Oneupload link: {str(e)}")
|
||||||
|
|
||||||
|
async def _extract_with_playwright(self, url: str) -> str | None:
|
||||||
|
"""Extract video URL using Playwright with network interception"""
|
||||||
|
try:
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
print("[ONEUPLOAD] Launching browser with network interception...")
|
||||||
|
|
||||||
|
video_urls = []
|
||||||
|
|
||||||
|
async with async_playwright() as p:
|
||||||
|
browser = await p.chromium.launch(
|
||||||
|
headless=True,
|
||||||
|
args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
|
||||||
|
)
|
||||||
|
|
||||||
|
context = await browser.new_context(
|
||||||
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
|
||||||
|
)
|
||||||
|
|
||||||
|
page = await context.new_page()
|
||||||
|
|
||||||
|
# Set up response interception
|
||||||
|
async def handle_response(response):
|
||||||
|
try:
|
||||||
|
resp_url = response.url
|
||||||
|
content_type = response.headers.get('content-type', '')
|
||||||
|
|
||||||
|
# Look for video files in responses
|
||||||
|
if any(ext in resp_url.lower() for ext in ['.m3u8', '.mp4', '.mkv', '.ts']):
|
||||||
|
if 'oneupload' not in resp_url.lower() and 'google' not in resp_url.lower():
|
||||||
|
print(f"[ONEUPLOAD] 🎥 Captured video URL: {resp_url[:100]}...")
|
||||||
|
video_urls.append(resp_url)
|
||||||
|
# Also check by content-type
|
||||||
|
elif any(ct in content_type.lower() for ct in ['video/', 'application/x-mpegurl']):
|
||||||
|
if 'oneupload' not in resp_url.lower():
|
||||||
|
print(f"[ONEUPLOAD] 🎥 Captured video response: {resp_url[:100]}...")
|
||||||
|
video_urls.append(resp_url)
|
||||||
|
except Exception as e:
|
||||||
|
pass # Ignore interception errors
|
||||||
|
|
||||||
|
page.on('response', handle_response)
|
||||||
|
|
||||||
|
print("[ONEUPLOAD] Navigating to page...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
await page.goto(url, wait_until='networkidle', timeout=30000)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ONEUPLOAD] Navigation warning: {e}")
|
||||||
|
|
||||||
|
# Wait for page to load
|
||||||
|
print("[ONEUPLOAD] Waiting for video player to load...")
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
|
||||||
|
# Try to find and click play button
|
||||||
|
try:
|
||||||
|
play_selectors = [
|
||||||
|
'button[aria-label="Play"]',
|
||||||
|
'.play-button',
|
||||||
|
'button[class*="play"]',
|
||||||
|
'.jw-icon-display',
|
||||||
|
'video',
|
||||||
|
'.video-wrapper video',
|
||||||
|
]
|
||||||
|
|
||||||
|
for selector in play_selectors:
|
||||||
|
try:
|
||||||
|
element = await page.query_selector(selector)
|
||||||
|
if element:
|
||||||
|
print(f"[ONEUPLOAD] Found element: {selector}")
|
||||||
|
if 'button' in selector or 'jw' in selector:
|
||||||
|
await element.click()
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
break
|
||||||
|
except:
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ONEUPLOAD] Play button interaction: {e}")
|
||||||
|
|
||||||
|
# Wait more for network requests
|
||||||
|
await asyncio.sleep(4)
|
||||||
|
|
||||||
|
# Try JavaScript extraction
|
||||||
|
try:
|
||||||
|
js_code = r"""
|
||||||
|
() => {
|
||||||
|
// Check for JWPlayer setup
|
||||||
|
if (window.jwplayer) {
|
||||||
|
try {
|
||||||
|
const playlist = window.jwplayer().getPlaylist();
|
||||||
|
if (playlist && playlist[0] && playlist[0].sources) {
|
||||||
|
for (let source of playlist[0].sources) {
|
||||||
|
if (source.file && (source.file.includes('.m3u8') || source.file.includes('.mp4'))) {
|
||||||
|
return source.file;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch(e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check all video elements
|
||||||
|
const videos = document.querySelectorAll('video');
|
||||||
|
for (let v of videos) {
|
||||||
|
if (v.src && (v.src.includes('.m3u8') || v.src.includes('.mp4'))) {
|
||||||
|
return v.src;
|
||||||
|
}
|
||||||
|
const sources = v.querySelectorAll('source');
|
||||||
|
for (let s of sources) {
|
||||||
|
if (s.src && (s.src.includes('.m3u8') || s.src.includes('.mp4'))) {
|
||||||
|
return s.src;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check window object for video URLs
|
||||||
|
const searchKeys = ['player', 'video', 'source', 'file', 'url'];
|
||||||
|
for (let key of searchKeys) {
|
||||||
|
if (window[key] && typeof window[key] === 'object') {
|
||||||
|
try {
|
||||||
|
const json = JSON.stringify(window[key]);
|
||||||
|
const match = json.match(/(https?:\/\/[^\s"\'<>]+\.(m3u8|mp4))/);
|
||||||
|
if (match) return match[1];
|
||||||
|
} catch(e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
js_result = await page.evaluate(js_code)
|
||||||
|
|
||||||
|
if js_result and ('.m3u8' in js_result or '.mp4' in js_result):
|
||||||
|
print(f"[ONEUPLOAD] ✅ Found video URL via JavaScript: {js_result[:100]}...")
|
||||||
|
video_urls.append(js_result)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ONEUPLOAD] JS extraction error: {e}")
|
||||||
|
|
||||||
|
# Parse page HTML for video URLs
|
||||||
|
try:
|
||||||
|
content = await page.content()
|
||||||
|
patterns = [
|
||||||
|
r'"file"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'"source"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"source"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
r"url\s*[:=]\s*['\"]([^'\"]+\.m3u8[^'\"]*)['\"]",
|
||||||
|
r"url\s*[:=]\s*['\"]([^'\"]+\.mp4[^'\"]*)['\"]",
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, content, re.IGNORECASE)
|
||||||
|
for match in matches:
|
||||||
|
# Clean up the URL
|
||||||
|
match = match.replace('\\/', '/').replace('\\', '')
|
||||||
|
if 'http' in match and 'oneupload' not in match and 'google' not in match:
|
||||||
|
print(f"[ONEUPLOAD] Found in HTML: {match[:100]}...")
|
||||||
|
video_urls.append(match)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ONEUPLOAD] HTML parsing error: {e}")
|
||||||
|
|
||||||
|
await browser.close()
|
||||||
|
|
||||||
|
# Return first valid video URL (prefer .m3u8 over .mp4)
|
||||||
|
if video_urls:
|
||||||
|
seen = set()
|
||||||
|
unique_urls = []
|
||||||
|
for vid_url in video_urls:
|
||||||
|
if vid_url not in seen:
|
||||||
|
seen.add(vid_url)
|
||||||
|
unique_urls.append(vid_url)
|
||||||
|
|
||||||
|
if unique_urls:
|
||||||
|
# Sort to prefer .m3u8 (source quality)
|
||||||
|
unique_urls.sort(key=lambda x: 0 if '.m3u8' in x else 1)
|
||||||
|
print(f"[ONEUPLOAD] ✅ Found {len(unique_urls)} video URL(s)")
|
||||||
|
print(f"[ONEUPLOAD] Selected: {unique_urls[0][:100]}...")
|
||||||
|
return unique_urls[0]
|
||||||
|
|
||||||
|
print("[ONEUPLOAD] ❌ No video URLs found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
print("[ONEUPLOAD] ⚠️ Playwright not installed - using HTTP extraction only")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ONEUPLOAD] Playwright error: {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _extract_with_http(self, url: str) -> str | None:
|
||||||
|
"""Extract video URL using simple HTTP requests"""
|
||||||
|
try:
|
||||||
|
print(f"[ONEUPLOAD] Trying HTTP extraction from: {url}")
|
||||||
|
|
||||||
|
response = await self.client.get(url, follow_redirects=True)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Method 1: Look for video/source tags
|
||||||
|
videos = soup.find_all('video')
|
||||||
|
for video in videos:
|
||||||
|
src = video.get('src') or video.get('data-src')
|
||||||
|
if src and any(ext in src for ext in ['.m3u8', '.mp4']):
|
||||||
|
print(f"[ONEUPLOAD] ✅ Found video in video tag: {src[:100]}...")
|
||||||
|
return src
|
||||||
|
|
||||||
|
sources = video.find_all('source')
|
||||||
|
for source in sources:
|
||||||
|
src = source.get('src')
|
||||||
|
if src and any(ext in src for ext in ['.m3u8', '.mp4']):
|
||||||
|
print(f"[ONEUPLOAD] ✅ Found video in source tag: {src[:100]}...")
|
||||||
|
return src
|
||||||
|
|
||||||
|
# Method 2: Look in script tags for video URLs
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
patterns = [
|
||||||
|
r'"file"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'"source"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"source"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, script.string, re.IGNORECASE)
|
||||||
|
for match in matches:
|
||||||
|
match = match.replace('\\/', '/')
|
||||||
|
if 'http' in match and 'oneupload' not in match.lower():
|
||||||
|
print(f"[ONEUPLOAD] ✅ Found video in script: {match[:100]}...")
|
||||||
|
return match
|
||||||
|
|
||||||
|
print("[ONEUPLOAD] ❌ HTTP extraction failed - no video URLs found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ONEUPLOAD] HTTP extraction error: {e}")
|
||||||
|
return None
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
|
||||||
|
class RapidFileDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for rapidfile.net and similar hosts"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return any(domain in url.lower() for domain in ["rapidfile.net", "rapidfile.com", "rapid-file"])
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str) -> tuple[str, str]:
|
||||||
|
try:
|
||||||
|
# Get the initial page
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
download_url = None
|
||||||
|
filename = "rapidfile_download"
|
||||||
|
|
||||||
|
# Method 1: Look for download button/link
|
||||||
|
download_btn = soup.find('a', {'id': 'downloadbtn'}) or soup.find('a', class_='download-btn')
|
||||||
|
if download_btn and download_btn.get('href'):
|
||||||
|
download_url = download_btn['href']
|
||||||
|
|
||||||
|
# Method 2: Look for form with POST action
|
||||||
|
if not download_url:
|
||||||
|
forms = soup.find_all('form')
|
||||||
|
for form in forms:
|
||||||
|
action = form.get('action', '')
|
||||||
|
if action and ('download' in action.lower() or 'file' in action.lower()):
|
||||||
|
download_url = action if action.startswith('http') else url + action
|
||||||
|
break
|
||||||
|
|
||||||
|
# Method 3: Look for any link with download/file in URL
|
||||||
|
if not download_url:
|
||||||
|
for link in soup.find_all('a', href=True):
|
||||||
|
href = link['href']
|
||||||
|
if any(keyword in href.lower() for keyword in ['download', 'get_file', 'file.php']):
|
||||||
|
if href.startswith('http'):
|
||||||
|
download_url = href
|
||||||
|
break
|
||||||
|
|
||||||
|
# Method 4: Check for direct file links in scripts
|
||||||
|
if not download_url:
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
match = re.search(r'(https?://[^\s\"\'<>]+/(?:download|file)[^\s\"\'<>]+)', script.string)
|
||||||
|
if match:
|
||||||
|
download_url = match.group(0)
|
||||||
|
break
|
||||||
|
|
||||||
|
if download_url:
|
||||||
|
# Get filename from headers or URL
|
||||||
|
try:
|
||||||
|
head_resp = await self.client.head(download_url, timeout=5.0)
|
||||||
|
fname = self._extract_filename_from_headers(head_resp.headers)
|
||||||
|
if fname:
|
||||||
|
filename = fname
|
||||||
|
else:
|
||||||
|
filename = download_url.split('/')[-1] or "rapidfile_download"
|
||||||
|
except:
|
||||||
|
filename = download_url.split('/')[-1] or "rapidfile_download"
|
||||||
|
|
||||||
|
return download_url, filename
|
||||||
|
|
||||||
|
# If all else fails, return the original URL
|
||||||
|
filename = url.split('/')[-1] or "rapidfile_download"
|
||||||
|
return url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Rapidfile link: {str(e)}")
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
from typing import Optional
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from .base import BaseVideoPlayer
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
class SendVidDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for SendVid videos"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return "sendvid.com" in url.lower()
|
||||||
|
|
||||||
|
async def _fetch_page(self, url: str) -> str:
|
||||||
|
"""Fetch page with proper headers to avoid 403 errors"""
|
||||||
|
headers = {
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
|
||||||
|
'Referer': 'https://sendvid.com/',
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.5',
|
||||||
|
}
|
||||||
|
response = await self.client.get(url, headers=headers)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.text
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: str = None) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract direct download link from SendVid embed page
|
||||||
|
SendVid embed pages contain the direct MP4 URL in a <source> tag
|
||||||
|
"""
|
||||||
|
print(f"[SENDVID] Fetching page: {url}")
|
||||||
|
|
||||||
|
html = await self._fetch_page(url)
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
# Try to find the video source in the <source> tag
|
||||||
|
source_tag = soup.find('source', {'id': 'video_source'})
|
||||||
|
if source_tag and source_tag.get('src'):
|
||||||
|
video_url = source_tag['src']
|
||||||
|
print(f"[SENDVID] Found video URL in <source> tag")
|
||||||
|
|
||||||
|
# Generate filename
|
||||||
|
if target_filename:
|
||||||
|
filename = target_filename
|
||||||
|
else:
|
||||||
|
# Extract filename from video URL or generate one
|
||||||
|
filename = self._extract_filename_from_url(url, video_url)
|
||||||
|
|
||||||
|
print(f"[SENDVID] Download URL: {video_url}")
|
||||||
|
print(f"[SENDVID] Filename: {filename}")
|
||||||
|
return video_url, filename
|
||||||
|
|
||||||
|
# Fallback: try to find in og:video meta property
|
||||||
|
og_video = soup.find('meta', {'property': 'og:video'})
|
||||||
|
if og_video and og_video.get('content'):
|
||||||
|
video_url = og_video['content']
|
||||||
|
print(f"[SENDVID] Found video URL in og:video meta")
|
||||||
|
|
||||||
|
if target_filename:
|
||||||
|
filename = target_filename
|
||||||
|
else:
|
||||||
|
filename = self._extract_filename_from_url(url, video_url)
|
||||||
|
|
||||||
|
print(f"[SENDVID] Download URL: {video_url}")
|
||||||
|
print(f"[SENDVID] Filename: {filename}")
|
||||||
|
return video_url, filename
|
||||||
|
|
||||||
|
raise Exception("Could not extract video URL from SendVid page")
|
||||||
|
|
||||||
|
def _extract_filename_from_url(self, page_url: str, video_url: str) -> str:
|
||||||
|
"""Generate filename from SendVod URLs"""
|
||||||
|
# Try to extract video ID from page URL
|
||||||
|
video_id_match = re.search(r'/embed/([a-z0-9]+)', page_url)
|
||||||
|
if video_id_match:
|
||||||
|
video_id = video_id_match.group(1)
|
||||||
|
# Try to get title from page (might need to fetch, but for now use ID)
|
||||||
|
return f"sendvid_{video_id}.mp4"
|
||||||
|
|
||||||
|
# Fallback: extract from video URL
|
||||||
|
filename_match = re.search(r'/([^/]+\.mp4)', video_url)
|
||||||
|
if filename_match:
|
||||||
|
return filename_match.group(1)
|
||||||
|
|
||||||
|
return "sendvid_video.mp4"
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
|
||||||
|
class SibnetDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for sibnet.ru video player"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return 'sibnet.ru' in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: str = None) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract download link from Sibnet video page
|
||||||
|
Sibnet uses a JavaScript player with direct MP4 links
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
print(f"[SIBNET] Extracting link from: {url}")
|
||||||
|
|
||||||
|
# If it's already a direct MP4 URL, return it as-is
|
||||||
|
if url.endswith('.mp4'):
|
||||||
|
print(f"[SIBNET] Direct MP4 URL detected")
|
||||||
|
filename = url.split('/')[-1] or "sibnet_video.mp4"
|
||||||
|
return url, filename
|
||||||
|
|
||||||
|
# Fetch the video page
|
||||||
|
response = await self.client.get(
|
||||||
|
url,
|
||||||
|
headers={
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Parse HTML to find the video source
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Look for player.src in JavaScript
|
||||||
|
# Pattern: player.src([{src: "/v/HASH/ID.mp4", type: "video/mp4"},]);
|
||||||
|
script_tags = soup.find_all('script')
|
||||||
|
video_url = None
|
||||||
|
|
||||||
|
for script in script_tags:
|
||||||
|
if script.string:
|
||||||
|
# Look for player.src pattern
|
||||||
|
match = re.search(r'player\.src\(\[\{src:\s*"([^"]+\.mp4)"', script.string)
|
||||||
|
if match:
|
||||||
|
video_url = match.group(1)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Alternative pattern
|
||||||
|
match = re.search(r'"([^"]+\.mp4)"[^}]*type:\s*"video/mp4"', script.string)
|
||||||
|
if match:
|
||||||
|
video_url = match.group(1)
|
||||||
|
# Make sure it's from /v/ directory
|
||||||
|
if video_url.startswith('/v/'):
|
||||||
|
break
|
||||||
|
video_url = None
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
# Try to find any .mp4 URL in the page
|
||||||
|
mp4_match = re.search(r'"/v/[^"]+\.mp4"', response.text)
|
||||||
|
if mp4_match:
|
||||||
|
video_url = mp4_match.group(0).strip('"')
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
raise Exception("Could not find video URL in Sibnet page")
|
||||||
|
|
||||||
|
# Convert relative URL to absolute
|
||||||
|
if video_url.startswith('/'):
|
||||||
|
video_url = urljoin('https://video.sibnet.ru/', video_url)
|
||||||
|
|
||||||
|
print(f"[SIBNET] Found video URL: {video_url[:80]}...")
|
||||||
|
|
||||||
|
# Generate filename from URL or use default
|
||||||
|
filename_match = re.search(r'/([^/]+)\.mp4', video_url)
|
||||||
|
if filename_match:
|
||||||
|
filename = f"{filename_match.group(1)}.mp4"
|
||||||
|
else:
|
||||||
|
filename = "sibnet_video.mp4"
|
||||||
|
|
||||||
|
return video_url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Sibnet link: {str(e)}")
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
import asyncio
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
class SmoothpreDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for smoothpre.com video player (JWPlayer-based)"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return 'smoothpre.com' in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: Optional[str] = None) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract download link from Smoothpre video page
|
||||||
|
Smoothpre uses JWPlayer with dynamic JavaScript - requires Playwright
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: The Smoothpre video page URL
|
||||||
|
target_filename: Optional filename override
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (direct_video_url, filename)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
print(f"[SMOOTHPRE] Extracting link from: {url}")
|
||||||
|
|
||||||
|
# Try using Playwright to extract video URL
|
||||||
|
video_url = await self._extract_with_playwright(url)
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
raise Exception("Could not find video URL in Smoothpre page")
|
||||||
|
|
||||||
|
print(f"[SMOOTHPRE] Found video URL: {video_url[:80]}...")
|
||||||
|
|
||||||
|
# Generate filename
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
if target_filename:
|
||||||
|
filename = sanitize_filename(target_filename)
|
||||||
|
else:
|
||||||
|
filename = "smoothpre_video.mp4"
|
||||||
|
|
||||||
|
return video_url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Smoothpre link: {str(e)}")
|
||||||
|
|
||||||
|
async def _extract_with_playwright(self, url: str) -> str | None:
|
||||||
|
"""Extract video URL using Playwright with network interception"""
|
||||||
|
try:
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
print("[SMOOTHPRE] Launching browser with network interception...")
|
||||||
|
|
||||||
|
video_urls = []
|
||||||
|
|
||||||
|
async with async_playwright() as p:
|
||||||
|
browser = await p.chromium.launch(
|
||||||
|
headless=True,
|
||||||
|
args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
|
||||||
|
)
|
||||||
|
|
||||||
|
context = await browser.new_context(
|
||||||
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
|
||||||
|
)
|
||||||
|
|
||||||
|
page = await context.new_page()
|
||||||
|
|
||||||
|
# Set up response interception
|
||||||
|
async def handle_response(response):
|
||||||
|
try:
|
||||||
|
resp_url = response.url
|
||||||
|
content_type = response.headers.get('content-type', '')
|
||||||
|
|
||||||
|
# Look for video files in responses
|
||||||
|
if any(ext in resp_url.lower() for ext in ['.m3u8', '.mp4', '.mkv', '.ts']):
|
||||||
|
if 'smoothpre' not in resp_url.lower() and 'google' not in resp_url.lower():
|
||||||
|
print(f"[SMOOTHPRE] 🎥 Captured video URL: {resp_url[:100]}...")
|
||||||
|
video_urls.append(resp_url)
|
||||||
|
# Also check by content-type
|
||||||
|
elif any(ct in content_type.lower() for ct in ['video/', 'application/x-mpegurl']):
|
||||||
|
if 'smoothpre' not in resp_url.lower():
|
||||||
|
print(f"[SMOOTHPRE] 🎥 Captured video response: {resp_url[:100]}...")
|
||||||
|
video_urls.append(resp_url)
|
||||||
|
except Exception as e:
|
||||||
|
pass # Ignore interception errors
|
||||||
|
|
||||||
|
page.on('response', handle_response)
|
||||||
|
|
||||||
|
print("[SMOOTHPRE] Navigating to page...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
await page.goto(url, wait_until='networkidle', timeout=30000)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[SMOOTHPRE] Navigation warning: {e}")
|
||||||
|
|
||||||
|
# Wait for page to load
|
||||||
|
print("[SMOOTHPRE] Waiting for video player to load...")
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
|
||||||
|
# Try to find and click play button
|
||||||
|
try:
|
||||||
|
play_selectors = [
|
||||||
|
'button[aria-label="Play"]',
|
||||||
|
'.play-button',
|
||||||
|
'button[class*="play"]',
|
||||||
|
'.jw-icon-display',
|
||||||
|
'video',
|
||||||
|
]
|
||||||
|
|
||||||
|
for selector in play_selectors:
|
||||||
|
try:
|
||||||
|
element = await page.query_selector(selector)
|
||||||
|
if element:
|
||||||
|
print(f"[SMOOTHPRE] Found element: {selector}")
|
||||||
|
if 'button' in selector or 'jw' in selector:
|
||||||
|
await element.click()
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
break
|
||||||
|
except:
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[SMOOTHPRE] Play button interaction: {e}")
|
||||||
|
|
||||||
|
# Wait more for network requests
|
||||||
|
await asyncio.sleep(4)
|
||||||
|
|
||||||
|
# Try JavaScript extraction - JWPlayer specific
|
||||||
|
try:
|
||||||
|
js_code = r"""
|
||||||
|
() => {
|
||||||
|
// Check for JWPlayer setup (primary method for Smoothpre)
|
||||||
|
if (window.jwplayer) {
|
||||||
|
try {
|
||||||
|
const playlist = window.jwplayer().getPlaylist();
|
||||||
|
if (playlist && playlist[0] && playlist[0].sources) {
|
||||||
|
for (let source of playlist[0].sources) {
|
||||||
|
if (source.file && (source.file.includes('.m3u8') || source.file.includes('.mp4'))) {
|
||||||
|
return source.file;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch(e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check all video elements
|
||||||
|
const videos = document.querySelectorAll('video');
|
||||||
|
for (let v of videos) {
|
||||||
|
if (v.src && (v.src.includes('.m3u8') || v.src.includes('.mp4'))) {
|
||||||
|
return v.src;
|
||||||
|
}
|
||||||
|
const sources = v.querySelectorAll('source');
|
||||||
|
for (let s of sources) {
|
||||||
|
if (s.src && (s.src.includes('.m3u8') || s.src.includes('.mp4'))) {
|
||||||
|
return s.src;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check window object for video URLs
|
||||||
|
const searchKeys = ['player', 'video', 'source', 'file', 'url', 'jw'];
|
||||||
|
for (let key of searchKeys) {
|
||||||
|
if (window[key] && typeof window[key] === 'object') {
|
||||||
|
try {
|
||||||
|
const json = JSON.stringify(window[key]);
|
||||||
|
const match = json.match(/(https?:\/\/[^\s"\'<>]+\.(m3u8|mp4))/);
|
||||||
|
if (match) return match[1];
|
||||||
|
} catch(e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
js_result = await page.evaluate(js_code)
|
||||||
|
|
||||||
|
if js_result and ('.m3u8' in js_result or '.mp4' in js_result):
|
||||||
|
print(f"[SMOOTHPRE] ✅ Found video URL via JavaScript: {js_result[:100]}...")
|
||||||
|
video_urls.append(js_result)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[SMOOTHPRE] JS extraction error: {e}")
|
||||||
|
|
||||||
|
# Parse page HTML for video URLs - enhanced patterns
|
||||||
|
try:
|
||||||
|
content = await page.content()
|
||||||
|
patterns = [
|
||||||
|
r'"file"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'"source"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"source"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
r"url\s*[:=]\s*['\"]([^'\"]+\.m3u8[^'\"]*)['\"]",
|
||||||
|
r"url\s*[:=]\s*['\"]([^'\"]+\.mp4[^'\"]*)['\"]",
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, content, re.IGNORECASE)
|
||||||
|
for match in matches:
|
||||||
|
# Clean up the URL
|
||||||
|
match = match.replace('\\/', '/').replace('\\', '')
|
||||||
|
if 'http' in match and 'smoothpre' not in match and 'google' not in match:
|
||||||
|
print(f"[SMOOTHPRE] Found in HTML: {match[:100]}...")
|
||||||
|
video_urls.append(match)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[SMOOTHPRE] HTML parsing error: {e}")
|
||||||
|
|
||||||
|
await browser.close()
|
||||||
|
|
||||||
|
# Return first valid video URL (prefer .m3u8 over .mp4 as it's usually the source)
|
||||||
|
if video_urls:
|
||||||
|
seen = set()
|
||||||
|
unique_urls = []
|
||||||
|
for vid_url in video_urls:
|
||||||
|
if vid_url not in seen:
|
||||||
|
seen.add(vid_url)
|
||||||
|
unique_urls.append(vid_url)
|
||||||
|
|
||||||
|
if unique_urls:
|
||||||
|
# Sort to prefer .m3u8 (source quality)
|
||||||
|
unique_urls.sort(key=lambda x: 0 if '.m3u8' in x else 1)
|
||||||
|
print(f"[SMOOTHPRE] ✅ Found {len(unique_urls)} video URL(s)")
|
||||||
|
print(f"[SMOOTHPRE] Selected: {unique_urls[0][:100]}...")
|
||||||
|
return unique_urls[0]
|
||||||
|
|
||||||
|
print("[SMOOTHPRE] ❌ No video URLs found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
print("[SMOOTHPRE] ⚠️ Playwright not installed - falling back to HTTP extraction")
|
||||||
|
return await self._extract_with_http(url)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[SMOOTHPRE] Playwright error: {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
# Fallback to HTTP extraction
|
||||||
|
return await self._extract_with_http(url)
|
||||||
|
|
||||||
|
async def _extract_with_http(self, url: str) -> str | None:
|
||||||
|
"""Extract video URL using simple HTTP requests (fallback when Playwright fails)"""
|
||||||
|
try:
|
||||||
|
print(f"[SMOOTHPRE] Trying HTTP extraction from: {url}")
|
||||||
|
|
||||||
|
response = await self.client.get(url, follow_redirects=True)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Method 1: Look for video/source tags
|
||||||
|
videos = soup.find_all('video')
|
||||||
|
for video in videos:
|
||||||
|
src = video.get('src') or video.get('data-src')
|
||||||
|
if src and any(ext in src for ext in ['.m3u8', '.mp4']):
|
||||||
|
print(f"[SMOOTHPRE] ✅ Found video in video tag: {src[:100]}...")
|
||||||
|
return src
|
||||||
|
|
||||||
|
sources = video.find_all('source')
|
||||||
|
for source in sources:
|
||||||
|
src = source.get('src')
|
||||||
|
if src and any(ext in src for ext in ['.m3u8', '.mp4']):
|
||||||
|
print(f"[SMOOTHPRE] ✅ Found video in source tag: {src[:100]}...")
|
||||||
|
return src
|
||||||
|
|
||||||
|
# Method 2: Look in script tags for JWPlayer configuration
|
||||||
|
scripts = soup.find_all('script')
|
||||||
|
for script in scripts:
|
||||||
|
if script.string:
|
||||||
|
# JWPlayer patterns
|
||||||
|
patterns = [
|
||||||
|
r'"file"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'"source"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"source"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, script.string, re.IGNORECASE)
|
||||||
|
for match in matches:
|
||||||
|
match = match.replace('\\/', '/')
|
||||||
|
if 'http' in match and 'smoothpre' not in match.lower():
|
||||||
|
print(f"[SMOOTHPRE] ✅ Found video in script: {match[:100]}...")
|
||||||
|
return match
|
||||||
|
|
||||||
|
print("[SMOOTHPRE] ❌ HTTP extraction failed - no video URLs found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[SMOOTHPRE] HTTP extraction error: {e}")
|
||||||
|
return None
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
|
||||||
|
class UnFichierDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for 1fichier.com"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return any(domain in url.lower() for domain in ["1fichier.com", "1fichier.fr"])
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str) -> tuple[str, str]:
|
||||||
|
try:
|
||||||
|
# Initial page
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
# Check if we need to wait (download button)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Check for direct download link
|
||||||
|
download_link = soup.find('a', class_='btn btn-download')
|
||||||
|
if download_link and download_link.get('href'):
|
||||||
|
download_url = download_link['href']
|
||||||
|
# Follow to get headers for filename
|
||||||
|
head_resp = await self.client.head(download_url)
|
||||||
|
filename = self._extract_filename_from_headers(head_resp.headers)
|
||||||
|
if not filename:
|
||||||
|
filename = download_url.split('/')[-1] or "downloaded_file"
|
||||||
|
return download_url, filename
|
||||||
|
|
||||||
|
# Alternative: look for any download link in the page
|
||||||
|
for link in soup.find_all('a', href=True):
|
||||||
|
href = link['href']
|
||||||
|
if href.startswith('http') and '1fichier' not in href:
|
||||||
|
# Try to head the URL to see if it's a file
|
||||||
|
try:
|
||||||
|
head_resp = await self.client.head(href, timeout=5.0)
|
||||||
|
if 'content-length' in head_resp.headers or 'attachment' in head_resp.headers.get('content-disposition', ''):
|
||||||
|
filename = self._extract_filename_from_headers(head_resp.headers)
|
||||||
|
if not filename:
|
||||||
|
filename = href.split('/')[-1] or "downloaded_file"
|
||||||
|
return href, filename
|
||||||
|
except:
|
||||||
|
continue
|
||||||
|
|
||||||
|
raise Exception("Could not find download link on page")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting 1fichier link: {str(e)}")
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
class UptoboxDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for uptobox.com"""
|
||||||
|
|
||||||
|
BASE_DOMAINS = ["uptobox.com", "uptobox.fr"]
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return any(domain in url.lower() for domain in self.BASE_DOMAINS)
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str) -> tuple[str, str]:
|
||||||
|
"""Extract direct download link from uptobox"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(url, follow_redirects=True)
|
||||||
|
soup = BeautifulSoup(response.text, 'lxml')
|
||||||
|
|
||||||
|
# Method 1: Look for direct download button/link
|
||||||
|
download_btn = soup.find('a', {'id': 'directDownload'}) or soup.find('a', class_='download-btn')
|
||||||
|
if download_btn and download_btn.get('href'):
|
||||||
|
href = download_btn['href']
|
||||||
|
filename = self._extract_filename_from_url(url) or "uptobox_file"
|
||||||
|
return href, filename
|
||||||
|
|
||||||
|
# Method 2: Look for any download link in page
|
||||||
|
links = soup.find_all('a', href=True)
|
||||||
|
for link in links:
|
||||||
|
href = link['href']
|
||||||
|
text = link.get_text().lower()
|
||||||
|
if any(keyword in text for keyword in ['download', 'télécharger', 'ddl']):
|
||||||
|
if href.startswith('http'):
|
||||||
|
filename = self._extract_filename_from_url(url) or "uptobox_file"
|
||||||
|
return href, filename
|
||||||
|
|
||||||
|
# Method 3: Return the original URL (uptobox handles downloads directly)
|
||||||
|
filename = self._extract_filename_from_url(url) or "uptobox_file"
|
||||||
|
return url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting Uptobox link: {str(e)}")
|
||||||
|
|
||||||
|
def _extract_filename_from_url(self, url: str) -> str | None:
|
||||||
|
"""Try to extract filename from URL"""
|
||||||
|
# Look for filename parameter in URL
|
||||||
|
match = re.search(r'[&?]filename=([^&]+)', url)
|
||||||
|
if match:
|
||||||
|
from urllib.parse import unquote
|
||||||
|
return unquote(match.group(1))
|
||||||
|
|
||||||
|
# Extract from path
|
||||||
|
parts = url.split('/')
|
||||||
|
if len(parts) > 0:
|
||||||
|
last_part = parts[-1]
|
||||||
|
if '.' in last_part:
|
||||||
|
return last_part
|
||||||
|
|
||||||
|
return None
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
"""Uqload video hosting service downloader"""
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from typing import Optional
|
||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class UqloadDownloader(BaseVideoPlayer):
|
||||||
|
"""
|
||||||
|
Downloader for Uqload video hosting service.
|
||||||
|
|
||||||
|
Uqload is a video hosting platform used by French Stream and other streaming sites.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this downloader can handle the given URL"""
|
||||||
|
return "uqload" in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
target_filename: Optional[str] = None
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract direct download link and filename from Uqload URL.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: The Uqload video player URL
|
||||||
|
target_filename: Optional filename override
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (download_url, filename)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
logger.info(f"Fetching Uqload URL: {url}")
|
||||||
|
|
||||||
|
# Fetch the page
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
# Method 1: Look for video URL in JavaScript
|
||||||
|
# Uqload stores the video URL in a JavaScript variable like: sources: ["URL"]
|
||||||
|
patterns = [
|
||||||
|
r'sources:\s*\["([^"]+\.mp4[^"]*)"\]',
|
||||||
|
r'sources:\s*\[["\']([^"\']+\.mp4[^"\']*)["\']\]',
|
||||||
|
r'"sources":\s*\["([^"]+\.mp4[^"]*)"\]',
|
||||||
|
r'file:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r'file:\s*["\']([^"\']+\.mp4[^"\']*)["\']',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
match = re.search(pattern, html)
|
||||||
|
if match:
|
||||||
|
download_url = match.group(1)
|
||||||
|
# Clean up any escape characters
|
||||||
|
download_url = download_url.replace('\\/', '/')
|
||||||
|
logger.info(f"Found video source from JavaScript pattern: {pattern[:20]}...")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
# Method 2: Try parsing with BeautifulSoup
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
# Look for video tag
|
||||||
|
video_tag = soup.find('video')
|
||||||
|
if video_tag and video_tag.get('src'):
|
||||||
|
download_url = video_tag['src']
|
||||||
|
logger.info(f"Found video source from <video> tag")
|
||||||
|
else:
|
||||||
|
# Look for source tag
|
||||||
|
source_tag = soup.find('source')
|
||||||
|
if source_tag and source_tag.get('src'):
|
||||||
|
download_url = source_tag['src']
|
||||||
|
logger.info(f"Found video source from <source> tag")
|
||||||
|
else:
|
||||||
|
raise ValueError("Could not find video URL in Uqload page")
|
||||||
|
|
||||||
|
# Ensure URL is absolute
|
||||||
|
if not download_url.startswith('http'):
|
||||||
|
if download_url.startswith('//'):
|
||||||
|
download_url = 'https:' + download_url
|
||||||
|
else:
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
download_url = urljoin(url, download_url)
|
||||||
|
|
||||||
|
# Generate filename
|
||||||
|
if target_filename:
|
||||||
|
filename = sanitize_filename(target_filename)
|
||||||
|
else:
|
||||||
|
# Try to extract filename from URL
|
||||||
|
filename = download_url.split('/')[-1].split('?')[0]
|
||||||
|
if not filename or len(filename) < 5:
|
||||||
|
filename = "uqload_video.mp4"
|
||||||
|
filename = sanitize_filename(filename)
|
||||||
|
|
||||||
|
# Ensure .mp4 extension
|
||||||
|
if not filename.endswith('.mp4'):
|
||||||
|
filename += '.mp4'
|
||||||
|
|
||||||
|
logger.info(f"Successfully extracted Uqload download link: {filename}")
|
||||||
|
return download_url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error extracting Uqload download link: {e}")
|
||||||
|
raise ValueError(f"Failed to extract download link from Uqload: {str(e)}")
|
||||||
@@ -0,0 +1,447 @@
|
|||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import re
|
||||||
|
import httpx
|
||||||
|
import subprocess
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
import asyncio
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
class VidMolyDownloader(BaseVideoPlayer):
|
||||||
|
"""Downloader for vidmoly.to using Playwright network interception"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
return any(domain in url.lower() for domain in ["vidmoly.to", "vidmoly.org", "vidmoly.biz"])
|
||||||
|
|
||||||
|
async def get_download_link(self, url: str, target_filename: str = None) -> tuple[str, str]:
|
||||||
|
try:
|
||||||
|
# Extract VidMoly ID from URL
|
||||||
|
vidmoly_id = self._extract_vidmoly_id(url)
|
||||||
|
if not vidmoly_id:
|
||||||
|
raise Exception("Could not extract VidMoly ID from URL")
|
||||||
|
|
||||||
|
# Construct embed URL - try vidmoly.biz first (it works better than .to/.org)
|
||||||
|
# If original URL uses .biz, keep it. Otherwise try .biz first
|
||||||
|
domains_to_try = []
|
||||||
|
|
||||||
|
if "vidmoly.biz" in url.lower():
|
||||||
|
domains_to_try = ["vidmoly.biz"]
|
||||||
|
elif "vidmoly.to" in url.lower() or "vidmoly.org" in url.lower():
|
||||||
|
# For .to/.org, try .biz first (it has actual content), then original
|
||||||
|
domains_to_try = ["vidmoly.biz", url.split("//")[1].split("/")[0]]
|
||||||
|
else:
|
||||||
|
domains_to_try = ["vidmoly.biz", "vidmoly.to"]
|
||||||
|
|
||||||
|
video_source = None
|
||||||
|
last_error = None
|
||||||
|
working_domain = None
|
||||||
|
|
||||||
|
for domain in domains_to_try:
|
||||||
|
embed_url = f"https://{domain}/embed-{vidmoly_id}.html"
|
||||||
|
|
||||||
|
print(f"[VIDMOLY] Trying: {embed_url}")
|
||||||
|
print(f"[VIDMOLY] VidMoly ID: {vidmoly_id}")
|
||||||
|
|
||||||
|
# Use Playwright with network interception
|
||||||
|
video_source = await self._extract_with_playwright_network(embed_url)
|
||||||
|
|
||||||
|
if not video_source:
|
||||||
|
# Fallback to HTTP method
|
||||||
|
print("[VIDMOLY] Playwright failed, trying HTTP fallback...")
|
||||||
|
video_source = await self._extract_with_http(embed_url)
|
||||||
|
|
||||||
|
if video_source:
|
||||||
|
print(f"[VIDMOLY] ✅ Found video on {domain}")
|
||||||
|
working_domain = domain
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print(f"[VIDMOLY] ❌ No video on {domain}")
|
||||||
|
last_error = f"No video found on {domain}"
|
||||||
|
|
||||||
|
if not video_source:
|
||||||
|
raise Exception(f"Could not find video source - tried: {', '.join(domains_to_try)}. Last error: {last_error}")
|
||||||
|
|
||||||
|
# Validate that video_source is not an embed URL
|
||||||
|
if 'vidmoly' in video_source.lower() and ('embed-' in video_source or '.html' in video_source):
|
||||||
|
raise Exception(f"Extracted URL is still a VidMoly embed page, not a video: {video_source[:100]}")
|
||||||
|
|
||||||
|
# Use target_filename if provided, otherwise generate default
|
||||||
|
filename = target_filename if target_filename else f"vidmoly_{vidmoly_id}"
|
||||||
|
|
||||||
|
# Check if it's an M3U8 playlist
|
||||||
|
if '.m3u8' in video_source:
|
||||||
|
print(f"[VIDMOLY] Found M3U8 source: {video_source[:100]}...")
|
||||||
|
|
||||||
|
# Download and convert M3U8 to MP4 directly
|
||||||
|
headers = {
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||||
|
'Referer': f'https://{working_domain}/',
|
||||||
|
}
|
||||||
|
|
||||||
|
mp4_path = await self._download_m3u8_as_mp4(video_source, filename, headers)
|
||||||
|
|
||||||
|
return mp4_path, filename
|
||||||
|
|
||||||
|
# It's a direct MP4 link
|
||||||
|
if not video_source.endswith('.mp4'):
|
||||||
|
filename += '.mp4'
|
||||||
|
|
||||||
|
print(f"[VIDMOLY] Found MP4 source")
|
||||||
|
return video_source, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error extracting VidMoly link: {str(e)}")
|
||||||
|
|
||||||
|
async def _extract_with_playwright_network(self, url: str) -> Optional[str]:
|
||||||
|
"""Extract video source using Playwright with network interception (like DownloadHelper)"""
|
||||||
|
try:
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
print("[VIDMOLY] Launching browser with network interception...")
|
||||||
|
|
||||||
|
video_urls = []
|
||||||
|
|
||||||
|
async with async_playwright() as p:
|
||||||
|
# Launch browser in headless mode
|
||||||
|
browser = await p.chromium.launch(
|
||||||
|
headless=True,
|
||||||
|
args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
|
||||||
|
)
|
||||||
|
|
||||||
|
context = await browser.new_context(
|
||||||
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||||
|
viewport={'width': 1920, 'height': 1080}
|
||||||
|
)
|
||||||
|
|
||||||
|
page = await context.new_page()
|
||||||
|
|
||||||
|
# Set up request interception BEFORE navigation
|
||||||
|
async def handle_request(route):
|
||||||
|
# Capture all requests
|
||||||
|
req_url = route.request.url
|
||||||
|
print(f"[VIDMOLY] Request: {req_url[:80]}...")
|
||||||
|
|
||||||
|
# Look for video files (m3u8, mp4, etc.)
|
||||||
|
if any(ext in req_url.lower() for ext in ['.m3u8', '.mp4', '.mkv']):
|
||||||
|
# Only capture non-vidmoly URLs (the actual video files)
|
||||||
|
if 'vidmoly' not in req_url.lower():
|
||||||
|
print(f"[VIDMOLY] 🎥 Captured video URL: {req_url[:100]}...")
|
||||||
|
video_urls.append(req_url)
|
||||||
|
|
||||||
|
# Continue with the request
|
||||||
|
await route.continue_()
|
||||||
|
|
||||||
|
# Enable request interception
|
||||||
|
await page.route('**', handle_request)
|
||||||
|
|
||||||
|
# Log page URL for debugging
|
||||||
|
print(f"[VIDMOLY] Page URL: {url}")
|
||||||
|
|
||||||
|
# Also set up response interception to catch redirects
|
||||||
|
page.on("response", lambda response: None)
|
||||||
|
|
||||||
|
print("[VIDMOLY] Navigating to page...")
|
||||||
|
|
||||||
|
# Navigate to URL and wait for load
|
||||||
|
try:
|
||||||
|
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[VIDMOLY] Navigation warning: {e}")
|
||||||
|
|
||||||
|
# Wait for page to fully load and JavaScript to execute
|
||||||
|
print("[VIDMOLY] Waiting for video player to load...")
|
||||||
|
await asyncio.sleep(5)
|
||||||
|
|
||||||
|
# Try to find and click play button if exists
|
||||||
|
try:
|
||||||
|
# Look for common play button selectors
|
||||||
|
play_selectors = [
|
||||||
|
'button.jw-icon-play',
|
||||||
|
'.jw-play-btn',
|
||||||
|
'button[aria-label="Play"]',
|
||||||
|
'.play-button',
|
||||||
|
'video',
|
||||||
|
]
|
||||||
|
|
||||||
|
for selector in play_selectors:
|
||||||
|
try:
|
||||||
|
element = await page.query_selector(selector)
|
||||||
|
if element:
|
||||||
|
print(f"[VIDMOLY] Found element: {selector}")
|
||||||
|
# For video tags, we can just wait
|
||||||
|
# For buttons, click them
|
||||||
|
if 'button' in selector or '.jw-' in selector:
|
||||||
|
await element.click()
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
break
|
||||||
|
except:
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[VIDMOLY] Play button interaction: {e}")
|
||||||
|
|
||||||
|
# Wait a bit more for network requests to complete
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
|
||||||
|
# Also try JavaScript extraction as backup
|
||||||
|
try:
|
||||||
|
js_result = await page.evaluate("""
|
||||||
|
() => {
|
||||||
|
// Check all video elements
|
||||||
|
const videos = document.querySelectorAll('video');
|
||||||
|
for (let v of videos) {
|
||||||
|
if (v.src) {
|
||||||
|
console.log('Found video src:', v.src);
|
||||||
|
return v.src;
|
||||||
|
}
|
||||||
|
const sources = v.querySelectorAll('source');
|
||||||
|
for (let s of sources) {
|
||||||
|
if (s.src && (s.src.includes('.m3u8') || s.src.includes('.mp4'))) {
|
||||||
|
console.log('Found source src:', s.src);
|
||||||
|
return s.src;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for jwplayer
|
||||||
|
if (window.jwplayer) {
|
||||||
|
try {
|
||||||
|
const player = jwplayer();
|
||||||
|
const playlist = player.getPlaylist();
|
||||||
|
if (playlist && playlist[0] && playlist[0].sources) {
|
||||||
|
const src = playlist[0].sources[0].file;
|
||||||
|
console.log('Found jwplayer source:', src);
|
||||||
|
return src;
|
||||||
|
}
|
||||||
|
} catch(e) {
|
||||||
|
console.log('jwplayer error:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for other player configurations
|
||||||
|
if (window.player && window.player.config) {
|
||||||
|
if (window.player.config.sources && window.player.config.sources[0]) {
|
||||||
|
return window.player.config.sources[0].file;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look in window object for video URLs
|
||||||
|
for (let key in window) {
|
||||||
|
if (typeof window[key] === 'string') {
|
||||||
|
const str = window[key];
|
||||||
|
if ((str.includes('.m3u8') || str.includes('.mp4')) && str.startsWith('http')) {
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
""")
|
||||||
|
|
||||||
|
if js_result and ('.m3u8' in js_result or '.mp4' in js_result):
|
||||||
|
print(f"[VIDMOLY] Found video URL via JavaScript")
|
||||||
|
video_urls.append(js_result)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[VIDMOLY] JS extraction error: {e}")
|
||||||
|
|
||||||
|
# Final check: parse page HTML for video URLs
|
||||||
|
try:
|
||||||
|
content = await page.content()
|
||||||
|
patterns = [
|
||||||
|
r'"file"\s*:\s*"([^"]+\.m3u8[^"]*)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+\.mp4[^"]*)"',
|
||||||
|
r"'file'\s*:\s*'([^']+\.m3u8[^']*)'",
|
||||||
|
r"'file'\s*:\s*'([^']+\.mp4[^']*)'",
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, content)
|
||||||
|
for match in matches:
|
||||||
|
# Clean up the URL
|
||||||
|
match = match.replace('\\', '').replace('\/', '/')
|
||||||
|
if 'http' in match and 'vidmoly' not in match:
|
||||||
|
print(f"[VIDMOLY] Found in HTML: {match[:100]}...")
|
||||||
|
video_urls.append(match)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[VIDMOLY] HTML parsing error: {e}")
|
||||||
|
|
||||||
|
await browser.close()
|
||||||
|
|
||||||
|
# Return the first valid video URL found
|
||||||
|
if video_urls:
|
||||||
|
# Deduplicate while preserving order
|
||||||
|
seen = set()
|
||||||
|
unique_urls = []
|
||||||
|
for url in video_urls:
|
||||||
|
if url not in seen:
|
||||||
|
seen.add(url)
|
||||||
|
unique_urls.append(url)
|
||||||
|
|
||||||
|
if unique_urls:
|
||||||
|
print(f"[VIDMOLY] ✅ Found {len(unique_urls)} video URL(s)")
|
||||||
|
return unique_urls[0]
|
||||||
|
|
||||||
|
print("[VIDMOLY] ❌ No video URLs found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
print("[VIDMOLY] Playwright not installed")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[VIDMOLY] Playwright error: {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _extract_with_http(self, url: str) -> Optional[str]:
|
||||||
|
"""Fallback: Extract video source using pure HTTP requests"""
|
||||||
|
try:
|
||||||
|
headers = {
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||||
|
'Referer': 'https://vidmoly.biz/',
|
||||||
|
'Accept': '*/*',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
|
}
|
||||||
|
|
||||||
|
response = await self.client.get(url, headers=headers)
|
||||||
|
|
||||||
|
# Follow JS redirect if present
|
||||||
|
if 'window.location.replace' in response.text:
|
||||||
|
redirect_match = re.search(r"window\.location\.replace\('([^']+)'", response.text)
|
||||||
|
if redirect_match:
|
||||||
|
redirect_url = redirect_match.group(1)
|
||||||
|
response = await self.client.get(redirect_url, headers=headers, follow_redirects=True)
|
||||||
|
|
||||||
|
# Try to find video source
|
||||||
|
patterns = [
|
||||||
|
r'file:"([^"]+)"',
|
||||||
|
r'"file"\s*:\s*"([^"]+)"',
|
||||||
|
r"'file'\s*:\s*'([^']+)'",
|
||||||
|
r'(https?://[^\s"\'<>]+\.m3u8[^\s"\'<>]*)',
|
||||||
|
r'(https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*)',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns:
|
||||||
|
matches = re.findall(pattern, response.text)
|
||||||
|
if matches:
|
||||||
|
for match in matches:
|
||||||
|
match = match.replace('\\', '').replace('\/', '/')
|
||||||
|
if 'http' in match and 'vidmoly' not in match:
|
||||||
|
return match
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[VIDMOLY] HTTP extraction error: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _get_m3u8_qualities(self, master_m3u8_url: str, headers: dict) -> list[dict]:
|
||||||
|
"""Fetch master M3U8 and extract available qualities"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(master_m3u8_url, headers=headers)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
content = response.text
|
||||||
|
lines = [line.strip() for line in content.split('\n') if line.strip()]
|
||||||
|
|
||||||
|
qualities = []
|
||||||
|
current_quality = {}
|
||||||
|
|
||||||
|
for line in lines:
|
||||||
|
if line.startswith('#EXT-X-STREAM-INF'):
|
||||||
|
resolution_match = re.search(r'RESOLUTION=\d+x(\d+)', line)
|
||||||
|
if resolution_match:
|
||||||
|
current_quality['label'] = resolution_match.group(1)
|
||||||
|
elif line.endswith('.m3u8') and current_quality:
|
||||||
|
current_quality['url'] = line if line.startswith('http') else master_m3u8_url.rsplit('/', 1)[0] + '/' + line
|
||||||
|
qualities.append(current_quality)
|
||||||
|
current_quality = {}
|
||||||
|
|
||||||
|
qualities.sort(key=lambda x: int(x['label']), reverse=True)
|
||||||
|
return qualities
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error fetching M3U8 qualities: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def _download_m3u8_as_mp4(self, m3u8_url: str, filename: str, headers: dict, download_dir: str = "downloads") -> str:
|
||||||
|
"""Download M3U8 stream and convert to MP4 using ffmpeg"""
|
||||||
|
# Create downloads directory if it doesn't exist
|
||||||
|
os.makedirs(download_dir, exist_ok=True)
|
||||||
|
|
||||||
|
output_path = os.path.join(download_dir, filename)
|
||||||
|
|
||||||
|
# Build headers for ffmpeg - using multiple -headers options
|
||||||
|
header_args = []
|
||||||
|
for key, value in headers.items():
|
||||||
|
header_args.extend(['-headers', f'{key}: {value}'])
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
'ffmpeg',
|
||||||
|
*header_args,
|
||||||
|
'-i', m3u8_url,
|
||||||
|
'-c', 'copy',
|
||||||
|
'-bsf:a', 'aac_adtstoasc',
|
||||||
|
'-y',
|
||||||
|
output_path
|
||||||
|
]
|
||||||
|
|
||||||
|
try:
|
||||||
|
print(f"[VIDMOLY] Downloading M3U8 with ffmpeg...")
|
||||||
|
print(f"[VIDMOLY] URL: {m3u8_url[:80]}...")
|
||||||
|
print(f"[VIDMOLY] Output: {output_path}")
|
||||||
|
|
||||||
|
# Run ffmpeg without capturing output to avoid buffering issues
|
||||||
|
# Use a log file instead
|
||||||
|
log_path = output_path + '.log'
|
||||||
|
with open(log_path, 'w') as log_file:
|
||||||
|
result = subprocess.run(
|
||||||
|
cmd,
|
||||||
|
stdout=log_file,
|
||||||
|
stderr=log_file,
|
||||||
|
timeout=600 # 10 minutes for very long videos
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check if file was created even if ffmpeg had issues
|
||||||
|
if os.path.exists(output_path):
|
||||||
|
file_size = os.path.getsize(output_path)
|
||||||
|
if file_size > 1000: # At least 1KB
|
||||||
|
print(f"[VIDMOLY] ✅ Download complete: {file_size / (1024*1024):.2f} MB")
|
||||||
|
return output_path
|
||||||
|
|
||||||
|
# If we get here, something went wrong
|
||||||
|
raise Exception(f"FFmpeg failed - no output file created")
|
||||||
|
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
# Check if file was created despite timeout
|
||||||
|
if os.path.exists(output_path):
|
||||||
|
file_size = os.path.getsize(output_path)
|
||||||
|
if file_size > 1000: # At least 1KB
|
||||||
|
print(f"[VIDMOLY] ⚠️ Timeout but file created: {file_size / (1024*1024):.2f} MB")
|
||||||
|
return output_path
|
||||||
|
raise Exception("FFmpeg timeout (10 minutes) - video too large")
|
||||||
|
|
||||||
|
except FileNotFoundError:
|
||||||
|
raise Exception("ffmpeg not found - please install ffmpeg: apt install ffmpeg")
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error downloading M3U8: {str(e)}")
|
||||||
|
|
||||||
|
def _extract_vidmoly_id(self, url: str) -> Optional[str]:
|
||||||
|
"""Extract VidMoly video ID from URL"""
|
||||||
|
embed_match = re.search(r'embed-([a-z0-9]+)', url, re.IGNORECASE)
|
||||||
|
if embed_match:
|
||||||
|
return embed_match.group(1)
|
||||||
|
|
||||||
|
param_match = re.search(r'[?&]v=([a-z0-9]+)', url, re.IGNORECASE)
|
||||||
|
if param_match:
|
||||||
|
return param_match.group(1)
|
||||||
|
|
||||||
|
path_match = re.search(r'vidmoly\.(?:to|org|biz)/([a-z0-9]+)', url, re.IGNORECASE)
|
||||||
|
if path_match:
|
||||||
|
return path_match.group(1)
|
||||||
|
|
||||||
|
return None
|
||||||
@@ -0,0 +1,344 @@
|
|||||||
|
"""Vidzy video hosting service downloader"""
|
||||||
|
import logging
|
||||||
|
import asyncio
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import os
|
||||||
|
from typing import Optional
|
||||||
|
from .base import BaseVideoPlayer
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from app.utils import sanitize_filename
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class VidzyDownloader(BaseVideoPlayer):
|
||||||
|
"""
|
||||||
|
Downloader for Vidzy video hosting service.
|
||||||
|
|
||||||
|
Vidzy is a video hosting platform used by various anime streaming sites.
|
||||||
|
Uses heavy JavaScript obfuscation, so Playwright is required.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def can_handle(self, url: str) -> bool:
|
||||||
|
"""Check if this downloader can handle the given URL"""
|
||||||
|
return "vidzy" in url.lower()
|
||||||
|
|
||||||
|
async def get_download_link(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
target_filename: Optional[str] = None
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Extract direct download link and filename from Vidzy URL.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: The Vidzy video player URL
|
||||||
|
target_filename: Optional filename override
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (download_url, filename)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Extract actual Vidzy URL from pipe-separated format if present
|
||||||
|
# Format: video_url|anime_url|episode_title
|
||||||
|
if '|' in url:
|
||||||
|
url = url.split('|')[0].strip()
|
||||||
|
logger.debug(f"Extracted Vidzy URL from pipe format: {url}")
|
||||||
|
|
||||||
|
logger.info(f"Fetching Vidzy URL: {url}")
|
||||||
|
|
||||||
|
# Try using Playwright first (Vidzy uses heavy JS obfuscation)
|
||||||
|
video_url = await self._extract_with_playwright(url)
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
# Fallback to static HTML parsing
|
||||||
|
logger.warning("Playwright extraction failed, trying static parsing...")
|
||||||
|
video_url = await self._extract_static(url)
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
raise ValueError(f"Could not extract video URL from Vidzy")
|
||||||
|
|
||||||
|
logger.info(f"Successfully extracted Vidzy URL: {video_url[:100]}...")
|
||||||
|
|
||||||
|
# Generate filename
|
||||||
|
if target_filename:
|
||||||
|
filename = sanitize_filename(target_filename)
|
||||||
|
else:
|
||||||
|
# Try to extract filename from URL
|
||||||
|
filename = video_url.split('/')[-1].split('?')[0]
|
||||||
|
if not filename or len(filename) < 5:
|
||||||
|
filename = "vidzy_video.mp4"
|
||||||
|
filename = sanitize_filename(filename)
|
||||||
|
|
||||||
|
# Ensure .mp4 extension
|
||||||
|
if not filename.endswith('.mp4'):
|
||||||
|
filename += '.mp4'
|
||||||
|
|
||||||
|
# Check if it's an M3U8 playlist (HLS stream)
|
||||||
|
if '.m3u8' in video_url:
|
||||||
|
logger.info(f"Detected M3U8 stream, will download with ffmpeg")
|
||||||
|
|
||||||
|
# Download and convert M3U8 to MP4 directly
|
||||||
|
headers = {
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||||
|
'Referer': 'https://vidzy.org/',
|
||||||
|
}
|
||||||
|
|
||||||
|
mp4_path = await self._download_m3u8_as_mp4(video_url, filename, headers)
|
||||||
|
logger.info(f"Successfully extracted Vidzy download link: {filename}")
|
||||||
|
return mp4_path, filename
|
||||||
|
|
||||||
|
# It's a direct MP4 link
|
||||||
|
logger.info(f"Successfully extracted Vidzy download link: {filename}")
|
||||||
|
return video_url, filename
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error extracting Vidzy download link: {e}")
|
||||||
|
raise ValueError(f"Failed to extract download link from Vidzy: {str(e)}")
|
||||||
|
|
||||||
|
async def _extract_with_playwright(self, url: str) -> Optional[str]:
|
||||||
|
"""Extract video URL using Playwright with network interception"""
|
||||||
|
try:
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
logger.info("Launching Playwright for Vidzy...")
|
||||||
|
|
||||||
|
video_urls = []
|
||||||
|
|
||||||
|
async with async_playwright() as p:
|
||||||
|
browser = await p.chromium.launch(
|
||||||
|
headless=True,
|
||||||
|
args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
|
||||||
|
)
|
||||||
|
|
||||||
|
context = await browser.new_context(
|
||||||
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
|
||||||
|
)
|
||||||
|
|
||||||
|
page = await context.new_page()
|
||||||
|
|
||||||
|
# Set up request interception
|
||||||
|
async def handle_request(route):
|
||||||
|
req_url = route.request.url
|
||||||
|
|
||||||
|
# Look for video files (HLS streams and MP4s)
|
||||||
|
if any(ext in req_url.lower() for ext in ['.m3u8', '.mp4', 'master']):
|
||||||
|
if 'vidzy' not in req_url.lower() or 'master' in req_url.lower():
|
||||||
|
logger.info(f"🎥 Captured video URL: {req_url[:100]}...")
|
||||||
|
video_urls.append(req_url)
|
||||||
|
|
||||||
|
await route.continue_()
|
||||||
|
|
||||||
|
await page.route('**', handle_request)
|
||||||
|
|
||||||
|
logger.info("Navigating to Vidzy page...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Navigation warning: {e}")
|
||||||
|
|
||||||
|
# Wait for page to load and initialize player
|
||||||
|
logger.info("Waiting for video player to load...")
|
||||||
|
await asyncio.sleep(5)
|
||||||
|
|
||||||
|
# Try JavaScript extraction from VideoJS player
|
||||||
|
try:
|
||||||
|
js_result = await page.evaluate("""
|
||||||
|
() => {
|
||||||
|
// Check if videojs is available
|
||||||
|
if (typeof videojs !== 'undefined' && videojs.players) {
|
||||||
|
// Get all players
|
||||||
|
const players = Object.values(videojs.players);
|
||||||
|
if (players.length > 0) {
|
||||||
|
const player = players[0];
|
||||||
|
|
||||||
|
// Try to get source from player
|
||||||
|
if (player.currentSrc()) {
|
||||||
|
return player.currentSrc();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to get sources array
|
||||||
|
if (player.currentSources() && player.currentSources().length > 0) {
|
||||||
|
return player.currentSources()[0].src;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check all video elements
|
||||||
|
const videos = document.querySelectorAll('video');
|
||||||
|
for (let v of videos) {
|
||||||
|
if (v.src) {
|
||||||
|
return v.src;
|
||||||
|
}
|
||||||
|
const sources = v.querySelectorAll('source');
|
||||||
|
for (let s of sources) {
|
||||||
|
if (s.src) {
|
||||||
|
return s.src;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for sources in scripts (VideoJS config)
|
||||||
|
const scripts = document.querySelectorAll('script');
|
||||||
|
for (let script of scripts) {
|
||||||
|
const text = script.textContent;
|
||||||
|
// Look for sources array with .m3u8 URLs
|
||||||
|
const sourcesMatch = text.match(/sources\s*:\s*\[\s*\{\s*src\s*:\s*['"](https?:\/\/[^'"]+\.m3u8[^'"]*)['"]/i);
|
||||||
|
if (sourcesMatch) {
|
||||||
|
return sourcesMatch[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
""")
|
||||||
|
|
||||||
|
if js_result and ('.m3u8' in js_result or '.mp4' in js_result):
|
||||||
|
logger.info(f"Found video URL via JavaScript evaluation")
|
||||||
|
video_urls.append(js_result)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"JS extraction error: {e}")
|
||||||
|
|
||||||
|
# Wait more for network requests
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
|
||||||
|
await browser.close()
|
||||||
|
|
||||||
|
# Return best video URL (prefer master.m3u8 for HLS)
|
||||||
|
if video_urls:
|
||||||
|
seen = set()
|
||||||
|
unique_urls = []
|
||||||
|
for url in video_urls:
|
||||||
|
if url not in seen:
|
||||||
|
seen.add(url)
|
||||||
|
unique_urls.append(url)
|
||||||
|
|
||||||
|
if unique_urls:
|
||||||
|
logger.info(f"✅ Found {len(unique_urls)} video URL(s)")
|
||||||
|
|
||||||
|
# Prefer master.m3u8 (HLS playlist)
|
||||||
|
for url in unique_urls:
|
||||||
|
if 'master.m3u8' in url or '.m3u8' in url:
|
||||||
|
logger.info(f"Using HLS playlist: {url[:100]}...")
|
||||||
|
return url
|
||||||
|
|
||||||
|
# Fall back to first URL
|
||||||
|
return unique_urls[0]
|
||||||
|
|
||||||
|
logger.warning("❌ No video URLs found via Playwright")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
logger.warning("Playwright not installed, falling back to static parsing")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Playwright error: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _extract_static(self, url: str) -> Optional[str]:
|
||||||
|
"""Static HTML parsing fallback"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
html = response.text
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, 'lxml')
|
||||||
|
|
||||||
|
# Method 1: Look for video source in <video> tag
|
||||||
|
video_tag = soup.find('video')
|
||||||
|
if video_tag and video_tag.get('src'):
|
||||||
|
logger.info(f"Found video source from <video> tag")
|
||||||
|
return video_tag['src']
|
||||||
|
|
||||||
|
# Method 2: Look for source in <source> tag
|
||||||
|
source_tag = soup.find('source')
|
||||||
|
if source_tag and source_tag.get('src'):
|
||||||
|
logger.info(f"Found video source from <source> tag")
|
||||||
|
return source_tag['src']
|
||||||
|
|
||||||
|
# Method 3: Search entire HTML for .m3u8 URLs (Vidzy uses HLS)
|
||||||
|
html_patterns = [
|
||||||
|
r'(https?://[^\s<>"\'`]+\.m3u8[^\s<>"\'`]*)',
|
||||||
|
r'(https?://[^\s<>"\'`]+/master[^\s<>"\'`]*)',
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in html_patterns:
|
||||||
|
matches = re.findall(pattern, html)
|
||||||
|
if matches:
|
||||||
|
# Filter out obvious false positives
|
||||||
|
for match in matches:
|
||||||
|
# Accept URLs with 'master' or from video hosts
|
||||||
|
if 'master' in match.lower() or any(host in match for host in ['hls', 'video', 'stream']):
|
||||||
|
logger.info(f"Found video URL in HTML: {match[:100]}...")
|
||||||
|
return match
|
||||||
|
|
||||||
|
logger.warning("Static parsing failed to find video URL")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Static parsing error: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _download_m3u8_as_mp4(self, m3u8_url: str, filename: str, headers: dict, download_dir: str = "downloads") -> str:
|
||||||
|
"""Download M3U8 stream and convert to MP4 using ffmpeg"""
|
||||||
|
# Create downloads directory if it doesn't exist
|
||||||
|
os.makedirs(download_dir, exist_ok=True)
|
||||||
|
|
||||||
|
output_path = os.path.join(download_dir, filename)
|
||||||
|
|
||||||
|
# Build headers for ffmpeg - using multiple -headers options
|
||||||
|
header_args = []
|
||||||
|
for key, value in headers.items():
|
||||||
|
header_args.extend(['-headers', f'{key}: {value}'])
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
'ffmpeg',
|
||||||
|
*header_args,
|
||||||
|
'-i', m3u8_url,
|
||||||
|
'-c', 'copy',
|
||||||
|
'-bsf:a', 'aac_adtstoasc',
|
||||||
|
'-y',
|
||||||
|
output_path
|
||||||
|
]
|
||||||
|
|
||||||
|
try:
|
||||||
|
logger.info(f"Downloading M3U8 with ffmpeg...")
|
||||||
|
logger.info(f"URL: {m3u8_url[:80]}...")
|
||||||
|
logger.info(f"Output: {output_path}")
|
||||||
|
|
||||||
|
# Run ffmpeg without capturing output to avoid buffering issues
|
||||||
|
# Use a log file instead
|
||||||
|
log_path = output_path + '.log'
|
||||||
|
with open(log_path, 'w') as log_file:
|
||||||
|
result = subprocess.run(
|
||||||
|
cmd,
|
||||||
|
stdout=log_file,
|
||||||
|
stderr=log_file,
|
||||||
|
timeout=600 # 10 minutes for very long videos
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check if file was created even if ffmpeg had issues
|
||||||
|
if os.path.exists(output_path):
|
||||||
|
file_size = os.path.getsize(output_path)
|
||||||
|
if file_size > 1000: # At least 1KB
|
||||||
|
logger.info(f"✅ Download complete: {file_size / (1024*1024):.2f} MB")
|
||||||
|
return output_path
|
||||||
|
|
||||||
|
# If we get here, something went wrong
|
||||||
|
raise Exception(f"FFmpeg failed - no output file created")
|
||||||
|
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
# Check if file was created despite timeout
|
||||||
|
if os.path.exists(output_path):
|
||||||
|
file_size = os.path.getsize(output_path)
|
||||||
|
if file_size > 1000: # At least 1KB
|
||||||
|
logger.warning(f"⚠️ Timeout but file created: {file_size / (1024*1024):.2f} MB")
|
||||||
|
return output_path
|
||||||
|
raise Exception("FFmpeg timeout (10 minutes) - video too large")
|
||||||
|
|
||||||
|
except FileNotFoundError:
|
||||||
|
raise Exception("ffmpeg not found - please install ffmpeg: apt install ffmpeg")
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Error downloading M3U8: {str(e)}")
|
||||||
@@ -0,0 +1,245 @@
|
|||||||
|
"""Episode checker for detecting and downloading new episodes automatically"""
|
||||||
|
import logging
|
||||||
|
from typing import List, Optional, Dict
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from app.watchlist import watchlist_manager, WatchlistManager
|
||||||
|
from app.models import DownloadRequest, DownloadTask, DownloadStatus
|
||||||
|
from app.models.watchlist import (
|
||||||
|
WatchlistItem,
|
||||||
|
WatchlistSettings,
|
||||||
|
NewEpisodeInfo,
|
||||||
|
AutoDownloadResult
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class EpisodeChecker:
|
||||||
|
"""Checks for new episodes and downloads them automatically"""
|
||||||
|
|
||||||
|
def __init__(self, wlm: Optional[WatchlistManager] = None):
|
||||||
|
self.wlm = wlm or watchlist_manager
|
||||||
|
self.download_manager = None # Will be set by main.py
|
||||||
|
|
||||||
|
def set_download_manager(self, download_manager):
|
||||||
|
"""Set the download manager (called by main.py to avoid circular import)"""
|
||||||
|
self.download_manager = download_manager
|
||||||
|
|
||||||
|
async def check_anime(self, item: WatchlistItem) -> List[NewEpisodeInfo]:
|
||||||
|
"""
|
||||||
|
Check for new episodes of a specific anime
|
||||||
|
|
||||||
|
Args:
|
||||||
|
item: WatchlistItem to check
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of NewEpisodeInfo objects
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
logger.info(f"Checking for new episodes: {item.anime_title}")
|
||||||
|
|
||||||
|
# Import here to avoid circular imports
|
||||||
|
from app.downloaders import get_downloader
|
||||||
|
|
||||||
|
# Get the appropriate downloader
|
||||||
|
downloader = get_downloader(item.anime_url)
|
||||||
|
if not downloader:
|
||||||
|
logger.error(f"No downloader found for URL: {item.anime_url}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Get episodes list
|
||||||
|
episodes = await downloader.get_episodes(item.anime_url, item.lang)
|
||||||
|
if not episodes:
|
||||||
|
logger.warning(f"No episodes found for {item.anime_title}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Filter new episodes
|
||||||
|
new_episodes = []
|
||||||
|
for ep in episodes:
|
||||||
|
ep_num = ep.get('episode_number', 0)
|
||||||
|
if ep_num > item.last_episode_downloaded:
|
||||||
|
new_episodes.append(NewEpisodeInfo(
|
||||||
|
episode_number=ep_num,
|
||||||
|
episode_title=ep.get('title'),
|
||||||
|
episode_url=ep['url'],
|
||||||
|
season_number=ep.get('season'),
|
||||||
|
anime_title=item.anime_title,
|
||||||
|
provider_id=item.provider_id
|
||||||
|
))
|
||||||
|
|
||||||
|
if new_episodes:
|
||||||
|
logger.info(f"Found {len(new_episodes)} new episodes for {item.anime_title}")
|
||||||
|
else:
|
||||||
|
logger.info(f"No new episodes for {item.anime_title}")
|
||||||
|
|
||||||
|
return new_episodes
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error checking anime {item.anime_title}: {e}", exc_info=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def download_new_episodes(
|
||||||
|
self,
|
||||||
|
item: WatchlistItem,
|
||||||
|
episodes: List[NewEpisodeInfo]
|
||||||
|
) -> AutoDownloadResult:
|
||||||
|
"""
|
||||||
|
Download new episodes for a watchlist item
|
||||||
|
|
||||||
|
Args:
|
||||||
|
item: WatchlistItem
|
||||||
|
episodes: List of new episodes to download
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
AutoDownloadResult with download status
|
||||||
|
"""
|
||||||
|
result = AutoDownloadResult(
|
||||||
|
watchlist_item_id=item.id,
|
||||||
|
anime_title=item.anime_title,
|
||||||
|
new_episodes_found=len(episodes),
|
||||||
|
checked_at=datetime.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
if not episodes:
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Get settings
|
||||||
|
settings = self.wlm.get_settings()
|
||||||
|
if not settings.auto_download_enabled:
|
||||||
|
logger.info(f"Auto-download disabled, skipping {len(episodes)} episodes")
|
||||||
|
return result
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Import here to avoid circular imports
|
||||||
|
from app.downloaders import get_downloader
|
||||||
|
|
||||||
|
downloader = get_downloader(item.anime_url)
|
||||||
|
|
||||||
|
# Download each new episode
|
||||||
|
for ep_info in episodes:
|
||||||
|
try:
|
||||||
|
logger.info(f"Downloading {item.anime_title} Episode {ep_info.episode_number}")
|
||||||
|
|
||||||
|
# Get download link
|
||||||
|
download_link, filename = await downloader.get_download_link(ep_info.episode_url)
|
||||||
|
|
||||||
|
# Create download task
|
||||||
|
request = DownloadRequest(url=download_link, filename=filename)
|
||||||
|
task = self.download_manager.create_task(request)
|
||||||
|
|
||||||
|
if task:
|
||||||
|
await self.download_manager.start_download(task.id)
|
||||||
|
result.episodes_downloaded.append(ep_info.episode_number)
|
||||||
|
logger.info(f"Started download: {filename}")
|
||||||
|
else:
|
||||||
|
result.episodes_failed.append((ep_info.episode_number, "Failed to create download task"))
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
error_msg = str(e)
|
||||||
|
logger.error(f"Error downloading episode {ep_info.episode_number}: {error_msg}")
|
||||||
|
result.episodes_failed.append((ep_info.episode_number, error_msg))
|
||||||
|
|
||||||
|
# Update watchlist with last episode downloaded
|
||||||
|
if result.episodes_downloaded:
|
||||||
|
last_ep = max(result.episodes_downloaded)
|
||||||
|
self.wlm.update_check_time(item.id, last_ep)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error in download_new_episodes: {e}", exc_info=True)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
async def check_and_download(self, item: WatchlistItem) -> AutoDownloadResult:
|
||||||
|
"""
|
||||||
|
Check for new episodes and download them if auto_download is enabled
|
||||||
|
|
||||||
|
Args:
|
||||||
|
item: WatchlistItem to check
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
AutoDownloadResult
|
||||||
|
"""
|
||||||
|
# Check for new episodes
|
||||||
|
new_episodes = await self.check_anime(item)
|
||||||
|
|
||||||
|
result = AutoDownloadResult(
|
||||||
|
watchlist_item_id=item.id,
|
||||||
|
anime_title=item.anime_title,
|
||||||
|
new_episodes_found=len(new_episodes),
|
||||||
|
checked_at=datetime.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Download if auto_download is enabled
|
||||||
|
if item.auto_download and new_episodes:
|
||||||
|
settings = self.wlm.get_settings()
|
||||||
|
if settings.auto_download_enabled:
|
||||||
|
download_result = await self.download_new_episodes(item, new_episodes)
|
||||||
|
result = download_result
|
||||||
|
else:
|
||||||
|
logger.info(f"Auto-download globally disabled, skipping {len(new_episodes)} episodes")
|
||||||
|
|
||||||
|
# Update check time even if no downloads
|
||||||
|
self.wlm.update_check_time(item.id, item.last_episode_downloaded)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
async def check_all_due(self) -> List[AutoDownloadResult]:
|
||||||
|
"""
|
||||||
|
Check all watchlist items that are due for checking
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of AutoDownloadResult objects
|
||||||
|
"""
|
||||||
|
settings = self.wlm.get_settings()
|
||||||
|
due_items = self.wlm.get_due_for_check(settings.check_interval_hours)
|
||||||
|
|
||||||
|
logger.info(f"Checking {len(due_items)} due watchlist items")
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for item in due_items:
|
||||||
|
try:
|
||||||
|
result = await self.check_and_download(item)
|
||||||
|
results.append(result)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error processing {item.anime_title}: {e}", exc_info=True)
|
||||||
|
# Still add a result to track the failure
|
||||||
|
results.append(AutoDownloadResult(
|
||||||
|
watchlist_item_id=item.id,
|
||||||
|
anime_title=item.anime_title,
|
||||||
|
new_episodes_found=0,
|
||||||
|
checked_at=datetime.now()
|
||||||
|
))
|
||||||
|
|
||||||
|
# Log summary
|
||||||
|
total_new = sum(r.new_episodes_found for r in results)
|
||||||
|
total_downloaded = sum(len(r.episodes_downloaded) for r in results)
|
||||||
|
total_failed = sum(len(r.episodes_failed) for r in results)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"Check complete: {total_new} new episodes found, "
|
||||||
|
f"{total_downloaded} downloaded, {total_failed} failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
async def manual_check(self, item_id: str) -> Optional[AutoDownloadResult]:
|
||||||
|
"""
|
||||||
|
Manually trigger a check for a specific watchlist item
|
||||||
|
|
||||||
|
Args:
|
||||||
|
item_id: Watchlist item ID
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
AutoDownloadResult or None if item not found
|
||||||
|
"""
|
||||||
|
item = self.wlm.get_by_id(item_id)
|
||||||
|
if not item:
|
||||||
|
logger.error(f"Watchlist item not found: {item_id}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
return await self.check_and_download(item)
|
||||||
|
|
||||||
|
|
||||||
|
# Global episode checker instance
|
||||||
|
episode_checker = EpisodeChecker()
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
"""
|
||||||
|
Favorites management system for Ohm Stream Downloader
|
||||||
|
Stores user's favorite anime with metadata in a local JSON file
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Dict, Optional
|
||||||
|
from datetime import datetime
|
||||||
|
import aiofiles
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class FavoritesManager:
|
||||||
|
"""Manages user's favorite anime list"""
|
||||||
|
|
||||||
|
def __init__(self, storage_path: str = "data/favorites.json"):
|
||||||
|
self.storage_path = Path(storage_path)
|
||||||
|
self.storage_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._favorites: Dict[str, Dict] = {}
|
||||||
|
self._lock = asyncio.Lock()
|
||||||
|
|
||||||
|
async def _load(self):
|
||||||
|
"""Load favorites from disk"""
|
||||||
|
async with self._lock:
|
||||||
|
await self._load_for_operation()
|
||||||
|
|
||||||
|
async def _load_for_operation(self):
|
||||||
|
"""Load favorites from disk without acquiring lock (lock must already be held)"""
|
||||||
|
if self.storage_path.exists():
|
||||||
|
try:
|
||||||
|
async with aiofiles.open(self.storage_path, 'r', encoding='utf-8') as f:
|
||||||
|
content = await f.read()
|
||||||
|
self._favorites = json.loads(content) if content.strip() else {}
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error loading favorites: {e}")
|
||||||
|
self._favorites = {}
|
||||||
|
else:
|
||||||
|
self._favorites = {}
|
||||||
|
|
||||||
|
async def _save(self):
|
||||||
|
"""Save favorites to disk (assumes lock is already held)"""
|
||||||
|
try:
|
||||||
|
async with aiofiles.open(self.storage_path, 'w', encoding='utf-8') as f:
|
||||||
|
await f.write(json.dumps(self._favorites, indent=2, ensure_ascii=False))
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error saving favorites: {e}")
|
||||||
|
|
||||||
|
async def add_favorite(
|
||||||
|
self,
|
||||||
|
anime_id: str,
|
||||||
|
title: str,
|
||||||
|
url: str,
|
||||||
|
provider: str,
|
||||||
|
metadata: Optional[Dict] = None,
|
||||||
|
poster_url: Optional[str] = None
|
||||||
|
) -> Dict:
|
||||||
|
"""Add an anime to favorites"""
|
||||||
|
async with self._lock:
|
||||||
|
await self._load_for_operation()
|
||||||
|
|
||||||
|
if anime_id in self._favorites:
|
||||||
|
# Update existing favorite
|
||||||
|
self._favorites[anime_id]["updated_at"] = datetime.now().isoformat()
|
||||||
|
if metadata:
|
||||||
|
self._favorites[anime_id]["metadata"] = metadata
|
||||||
|
if poster_url:
|
||||||
|
self._favorites[anime_id]["poster_url"] = poster_url
|
||||||
|
else:
|
||||||
|
# Add new favorite
|
||||||
|
self._favorites[anime_id] = {
|
||||||
|
"id": anime_id,
|
||||||
|
"title": title,
|
||||||
|
"url": url,
|
||||||
|
"provider": provider,
|
||||||
|
"metadata": metadata or {},
|
||||||
|
"poster_url": poster_url,
|
||||||
|
"created_at": datetime.now().isoformat(),
|
||||||
|
"updated_at": datetime.now().isoformat()
|
||||||
|
}
|
||||||
|
|
||||||
|
await self._save()
|
||||||
|
return self._favorites[anime_id]
|
||||||
|
|
||||||
|
async def remove_favorite(self, anime_id: str) -> bool:
|
||||||
|
"""Remove an anime from favorites"""
|
||||||
|
async with self._lock:
|
||||||
|
await self._load_for_operation()
|
||||||
|
|
||||||
|
if anime_id in self._favorites:
|
||||||
|
del self._favorites[anime_id]
|
||||||
|
await self._save()
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def get_favorite(self, anime_id: str) -> Optional[Dict]:
|
||||||
|
"""Get a specific favorite by ID"""
|
||||||
|
await self._load()
|
||||||
|
return self._favorites.get(anime_id)
|
||||||
|
|
||||||
|
async def list_favorites(
|
||||||
|
self,
|
||||||
|
sort_by: str = "created_at",
|
||||||
|
order: str = "desc",
|
||||||
|
filter_provider: Optional[str] = None,
|
||||||
|
filter_genre: Optional[str] = None
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""List all favorites with optional sorting and filtering"""
|
||||||
|
await self._load()
|
||||||
|
|
||||||
|
favorites = list(self._favorites.values())
|
||||||
|
|
||||||
|
# Apply filters
|
||||||
|
if filter_provider:
|
||||||
|
favorites = [f for f in favorites if f["provider"] == filter_provider]
|
||||||
|
|
||||||
|
if filter_genre:
|
||||||
|
favorites = [
|
||||||
|
f for f in favorites
|
||||||
|
if filter_genre in f.get("metadata", {}).get("genres", [])
|
||||||
|
]
|
||||||
|
|
||||||
|
# Sort favorites
|
||||||
|
reverse = order == "desc"
|
||||||
|
if sort_by == "title":
|
||||||
|
favorites.sort(key=lambda x: x["title"].lower(), reverse=reverse)
|
||||||
|
elif sort_by == "rating":
|
||||||
|
favorites.sort(
|
||||||
|
key=lambda x: float(x.get("metadata", {}).get("rating", "0").split("/")[0]),
|
||||||
|
reverse=reverse
|
||||||
|
)
|
||||||
|
elif sort_by == "year":
|
||||||
|
favorites.sort(
|
||||||
|
key=lambda x: x.get("metadata", {}).get("release_year", 0),
|
||||||
|
reverse=reverse
|
||||||
|
)
|
||||||
|
else: # created_at, updated_at
|
||||||
|
favorites.sort(key=lambda x: x.get(sort_by, ""), reverse=reverse)
|
||||||
|
|
||||||
|
return favorites
|
||||||
|
|
||||||
|
async def is_favorite(self, anime_id: str) -> bool:
|
||||||
|
"""Check if an anime is in favorites"""
|
||||||
|
await self._load()
|
||||||
|
return anime_id in self._favorites
|
||||||
|
|
||||||
|
async def toggle_favorite(
|
||||||
|
self,
|
||||||
|
anime_id: str,
|
||||||
|
title: str,
|
||||||
|
url: str,
|
||||||
|
provider: str,
|
||||||
|
metadata: Optional[Dict] = None,
|
||||||
|
poster_url: Optional[str] = None
|
||||||
|
) -> Dict:
|
||||||
|
"""Toggle an anime in favorites (add if not exists, remove if exists)"""
|
||||||
|
is_fav = await self.is_favorite(anime_id)
|
||||||
|
|
||||||
|
if is_fav:
|
||||||
|
await self.remove_favorite(anime_id)
|
||||||
|
return {"action": "removed", "anime_id": anime_id}
|
||||||
|
else:
|
||||||
|
fav = await self.add_favorite(anime_id, title, url, provider, metadata, poster_url)
|
||||||
|
return {"action": "added", "anime_id": anime_id, "favorite": fav}
|
||||||
|
|
||||||
|
async def get_stats(self) -> Dict:
|
||||||
|
"""Get statistics about favorites"""
|
||||||
|
await self._load()
|
||||||
|
|
||||||
|
total = len(self._favorites)
|
||||||
|
|
||||||
|
# Count by provider
|
||||||
|
by_provider = {}
|
||||||
|
for fav in self._favorites.values():
|
||||||
|
provider = fav["provider"]
|
||||||
|
by_provider[provider] = by_provider.get(provider, 0) + 1
|
||||||
|
|
||||||
|
# Count by genre
|
||||||
|
by_genre = {}
|
||||||
|
for fav in self._favorites.values():
|
||||||
|
for genre in fav.get("metadata", {}).get("genres", []):
|
||||||
|
by_genre[genre] = by_genre.get(genre, 0) + 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"by_provider": by_provider,
|
||||||
|
"by_genre": by_genre
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Global favorites manager instance
|
||||||
|
_favorites_manager: Optional[FavoritesManager] = None
|
||||||
|
|
||||||
|
|
||||||
|
def get_favorites_manager() -> FavoritesManager:
|
||||||
|
"""Get the global favorites manager instance"""
|
||||||
|
global _favorites_manager
|
||||||
|
if _favorites_manager is None:
|
||||||
|
_favorites_manager = FavoritesManager()
|
||||||
|
return _favorites_manager
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"anime": "Frieren",
|
||||||
|
"seasons": {
|
||||||
|
"1": {
|
||||||
|
"name": "Saison 1",
|
||||||
|
"episodes": [
|
||||||
|
{"episode": "01", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100332.mp4"},
|
||||||
|
{"episode": "02", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100334.mp4"},
|
||||||
|
{"episode": "03", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100336.mp4"},
|
||||||
|
{"episode": "04", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100338.mp4"},
|
||||||
|
{"episode": "05", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100340.mp4"}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"name": "Saison 2",
|
||||||
|
"episodes": [
|
||||||
|
{"episode": "01", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100333.mp4"},
|
||||||
|
{"episode": "02", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100335.mp4"},
|
||||||
|
{"episode": "03", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100337.mp4"},
|
||||||
|
{"episode": "04", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100339.mp4"},
|
||||||
|
{"episode": "05", "sibnet_url": "https://video.sibnet.ru/v/ba709e92c00d8a592bdae62447185e9a/6100341.mp4"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
"""Kitsu API integration as alternative to MAL"""
|
||||||
|
import httpx
|
||||||
|
from typing import List, Dict, Optional
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class KitsuAPI:
|
||||||
|
"""Kitsu.io API for anime information - alternative to MAL"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.base_url = "https://kitsu.io/api/edge"
|
||||||
|
self.client = httpx.AsyncClient(timeout=30.0, follow_redirects=True)
|
||||||
|
|
||||||
|
async def search_anime(self, query: str, limit: int = 10) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Search for anime by name
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query
|
||||||
|
limit: Number of results
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(
|
||||||
|
f"{self.base_url}/anime",
|
||||||
|
params={
|
||||||
|
"filter[text]": query,
|
||||||
|
"page[limit]": limit,
|
||||||
|
"fields[anime]": "canonicalTitle,titles,averageRating,episodeCount,status,synopsis,posterImage,coverImage,genres,subtype,startDate,endDate"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
anime_list = []
|
||||||
|
for anime in data.get('data', []):
|
||||||
|
attributes = anime.get('attributes', {})
|
||||||
|
titles = attributes.get('titles', {})
|
||||||
|
|
||||||
|
anime_list.append({
|
||||||
|
'mal_id': anime.get('id'), # Using Kitsu ID
|
||||||
|
'title': attributes.get('canonicalTitle', ''),
|
||||||
|
'title_japanese': titles.get('en_jp', ''),
|
||||||
|
'title_english': titles.get('en', ''),
|
||||||
|
'episodes': attributes.get('episodeCount'),
|
||||||
|
'status': self._translate_status(attributes.get('status')),
|
||||||
|
'score': float(attributes.get('averageRating', 0)) / 10 if attributes.get('averageRating') else 0,
|
||||||
|
'synopsis': attributes.get('synopsis', ''),
|
||||||
|
'genres': self._extract_genres(anime),
|
||||||
|
'images': self._extract_images(attributes),
|
||||||
|
'url': f"https://kitsu.io/anime/{anime.get('id')}",
|
||||||
|
'subtype': attributes.get('subtype'),
|
||||||
|
'year': self._extract_year(attributes.get('startDate'))
|
||||||
|
})
|
||||||
|
|
||||||
|
return anime_list
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error searching anime on Kitsu: {e}", exc_info=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_anime_details(self, anime_id: str) -> Optional[Dict]:
|
||||||
|
"""
|
||||||
|
Get full details of an anime including related anime
|
||||||
|
|
||||||
|
Args:
|
||||||
|
anime_id: Kitsu anime ID
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with anime details
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = await self.client.get(
|
||||||
|
f"{self.base_url}/anime/{anime_id}",
|
||||||
|
params={
|
||||||
|
"include": "genres,relationships AnimeProductions"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
if 'data' not in data:
|
||||||
|
return None
|
||||||
|
|
||||||
|
anime = data['data']
|
||||||
|
attributes = anime.get('attributes', {})
|
||||||
|
titles = attributes.get('titles', {})
|
||||||
|
|
||||||
|
anime_details = {
|
||||||
|
'mal_id': anime.get('id'),
|
||||||
|
'title': attributes.get('canonicalTitle', ''),
|
||||||
|
'title_japanese': titles.get('en_jp', ''),
|
||||||
|
'title_english': titles.get('en', ''),
|
||||||
|
'episodes': attributes.get('episodeCount'),
|
||||||
|
'status': self._translate_status(attributes.get('status')),
|
||||||
|
'rating': attributes.get('ageRating', ''),
|
||||||
|
'score': float(attributes.get('averageRating', 0)) / 10 if attributes.get('averageRating') else 0,
|
||||||
|
'synopsis': attributes.get('synopsis', ''),
|
||||||
|
'background': '',
|
||||||
|
'genres': self._extract_genres(anime),
|
||||||
|
'themes': [],
|
||||||
|
'studios': [], # Would need separate API call
|
||||||
|
'producers': [],
|
||||||
|
'source': '',
|
||||||
|
'duration': '',
|
||||||
|
'season': '',
|
||||||
|
'year': self._extract_year(attributes.get('startDate')),
|
||||||
|
'images': self._extract_images(attributes),
|
||||||
|
'url': f"https://kitsu.io/anime/{anime.get('id')}",
|
||||||
|
'related': [] # Kitsu relationships are complex
|
||||||
|
}
|
||||||
|
|
||||||
|
return anime_details
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching anime details from Kitsu: {e}", exc_info=True)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _translate_status(self, status: str) -> str:
|
||||||
|
"""Translate Kitsu status to MAL format"""
|
||||||
|
translations = {
|
||||||
|
'current': 'Airing',
|
||||||
|
'finished': 'Finished Airing',
|
||||||
|
'tba': 'To Be Aired',
|
||||||
|
'unreleased': 'To Be Aired',
|
||||||
|
'upcoming': 'To Be Aired'
|
||||||
|
}
|
||||||
|
return translations.get(status, status or '')
|
||||||
|
|
||||||
|
def _extract_genres(self, anime: Dict) -> List[str]:
|
||||||
|
"""Extract genres from anime data"""
|
||||||
|
genres = []
|
||||||
|
if 'relationships' in anime:
|
||||||
|
genres_rel = anime['relationships'].get('genres', {})
|
||||||
|
if 'data' in genres_rel:
|
||||||
|
for genre in genres_rel['data']:
|
||||||
|
genres.append(genre.get('id', '').title())
|
||||||
|
return genres
|
||||||
|
|
||||||
|
def _extract_images(self, attributes: Dict) -> Dict:
|
||||||
|
"""Extract images from attributes"""
|
||||||
|
poster = attributes.get('posterImage', {})
|
||||||
|
cover = attributes.get('coverImage', {})
|
||||||
|
|
||||||
|
return {
|
||||||
|
'jpg': {
|
||||||
|
'image_url': poster.get('small') or poster.get('medium') or poster.get('large'),
|
||||||
|
'large_image_url': poster.get('large') or poster.get('medium')
|
||||||
|
},
|
||||||
|
'webp': {
|
||||||
|
'image_url': poster.get('small') or poster.get('medium'),
|
||||||
|
'large_image_url': poster.get('large') or poster.get('medium')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def _extract_year(self, date_str: Optional[str]) -> Optional[int]:
|
||||||
|
"""Extract year from date string"""
|
||||||
|
if date_str:
|
||||||
|
try:
|
||||||
|
return int(date_str.split('-')[0])
|
||||||
|
except (ValueError, IndexError):
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
"""Close the HTTP client"""
|
||||||
|
await self.client.aclose()
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
import logging
|
|
||||||
import sys
|
|
||||||
|
|
||||||
|
|
||||||
def setup_logging(debug: bool = False) -> None:
|
|
||||||
level = logging.DEBUG if debug else logging.INFO
|
|
||||||
logging.basicConfig(
|
|
||||||
level=level,
|
|
||||||
format="%(asctime)s | %(levelname)-7s | %(name)s | %(message)s",
|
|
||||||
stream=sys.stdout,
|
|
||||||
force=True,
|
|
||||||
)
|
|
||||||
# Réduit le bruit des libs tierces
|
|
||||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
||||||
logging.getLogger("httpcore").setLevel(logging.WARNING)
|
|
||||||
-84
@@ -1,84 +0,0 @@
|
|||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from contextlib import asynccontextmanager
|
|
||||||
|
|
||||||
from fastapi import FastAPI
|
|
||||||
from fastapi.staticfiles import StaticFiles
|
|
||||||
|
|
||||||
from app.config import BASE_DIR, get_settings
|
|
||||||
from app.db import db
|
|
||||||
from app.logging_config import setup_logging
|
|
||||||
from app.routers import (
|
|
||||||
admin,
|
|
||||||
auth,
|
|
||||||
discover,
|
|
||||||
downloads,
|
|
||||||
library,
|
|
||||||
pages,
|
|
||||||
proxy,
|
|
||||||
qbit,
|
|
||||||
search,
|
|
||||||
system,
|
|
||||||
torznab,
|
|
||||||
)
|
|
||||||
from app.scrapers.http import close_client
|
|
||||||
from app.services.discover import discover as discover_service
|
|
||||||
from app.services.downloads import download_manager
|
|
||||||
from app.services.settings import apply_source_base_urls
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def _log_task_error(task: asyncio.Task) -> None:
|
|
||||||
if task.cancelled():
|
|
||||||
return
|
|
||||||
exc = task.exception()
|
|
||||||
if exc is not None:
|
|
||||||
logger.warning("Réchauffe découverte échouée : %s", exc)
|
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
|
||||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
|
||||||
settings = get_settings()
|
|
||||||
setup_logging(settings.debug)
|
|
||||||
await db.connect()
|
|
||||||
await apply_source_base_urls()
|
|
||||||
await download_manager.start()
|
|
||||||
warmup = asyncio.create_task(discover_service.latest_by_type())
|
|
||||||
warmup.add_done_callback(_log_task_error)
|
|
||||||
logger.info("%s prêt", settings.app_name)
|
|
||||||
yield
|
|
||||||
warmup.cancel()
|
|
||||||
await download_manager.stop()
|
|
||||||
await close_client()
|
|
||||||
await db.close()
|
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
|
||||||
settings = get_settings()
|
|
||||||
app = FastAPI(title=settings.app_name, lifespan=lifespan)
|
|
||||||
app.include_router(torznab.router)
|
|
||||||
app.include_router(qbit.router)
|
|
||||||
app.include_router(system.router)
|
|
||||||
app.include_router(pages.router)
|
|
||||||
app.include_router(pages.protected)
|
|
||||||
|
|
||||||
app.include_router(auth.router)
|
|
||||||
app.include_router(search.router)
|
|
||||||
app.include_router(discover.router)
|
|
||||||
app.include_router(downloads.router)
|
|
||||||
app.include_router(library.router)
|
|
||||||
app.include_router(admin.router)
|
|
||||||
app.include_router(proxy.router)
|
|
||||||
|
|
||||||
app.mount("/static", StaticFiles(directory=BASE_DIR / "app" / "static"), name="static")
|
|
||||||
|
|
||||||
@app.get("/health")
|
|
||||||
async def health() -> dict[str, str]:
|
|
||||||
return {"status": "ok"}
|
|
||||||
|
|
||||||
return app
|
|
||||||
|
|
||||||
|
|
||||||
app = create_app()
|
|
||||||
@@ -0,0 +1,423 @@
|
|||||||
|
"""
|
||||||
|
Metadata enrichment service with Kitsu API fallback.
|
||||||
|
|
||||||
|
This module provides intelligent metadata enrichment by:
|
||||||
|
1. Merging provider metadata with Kitsu API data
|
||||||
|
2. Filling missing fields from Kitsu
|
||||||
|
3. Normalizing data formats across providers
|
||||||
|
4. Caching enriched metadata to reduce API calls
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from typing import Dict, Optional, List, Set
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
import json
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from app.kitsu_api import KitsuAPI
|
||||||
|
from app.models import AnimeMetadata
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class MetadataEnricher:
|
||||||
|
"""
|
||||||
|
Enriches anime metadata by combining provider data with Kitsu API fallback.
|
||||||
|
Caches results to minimize API calls.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Fields that Kitsu can provide as fallback
|
||||||
|
# Note: studio is not included as Kitsu API requires separate calls
|
||||||
|
KITSU_FIELDS = {
|
||||||
|
'synopsis', 'genres', 'rating', 'release_year',
|
||||||
|
'poster_image', 'banner_image', 'total_episodes', 'status',
|
||||||
|
'alternative_titles'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Cache duration in hours
|
||||||
|
CACHE_DURATION_HOURS = 24
|
||||||
|
|
||||||
|
def __init__(self, cache_dir: str = "config"):
|
||||||
|
self.cache_dir = Path(cache_dir)
|
||||||
|
self.cache_file = self.cache_dir / "metadata_cache.json"
|
||||||
|
self.kitsu_api = KitsuAPI()
|
||||||
|
self._cache: Dict[str, Dict] = {}
|
||||||
|
self._cache_dirty = False
|
||||||
|
|
||||||
|
# Load cache on initialization
|
||||||
|
self._load_cache()
|
||||||
|
|
||||||
|
def _load_cache(self):
|
||||||
|
"""Load metadata cache from disk."""
|
||||||
|
try:
|
||||||
|
if self.cache_file.exists():
|
||||||
|
with open(self.cache_file, 'r', encoding='utf-8') as f:
|
||||||
|
data = json.load(f)
|
||||||
|
# Filter out expired entries
|
||||||
|
now = datetime.now()
|
||||||
|
self._cache = {
|
||||||
|
k: v for k, v in data.items()
|
||||||
|
if datetime.fromisoformat(v.get('cached_at', '')) >
|
||||||
|
now - timedelta(hours=self.CACHE_DURATION_HOURS)
|
||||||
|
}
|
||||||
|
logger.info(f"Loaded {len(self._cache)} cached metadata entries")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to load metadata cache: {e}")
|
||||||
|
self._cache = {}
|
||||||
|
|
||||||
|
def _save_cache(self):
|
||||||
|
"""Save metadata cache to disk."""
|
||||||
|
if not self._cache_dirty:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(self.cache_file, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(self._cache, f, ensure_ascii=False, indent=2)
|
||||||
|
self._cache_dirty = False
|
||||||
|
logger.debug("Saved metadata cache")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to save metadata cache: {e}")
|
||||||
|
|
||||||
|
def _get_cache_key(self, title: str, url: Optional[str] = None) -> str:
|
||||||
|
"""Generate cache key from title and URL."""
|
||||||
|
# Use both title and URL for more precise caching
|
||||||
|
key_data = f"{title}|{url or ''}"
|
||||||
|
return hashlib.md5(key_data.encode()).hexdigest()
|
||||||
|
|
||||||
|
def _get_cached_metadata(self, cache_key: str) -> Optional[Dict]:
|
||||||
|
"""Get cached metadata if available and not expired."""
|
||||||
|
if cache_key in self._cache:
|
||||||
|
entry = self._cache[cache_key]
|
||||||
|
cached_at = datetime.fromisoformat(entry.get('cached_at', ''))
|
||||||
|
if cached_at > datetime.now() - timedelta(hours=self.CACHE_DURATION_HOURS):
|
||||||
|
logger.debug(f"Cache hit for key: {cache_key}")
|
||||||
|
return entry.get('metadata')
|
||||||
|
else:
|
||||||
|
# Remove expired entry
|
||||||
|
del self._cache[cache_key]
|
||||||
|
self._cache_dirty = True
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _set_cached_metadata(self, cache_key: str, metadata: Dict):
|
||||||
|
"""Cache enriched metadata."""
|
||||||
|
self._cache[cache_key] = {
|
||||||
|
'metadata': metadata,
|
||||||
|
'cached_at': datetime.now().isoformat()
|
||||||
|
}
|
||||||
|
self._cache_dirty = True
|
||||||
|
|
||||||
|
async def enrich_metadata(
|
||||||
|
self,
|
||||||
|
provider_metadata: Dict,
|
||||||
|
title: str,
|
||||||
|
url: Optional[str] = None,
|
||||||
|
use_kitsu_fallback: bool = True
|
||||||
|
) -> AnimeMetadata:
|
||||||
|
"""
|
||||||
|
Enrich provider metadata with Kitsu API fallback.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
provider_metadata: Metadata dict from anime provider
|
||||||
|
title: Anime title (for Kitsu search)
|
||||||
|
url: Optional anime URL (for cache key)
|
||||||
|
use_kitsu_fallback: Whether to use Kitsu API for missing fields
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Enriched AnimeMetadata object
|
||||||
|
"""
|
||||||
|
# Check cache first
|
||||||
|
cache_key = self._get_cache_key(title, url)
|
||||||
|
cached = self._get_cached_metadata(cache_key)
|
||||||
|
if cached:
|
||||||
|
return AnimeMetadata(**cached)
|
||||||
|
|
||||||
|
# Start with provider metadata
|
||||||
|
enriched = provider_metadata.copy()
|
||||||
|
|
||||||
|
# Check which fields are missing
|
||||||
|
missing_fields = self._get_missing_fields(enriched)
|
||||||
|
|
||||||
|
if missing_fields and use_kitsu_fallback:
|
||||||
|
logger.info(f"Missing fields for '{title}': {missing_fields} - fetching from Kitsu")
|
||||||
|
try:
|
||||||
|
# Fetch from Kitsu
|
||||||
|
kitsu_metadata = await self._fetch_from_kitsu(title)
|
||||||
|
|
||||||
|
if kitsu_metadata:
|
||||||
|
# Merge Kitsu data
|
||||||
|
enriched = self._merge_metadata(enriched, kitsu_metadata)
|
||||||
|
enriched['_kitsu_enriched'] = True
|
||||||
|
enriched['_enriched_fields'] = list(missing_fields)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to fetch Kitsu metadata for '{title}': {e}")
|
||||||
|
|
||||||
|
# Calculate quality score
|
||||||
|
enriched['_quality_score'] = self._calculate_quality_score(enriched)
|
||||||
|
|
||||||
|
# Convert to AnimeMetadata
|
||||||
|
result = AnimeMetadata(**{
|
||||||
|
k: v for k, v in enriched.items()
|
||||||
|
if not k.startswith('_') # Exclude internal fields
|
||||||
|
})
|
||||||
|
|
||||||
|
# Cache the result
|
||||||
|
self._set_cached_metadata(cache_key, result.model_dump())
|
||||||
|
|
||||||
|
# Periodically save cache
|
||||||
|
if self._cache_dirty and len(self._cache) % 10 == 0:
|
||||||
|
self._save_cache()
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _get_missing_fields(self, metadata: Dict) -> Set[str]:
|
||||||
|
"""Identify which metadata fields are missing or empty."""
|
||||||
|
missing = set()
|
||||||
|
for field in self.KITSU_FIELDS:
|
||||||
|
value = metadata.get(field)
|
||||||
|
if value is None or value == [] or value == '':
|
||||||
|
missing.add(field)
|
||||||
|
return missing
|
||||||
|
|
||||||
|
async def _fetch_from_kitsu(self, title: str) -> Optional[Dict]:
|
||||||
|
"""Fetch metadata from Kitsu API."""
|
||||||
|
try:
|
||||||
|
# Search for anime
|
||||||
|
results = await self.kitsu_api.search_anime(title, limit=1)
|
||||||
|
|
||||||
|
if results and len(results) > 0:
|
||||||
|
anime_data = results[0]
|
||||||
|
return self._convert_kitsu_to_metadata(anime_data)
|
||||||
|
else:
|
||||||
|
logger.debug(f"No Kitsu results for '{title}'")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching from Kitsu for '{title}': {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _convert_kitsu_to_metadata(self, kitsu_data: Dict) -> Dict:
|
||||||
|
"""Convert Kitsu API response to metadata format."""
|
||||||
|
metadata = {}
|
||||||
|
|
||||||
|
# Synopsis
|
||||||
|
if kitsu_data.get('synopsis'):
|
||||||
|
metadata['synopsis'] = kitsu_data['synopsis']
|
||||||
|
|
||||||
|
# Genres
|
||||||
|
if kitsu_data.get('genres'):
|
||||||
|
metadata['genres'] = kitsu_data['genres']
|
||||||
|
|
||||||
|
# Rating (Kitsu returns score out of 10, convert to string)
|
||||||
|
if kitsu_data.get('score'):
|
||||||
|
score = kitsu_data['score']
|
||||||
|
if score > 0:
|
||||||
|
metadata['rating'] = f"{score:.1f}/10"
|
||||||
|
|
||||||
|
# Release year
|
||||||
|
if kitsu_data.get('year'):
|
||||||
|
metadata['release_year'] = kitsu_data['year']
|
||||||
|
|
||||||
|
# Poster image
|
||||||
|
if kitsu_data.get('images', {}).get('jpg', {}).get('large_image_url'):
|
||||||
|
metadata['poster_image'] = kitsu_data['images']['jpg']['large_image_url']
|
||||||
|
elif kitsu_data.get('images', {}).get('jpg', {}).get('image_url'):
|
||||||
|
metadata['poster_image'] = kitsu_data['images']['jpg']['image_url']
|
||||||
|
|
||||||
|
# Banner image (Kitsu calls it coverImage)
|
||||||
|
# Note: Kitsu API structure doesn't clearly separate poster vs banner,
|
||||||
|
# but we can use different sizes if available
|
||||||
|
if kitsu_data.get('images', {}).get('webp', {}).get('large_image_url'):
|
||||||
|
metadata['banner_image'] = kitsu_data['images']['webp']['large_image_url']
|
||||||
|
|
||||||
|
# Total episodes
|
||||||
|
if kitsu_data.get('episodes'):
|
||||||
|
metadata['total_episodes'] = kitsu_data['episodes']
|
||||||
|
|
||||||
|
# Status
|
||||||
|
if kitsu_data.get('status'):
|
||||||
|
# Translate Kitsu status to our format
|
||||||
|
status_map = {
|
||||||
|
'Airing': 'Ongoing',
|
||||||
|
'Finished Airing': 'Completed',
|
||||||
|
'To Be Aired': 'Upcoming'
|
||||||
|
}
|
||||||
|
metadata['status'] = status_map.get(
|
||||||
|
kitsu_data['status'],
|
||||||
|
kitsu_data['status']
|
||||||
|
)
|
||||||
|
|
||||||
|
# Alternative titles
|
||||||
|
alt_titles = []
|
||||||
|
if kitsu_data.get('title_japanese'):
|
||||||
|
alt_titles.append(kitsu_data['title_japanese'])
|
||||||
|
if kitsu_data.get('title_english'):
|
||||||
|
alt_titles.append(kitsu_data['title_english'])
|
||||||
|
if alt_titles:
|
||||||
|
metadata['alternative_titles'] = alt_titles
|
||||||
|
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
def _merge_metadata(
|
||||||
|
self,
|
||||||
|
provider_metadata: Dict,
|
||||||
|
kitsu_metadata: Dict
|
||||||
|
) -> Dict:
|
||||||
|
"""
|
||||||
|
Merge provider and Kitsu metadata, preferring provider data.
|
||||||
|
|
||||||
|
Provider data takes priority except for missing fields.
|
||||||
|
"""
|
||||||
|
merged = provider_metadata.copy()
|
||||||
|
|
||||||
|
for field, value in kitsu_metadata.items():
|
||||||
|
# Only use Kitsu data if provider doesn't have it
|
||||||
|
if field not in merged or not merged[field]:
|
||||||
|
merged[field] = value
|
||||||
|
|
||||||
|
return merged
|
||||||
|
|
||||||
|
def _calculate_quality_score(self, metadata: Dict) -> float:
|
||||||
|
"""
|
||||||
|
Calculate metadata quality score (0-1).
|
||||||
|
|
||||||
|
Based on completeness of critical fields.
|
||||||
|
"""
|
||||||
|
weights = {
|
||||||
|
'synopsis': 0.2,
|
||||||
|
'genres': 0.15,
|
||||||
|
'rating': 0.1,
|
||||||
|
'release_year': 0.1,
|
||||||
|
'studio': 0.1,
|
||||||
|
'poster_image': 0.15,
|
||||||
|
'banner_image': 0.05,
|
||||||
|
'total_episodes': 0.05,
|
||||||
|
'status': 0.05,
|
||||||
|
'alternative_titles': 0.05
|
||||||
|
}
|
||||||
|
|
||||||
|
total_weight = sum(weights.values())
|
||||||
|
score = 0.0
|
||||||
|
|
||||||
|
for field, weight in weights.items():
|
||||||
|
value = metadata.get(field)
|
||||||
|
if value:
|
||||||
|
# For lists, check if not empty
|
||||||
|
if isinstance(value, list):
|
||||||
|
if len(value) > 0:
|
||||||
|
score += weight
|
||||||
|
# For strings, check if not empty
|
||||||
|
elif isinstance(value, str):
|
||||||
|
if len(value) > 10: # Minimum meaningful length
|
||||||
|
score += weight
|
||||||
|
# For numbers
|
||||||
|
else:
|
||||||
|
score += weight
|
||||||
|
|
||||||
|
return round(score / total_weight, 2) if total_weight > 0 else 0.0
|
||||||
|
|
||||||
|
async def enrich_search_results(
|
||||||
|
self,
|
||||||
|
results: List[Dict],
|
||||||
|
use_kitsu_fallback: bool = True
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Enrich metadata for a list of search results.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
results: List of search result dicts with optional 'metadata' field
|
||||||
|
use_kitsu_fallback: Whether to use Kitsu API
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of results with enriched metadata
|
||||||
|
"""
|
||||||
|
enriched_results = []
|
||||||
|
|
||||||
|
# Process results in parallel for better performance
|
||||||
|
enrichment_tasks = []
|
||||||
|
for result in results:
|
||||||
|
# Skip if no metadata - will add later in order
|
||||||
|
if 'metadata' not in result:
|
||||||
|
continue
|
||||||
|
|
||||||
|
task = self.enrich_metadata(
|
||||||
|
provider_metadata=result['metadata'],
|
||||||
|
title=result.get('title', ''),
|
||||||
|
url=result.get('url'),
|
||||||
|
use_kitsu_fallback=use_kitsu_fallback
|
||||||
|
)
|
||||||
|
enrichment_tasks.append(task)
|
||||||
|
|
||||||
|
# Wait for all enrichment tasks
|
||||||
|
if enrichment_tasks:
|
||||||
|
enriched_metadata_list = await asyncio.gather(
|
||||||
|
*enrichment_tasks,
|
||||||
|
return_exceptions=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# Update results with enriched metadata
|
||||||
|
# Create index mapping to preserve order
|
||||||
|
temp_results = {}
|
||||||
|
metadata_idx = 0
|
||||||
|
for i, result in enumerate(results):
|
||||||
|
if 'metadata' in result:
|
||||||
|
enriched_meta = enriched_metadata_list[metadata_idx]
|
||||||
|
|
||||||
|
if isinstance(enriched_meta, Exception):
|
||||||
|
logger.warning(
|
||||||
|
f"Failed to enrich metadata for '{result.get('title')}': {enriched_meta}"
|
||||||
|
)
|
||||||
|
# Keep original metadata
|
||||||
|
result_copy = result.copy()
|
||||||
|
else:
|
||||||
|
result_copy = result.copy()
|
||||||
|
result_copy['metadata'] = enriched_meta.model_dump()
|
||||||
|
|
||||||
|
temp_results[i] = result_copy
|
||||||
|
metadata_idx += 1
|
||||||
|
|
||||||
|
# Build final result list in correct order
|
||||||
|
enriched_results = []
|
||||||
|
for i in range(len(results)):
|
||||||
|
if i in temp_results:
|
||||||
|
enriched_results.append(temp_results[i])
|
||||||
|
else:
|
||||||
|
# No metadata result - use original
|
||||||
|
enriched_results.append(results[i].copy())
|
||||||
|
|
||||||
|
return enriched_results
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
"""Close resources and save cache."""
|
||||||
|
await self.kitsu_api.close()
|
||||||
|
self._save_cache()
|
||||||
|
logger.info("MetadataEnricher closed")
|
||||||
|
|
||||||
|
|
||||||
|
# Global instance
|
||||||
|
_enricher_instance: Optional[MetadataEnricher] = None
|
||||||
|
_enricher_lock = asyncio.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
async def get_metadata_enricher() -> MetadataEnricher:
|
||||||
|
"""Get or create the global MetadataEnricher instance."""
|
||||||
|
global _enricher_instance
|
||||||
|
|
||||||
|
if _enricher_instance is None:
|
||||||
|
async with _enricher_lock:
|
||||||
|
if _enricher_instance is None:
|
||||||
|
_enricher_instance = MetadataEnricher()
|
||||||
|
logger.info("Created global MetadataEnricher instance")
|
||||||
|
|
||||||
|
return _enricher_instance
|
||||||
|
|
||||||
|
|
||||||
|
async def close_metadata_enricher():
|
||||||
|
"""Close the global MetadataEnricher instance."""
|
||||||
|
global _enricher_instance
|
||||||
|
|
||||||
|
if _enricher_instance is not None:
|
||||||
|
await _enricher_instance.close()
|
||||||
|
_enricher_instance = None
|
||||||
|
logger.info("Closed global MetadataEnricher instance")
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
from pydantic import BaseModel
|
||||||
|
from enum import Enum
|
||||||
|
from typing import Optional
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadStatus(str, Enum):
|
||||||
|
PENDING = "pending"
|
||||||
|
DOWNLOADING = "downloading"
|
||||||
|
PAUSED = "paused"
|
||||||
|
COMPLETED = "completed"
|
||||||
|
FAILED = "failed"
|
||||||
|
CANCELLED = "cancelled"
|
||||||
|
|
||||||
|
|
||||||
|
class HostType(str, Enum):
|
||||||
|
RAPIDFILE = "rapidfile"
|
||||||
|
UNFICHIER = "1fichier"
|
||||||
|
DOODSTREAM = "doodstream"
|
||||||
|
OTHER = "other"
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadTask(BaseModel):
|
||||||
|
id: str
|
||||||
|
url: str
|
||||||
|
filename: str
|
||||||
|
host: HostType
|
||||||
|
status: DownloadStatus
|
||||||
|
progress: float = 0.0
|
||||||
|
downloaded_bytes: int = 0
|
||||||
|
total_bytes: Optional[int] = None
|
||||||
|
speed: float = 0.0
|
||||||
|
error: Optional[str] = None
|
||||||
|
created_at: datetime
|
||||||
|
started_at: Optional[datetime] = None
|
||||||
|
completed_at: Optional[datetime] = None
|
||||||
|
file_path: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadRequest(BaseModel):
|
||||||
|
url: str
|
||||||
|
filename: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class AnimeMetadata(BaseModel):
|
||||||
|
"""Metadata for anime series"""
|
||||||
|
synopsis: Optional[str] = None
|
||||||
|
genres: list[str] = []
|
||||||
|
rating: Optional[str] = None # Could be "PG-13", "R", etc., or numeric like "8.5/10"
|
||||||
|
release_year: Optional[int] = None
|
||||||
|
studio: Optional[str] = None
|
||||||
|
poster_image: Optional[str] = None
|
||||||
|
banner_image: Optional[str] = None
|
||||||
|
total_episodes: Optional[int] = None
|
||||||
|
status: Optional[str] = None # "Ongoing", "Completed", etc.
|
||||||
|
alternative_titles: list[str] = []
|
||||||
|
|
||||||
|
|
||||||
|
class AnimeSearchResult(BaseModel):
|
||||||
|
"""Enhanced search result with metadata"""
|
||||||
|
title: str
|
||||||
|
url: str
|
||||||
|
cover_image: Optional[str] = None
|
||||||
|
type: str # "search_result" or "direct"
|
||||||
|
metadata: Optional[AnimeMetadata] = None
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""Authentication models for user management"""
|
||||||
|
from pydantic import BaseModel, EmailStr, Field
|
||||||
|
from typing import Optional
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
class UserCreate(BaseModel):
|
||||||
|
"""Schema for user registration"""
|
||||||
|
username: str = Field(..., min_length=3, max_length=50)
|
||||||
|
email: Optional[EmailStr] = None
|
||||||
|
password: str = Field(..., min_length=6)
|
||||||
|
full_name: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class UserLogin(BaseModel):
|
||||||
|
"""Schema for user login"""
|
||||||
|
username: str
|
||||||
|
password: str
|
||||||
|
|
||||||
|
|
||||||
|
class User(BaseModel):
|
||||||
|
"""Schema for user data"""
|
||||||
|
id: str
|
||||||
|
username: str
|
||||||
|
email: Optional[str] = None
|
||||||
|
full_name: Optional[str] = None
|
||||||
|
is_active: bool = True
|
||||||
|
created_at: datetime
|
||||||
|
last_login: Optional[datetime] = None
|
||||||
|
|
||||||
|
|
||||||
|
class Token(BaseModel):
|
||||||
|
"""Schema for authentication token"""
|
||||||
|
access_token: str
|
||||||
|
token_type: str = "bearer"
|
||||||
|
|
||||||
|
|
||||||
|
class UserInDB(User):
|
||||||
|
"""Schema for user stored in database (with hashed password)"""
|
||||||
|
hashed_password: str
|
||||||
@@ -0,0 +1,198 @@
|
|||||||
|
"""Pydantic models for Sonarr webhook integration"""
|
||||||
|
from pydantic import BaseModel, Field, validator
|
||||||
|
from typing import Optional, Dict, Any, List
|
||||||
|
from datetime import datetime
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrEventType(str, Enum):
|
||||||
|
"""Sonarr event types"""
|
||||||
|
GRAB = "Grab"
|
||||||
|
DOWNLOAD = "Download"
|
||||||
|
MOVIE_DELETE = "MovieDelete"
|
||||||
|
MOVIE_FILE_DELETE = "MovieFileDelete"
|
||||||
|
RENAME = "Rename"
|
||||||
|
DELETE = "Delete"
|
||||||
|
TEST = "Test"
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrQuality(BaseModel):
|
||||||
|
"""Quality information from Sonarr"""
|
||||||
|
quality: Dict[str, Any]
|
||||||
|
revision: Dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrRelease(BaseModel):
|
||||||
|
"""Release information from Sonarr"""
|
||||||
|
indexer: str
|
||||||
|
releaseTitle: str
|
||||||
|
quality: SonarrQuality
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrEpisodeFile(BaseModel):
|
||||||
|
"""Episode file information"""
|
||||||
|
id: int
|
||||||
|
seriesId: int
|
||||||
|
seasonNumber: int
|
||||||
|
episodeNumber: int
|
||||||
|
relativePath: str
|
||||||
|
path: str
|
||||||
|
size: int
|
||||||
|
dateAdded: datetime
|
||||||
|
quality: SonarrQuality
|
||||||
|
mediaInfo: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrSeries(BaseModel):
|
||||||
|
"""Series information from Sonarr"""
|
||||||
|
tvdbId: int = Field(..., alias="tvdbId")
|
||||||
|
title: str
|
||||||
|
sortTitle: str
|
||||||
|
status: str
|
||||||
|
ended: bool
|
||||||
|
overview: str
|
||||||
|
network: str
|
||||||
|
airTime: str
|
||||||
|
images: List[Dict[str, Any]]
|
||||||
|
seasons: List[int]
|
||||||
|
year: int
|
||||||
|
path: str
|
||||||
|
qualityProfileId: int
|
||||||
|
languageProfileId: int
|
||||||
|
seasonFolder: bool
|
||||||
|
monitored: bool
|
||||||
|
useSceneNumbering: bool
|
||||||
|
runtime: int
|
||||||
|
tvRageId: Optional[int] = None
|
||||||
|
tvMazeId: Optional[int] = None
|
||||||
|
firstAired: Optional[datetime] = None
|
||||||
|
seriesType: str = "standard"
|
||||||
|
cleanTitle: str
|
||||||
|
imdbId: str
|
||||||
|
titleSlug: str
|
||||||
|
certification: str
|
||||||
|
genres: List[str]
|
||||||
|
tags: List[int]
|
||||||
|
added: datetime
|
||||||
|
ratings: Dict[str, Any]
|
||||||
|
id: int
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
populate_by_name = True
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrEpisode(BaseModel):
|
||||||
|
"""Episode information from Sonarr"""
|
||||||
|
seriesId: int
|
||||||
|
episodeFileId: int
|
||||||
|
seasonNumber: int
|
||||||
|
episodeNumber: int
|
||||||
|
title: str
|
||||||
|
airDate: str
|
||||||
|
airDateUtc: datetime
|
||||||
|
overview: str
|
||||||
|
hasFile: bool
|
||||||
|
monitored: bool
|
||||||
|
absoluteEpisodeNumber: Optional[int] = None
|
||||||
|
unverifiedSceneNumbering: bool = False
|
||||||
|
id: int
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrWebhookPayload(BaseModel):
|
||||||
|
"""Main Sonarr webhook payload"""
|
||||||
|
eventType: SonarrEventType
|
||||||
|
instanceName: str
|
||||||
|
applicationUrl: str
|
||||||
|
series: Optional[SonarrSeries] = None
|
||||||
|
episodes: Optional[List[SonarrEpisode]] = None
|
||||||
|
release: Optional[SonarrRelease] = None
|
||||||
|
episodeFile: Optional[SonarrEpisodeFile] = None
|
||||||
|
deletedFiles: Optional[List[str]] = None
|
||||||
|
deleteEpisodeFiles: bool = False
|
||||||
|
|
||||||
|
@validator('episodes')
|
||||||
|
def validate_episodes(cls, v, values):
|
||||||
|
"""Ensure episodes are present for relevant event types"""
|
||||||
|
event_type = values.get('eventType')
|
||||||
|
if event_type in [SonarrEventType.GRAB, SonarrEventType.DOWNLOAD, SonarrEventType.RENAME]:
|
||||||
|
if not v or len(v) == 0:
|
||||||
|
raise ValueError(f"Event type {event_type} requires episodes")
|
||||||
|
return v
|
||||||
|
|
||||||
|
@validator('series')
|
||||||
|
def validate_series(cls, v, values):
|
||||||
|
"""Ensure series is present for relevant event types"""
|
||||||
|
event_type = values.get('eventType')
|
||||||
|
if event_type in [SonarrEventType.GRAB, SonarrEventType.DOWNLOAD, SonarrEventType.RENAME, SonarrEventType.DELETE]:
|
||||||
|
if not v:
|
||||||
|
raise ValueError(f"Event type {event_type} requires series")
|
||||||
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrMapping(BaseModel):
|
||||||
|
"""Mapping between Sonarr series and anime providers"""
|
||||||
|
sonarr_series_id: int
|
||||||
|
sonarr_title: str
|
||||||
|
anime_provider: str # 'anime-sama', 'neko-sama', etc.
|
||||||
|
anime_url: str
|
||||||
|
anime_title: str
|
||||||
|
lang: str = "vostfr"
|
||||||
|
quality_preference: Optional[str] = None # '1080p', '720p', etc.
|
||||||
|
auto_download: bool = True
|
||||||
|
created_at: datetime = Field(default_factory=datetime.now)
|
||||||
|
updated_at: datetime = Field(default_factory=datetime.now)
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
json_encoders = {
|
||||||
|
datetime: lambda v: v.isoformat()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrConfig(BaseModel):
|
||||||
|
"""Sonarr webhook configuration"""
|
||||||
|
webhook_enabled: bool = False
|
||||||
|
webhook_secret: Optional[str] = None # HMAC SHA256 secret
|
||||||
|
auto_download_enabled: bool = True
|
||||||
|
default_language: str = "vostfr"
|
||||||
|
default_quality: Optional[str] = None
|
||||||
|
default_provider: str = "anime-sama"
|
||||||
|
verify_hmac: bool = False
|
||||||
|
log_webhooks: bool = True
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
json_schema_extra = {
|
||||||
|
"example": {
|
||||||
|
"webhook_enabled": True,
|
||||||
|
"webhook_secret": "your-secret-key-here",
|
||||||
|
"auto_download_enabled": True,
|
||||||
|
"default_language": "vostfr",
|
||||||
|
"default_quality": "1080p",
|
||||||
|
"default_provider": "anime-sama",
|
||||||
|
"verify_hmac": True,
|
||||||
|
"log_webhooks": True
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class SonarrDownloadRequest(BaseModel):
|
||||||
|
"""Request to download anime based on Sonarr event"""
|
||||||
|
sonarr_series_id: int
|
||||||
|
sonarr_title: str
|
||||||
|
season_number: int
|
||||||
|
episode_number: int
|
||||||
|
quality: Optional[str] = None
|
||||||
|
lang: str = "vostfr"
|
||||||
|
provider: str = "anime-sama"
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
json_schema_extra = {
|
||||||
|
"example": {
|
||||||
|
"sonarr_series_id": 123,
|
||||||
|
"sonarr_title": "Naruto Shippuden",
|
||||||
|
"season_number": 1,
|
||||||
|
"episode_number": 1,
|
||||||
|
"quality": "1080p",
|
||||||
|
"lang": "vostfr",
|
||||||
|
"provider": "anime-sama"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
"""Pydantic models for Watchlist and Auto-Download system"""
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from typing import Optional, Literal
|
||||||
|
from datetime import datetime
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
|
||||||
|
class WatchlistStatus(str, Enum):
|
||||||
|
"""Status of a watchlist item"""
|
||||||
|
ACTIVE = "active" # Currently tracking for new episodes
|
||||||
|
PAUSED = "paused" # Temporarily paused
|
||||||
|
COMPLETED = "completed" # Anime completed, no longer tracking
|
||||||
|
ARCHIVED = "archived" # Archived but kept for history
|
||||||
|
|
||||||
|
|
||||||
|
class QualityPreference(str, Enum):
|
||||||
|
"""Preferred video quality"""
|
||||||
|
AUTO = "auto" # Let provider decide
|
||||||
|
P1080 = "1080p" # Full HD
|
||||||
|
P720 = "720p" # HD
|
||||||
|
P480 = "480p" # SD
|
||||||
|
|
||||||
|
|
||||||
|
class WatchlistItem(BaseModel):
|
||||||
|
"""An anime being tracked for automatic episode downloads"""
|
||||||
|
id: str = Field(..., description="Unique identifier (UUID)")
|
||||||
|
user_id: str = Field(..., description="User ID who owns this watchlist item")
|
||||||
|
anime_title: str = Field(..., description="Title of the anime")
|
||||||
|
anime_url: str = Field(..., description="URL to the anime page")
|
||||||
|
provider_id: str = Field(..., description="Provider ID (animesama, nekosama, etc.)")
|
||||||
|
lang: Literal["vostfr", "vf"] = Field(default="vostfr", description="Language preference")
|
||||||
|
|
||||||
|
# Tracking state
|
||||||
|
last_checked: Optional[datetime] = Field(None, description="Last time we checked for new episodes")
|
||||||
|
last_episode_downloaded: int = Field(default=0, description="Last episode number downloaded")
|
||||||
|
total_episodes: Optional[int] = Field(None, description="Total episodes if known")
|
||||||
|
|
||||||
|
# Settings
|
||||||
|
auto_download: bool = Field(default=True, description="Automatically download new episodes")
|
||||||
|
quality_preference: QualityPreference = Field(default=QualityPreference.AUTO, description="Preferred quality")
|
||||||
|
status: WatchlistStatus = Field(default=WatchlistStatus.ACTIVE, description="Tracking status")
|
||||||
|
|
||||||
|
# Metadata
|
||||||
|
poster_image: Optional[str] = Field(None, description="URL to poster image")
|
||||||
|
cover_image: Optional[str] = Field(None, description="URL to cover image")
|
||||||
|
synopsis: Optional[str] = Field(None, description="Anime synopsis")
|
||||||
|
genres: list[str] = Field(default_factory=list, description="Anime genres")
|
||||||
|
|
||||||
|
# Timestamps
|
||||||
|
added_at: datetime = Field(default_factory=datetime.now, description="When added to watchlist")
|
||||||
|
updated_at: datetime = Field(default_factory=datetime.now, description="Last update time")
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
json_encoders = {
|
||||||
|
datetime: lambda v: v.isoformat()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class WatchlistItemCreate(BaseModel):
|
||||||
|
"""Model for creating a new watchlist item"""
|
||||||
|
anime_title: str
|
||||||
|
anime_url: str
|
||||||
|
provider_id: str
|
||||||
|
lang: Literal["vostfr", "vf"] = "vostfr"
|
||||||
|
auto_download: bool = True
|
||||||
|
quality_preference: QualityPreference = QualityPreference.AUTO
|
||||||
|
|
||||||
|
# Optional metadata
|
||||||
|
poster_image: Optional[str] = None
|
||||||
|
cover_image: Optional[str] = None
|
||||||
|
synopsis: Optional[str] = None
|
||||||
|
genres: list[str] = []
|
||||||
|
|
||||||
|
|
||||||
|
class WatchlistItemUpdate(BaseModel):
|
||||||
|
"""Model for updating a watchlist item"""
|
||||||
|
auto_download: Optional[bool] = None
|
||||||
|
quality_preference: Optional[QualityPreference] = None
|
||||||
|
status: Optional[WatchlistStatus] = None
|
||||||
|
last_episode_downloaded: Optional[int] = None
|
||||||
|
total_episodes: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
|
class NewEpisodeInfo(BaseModel):
|
||||||
|
"""Information about a newly detected episode"""
|
||||||
|
episode_number: int
|
||||||
|
episode_title: Optional[str] = None
|
||||||
|
episode_url: str
|
||||||
|
season_number: Optional[int] = None
|
||||||
|
anime_title: str
|
||||||
|
provider_id: str
|
||||||
|
|
||||||
|
|
||||||
|
class AutoDownloadResult(BaseModel):
|
||||||
|
"""Result of an automatic download check"""
|
||||||
|
watchlist_item_id: str
|
||||||
|
anime_title: str
|
||||||
|
new_episodes_found: int
|
||||||
|
episodes_downloaded: list[int] = Field(default_factory=list)
|
||||||
|
episodes_failed: list[tuple[int, str]] = Field(default_factory=list) # (episode_number, error_message)
|
||||||
|
checked_at: datetime = Field(default_factory=datetime.now)
|
||||||
|
|
||||||
|
|
||||||
|
class WatchlistSettings(BaseModel):
|
||||||
|
"""Global watchlist settings"""
|
||||||
|
check_interval_hours: int = Field(default=6, ge=1, le=168, description="Check interval (1-168 hours)")
|
||||||
|
auto_download_enabled: bool = Field(default=True, description="Global auto-download toggle")
|
||||||
|
max_concurrent_auto_downloads: int = Field(default=2, ge=1, le=10, description="Max concurrent auto-downloads")
|
||||||
|
notify_on_new_episodes: bool = Field(default=False, description="Send notifications for new episodes")
|
||||||
|
include_completed_anime: bool = Field(default=False, description="Check completed anime too")
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
json_schema_extra = {
|
||||||
|
"example": {
|
||||||
|
"check_interval_hours": 6,
|
||||||
|
"auto_download_enabled": True,
|
||||||
|
"max_concurrent_auto_downloads": 2,
|
||||||
|
"notify_on_new_episodes": False,
|
||||||
|
"include_completed_anime": False
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
"""Anime, series and file hosting providers configuration"""
|
||||||
|
|
||||||
|
ANIME_PROVIDERS = {
|
||||||
|
"anime-sama": {
|
||||||
|
"name": "Anime-Sama",
|
||||||
|
"domains": ["anime-sama.si", "www.anime-sama.si", "anime-sama.org", "anime-sama.store", "anime-sama.eu"],
|
||||||
|
"url_pattern": "https://anime-sama.si/catalogue/{anime}/saison{season}/{lang}/",
|
||||||
|
"icon": "🎬",
|
||||||
|
"color": "#00d9ff"
|
||||||
|
},
|
||||||
|
"anime-ultime": {
|
||||||
|
"name": "Anime-Ultime",
|
||||||
|
"domains": ["anime-ultime.net", "anime-ultime.com", "www.anime-ultime.net"],
|
||||||
|
"url_pattern": "https://www.anime-ultime.net/info-{id}-{slug}",
|
||||||
|
"icon": "▶️",
|
||||||
|
"color": "#00ff88"
|
||||||
|
},
|
||||||
|
"neko-sama": {
|
||||||
|
"name": "Neko-Sama",
|
||||||
|
"domains": ["neko-sama.fr", "nekosama.fr", "www.neko-sama.fr"],
|
||||||
|
"url_pattern": "https://neko-sama.fr/anime/{slug}",
|
||||||
|
"icon": "🐱",
|
||||||
|
"color": "#ff6b6b"
|
||||||
|
},
|
||||||
|
"vostfree": {
|
||||||
|
"name": "Vostfree",
|
||||||
|
"domains": ["vostfree.tv", "www.vostfree.tv"],
|
||||||
|
"url_pattern": "https://vostfree.tv/anime/{slug}",
|
||||||
|
"icon": "📺",
|
||||||
|
"color": "#ffd93d"
|
||||||
|
},
|
||||||
|
"french-manga": {
|
||||||
|
"name": "French-Manga",
|
||||||
|
"domains": ["french-manga.net", "w16.french-manga.net", "w15.french-manga.net", "www.french-manga.net"],
|
||||||
|
"url_pattern": "https://w16.french-manga.net/{slug}.html",
|
||||||
|
"icon": "🇫🇷",
|
||||||
|
"color": "#ff7675"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SERIES_PROVIDERS = {
|
||||||
|
"fs7": {
|
||||||
|
"name": "French Stream",
|
||||||
|
"domains": ["fs7.lol", "www.fs7.lol", "french-stream.tv", "www.french-stream.tv"],
|
||||||
|
"url_pattern": "https://fs7.lol/s-tv/{slug}.html",
|
||||||
|
"icon": "🎬",
|
||||||
|
"color": "#ff6b9d"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FILE_HOSTS = {
|
||||||
|
"1fichier": {
|
||||||
|
"name": "1fichier",
|
||||||
|
"domains": ["1fichier.com", "1fichier.fr"],
|
||||||
|
"icon": "📁",
|
||||||
|
"color": "#4ecdc4"
|
||||||
|
},
|
||||||
|
"uptobox": {
|
||||||
|
"name": "Uptobox",
|
||||||
|
"domains": ["uptobox.com", "uptobox.fr"],
|
||||||
|
"icon": "📦",
|
||||||
|
"color": "#45b7d1"
|
||||||
|
},
|
||||||
|
"doodstream": {
|
||||||
|
"name": "Doodstream",
|
||||||
|
"domains": ["doodstream.com", "dood.stream", "dood.to", "dood.lol", "dood.cx", "dood.so", "dood.watch", "dood.sh"],
|
||||||
|
"icon": "🎥",
|
||||||
|
"color": "#f7b731"
|
||||||
|
},
|
||||||
|
"rapidfile": {
|
||||||
|
"name": "Rapidfile",
|
||||||
|
"domains": ["rapidfile.net", "rapidfile.com"],
|
||||||
|
"icon": "⚡",
|
||||||
|
"color": "#ff6b6b"
|
||||||
|
},
|
||||||
|
"vidmoly": {
|
||||||
|
"name": "VidMoly",
|
||||||
|
"domains": ["vidmoly.to", "vidmoly.org", "vidmoly.biz"],
|
||||||
|
"icon": "🎬",
|
||||||
|
"color": "#a29bfe"
|
||||||
|
},
|
||||||
|
"sendvid": {
|
||||||
|
"name": "SendVid",
|
||||||
|
"domains": ["sendvid.com", "sendvid.io"],
|
||||||
|
"icon": "📤",
|
||||||
|
"color": "#fd79a8"
|
||||||
|
},
|
||||||
|
"sibnet": {
|
||||||
|
"name": "Sibnet",
|
||||||
|
"domains": ["sibnet.ru", "video.sibnet.ru"],
|
||||||
|
"icon": "🎞️",
|
||||||
|
"color": "#00cec9"
|
||||||
|
},
|
||||||
|
"lpayer": {
|
||||||
|
"name": "Lplayer",
|
||||||
|
"domains": ["lpayer.embed4me.com", "lpayer.com", "lplayer.fr"],
|
||||||
|
"icon": "▶️",
|
||||||
|
"color": "#e17055"
|
||||||
|
},
|
||||||
|
"vidzy": {
|
||||||
|
"name": "Vidzy",
|
||||||
|
"domains": ["vidzy.com", "vidzy.net", "www.vidzy.com"],
|
||||||
|
"icon": "🎞️",
|
||||||
|
"color": "#74b9ff"
|
||||||
|
},
|
||||||
|
"luluv": {
|
||||||
|
"name": "LuLuvid",
|
||||||
|
"domains": ["luluv.com", "luluvid.com", "www.luluv.com", "www.luluvid.com"],
|
||||||
|
"icon": "🎬",
|
||||||
|
"color": "#a29bfe"
|
||||||
|
},
|
||||||
|
"uqload": {
|
||||||
|
"name": "Uqload",
|
||||||
|
"domains": ["uqload.bz", "uqload.com", "www.uqload.bz", "www.uqload.com"],
|
||||||
|
"icon": "📺",
|
||||||
|
"color": "#fd79a8"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_all_providers():
|
||||||
|
"""Get all supported providers (anime + series + file hosts)"""
|
||||||
|
return {**ANIME_PROVIDERS, **SERIES_PROVIDERS, **FILE_HOSTS}
|
||||||
|
|
||||||
|
def get_anime_providers():
|
||||||
|
"""Get all anime streaming providers"""
|
||||||
|
return ANIME_PROVIDERS
|
||||||
|
|
||||||
|
def get_series_providers():
|
||||||
|
"""Get all series streaming providers"""
|
||||||
|
return SERIES_PROVIDERS
|
||||||
|
|
||||||
|
def get_file_hosts():
|
||||||
|
"""Get all file hosting providers"""
|
||||||
|
return FILE_HOSTS
|
||||||
|
|
||||||
|
def detect_provider_from_url(url: str) -> str | None:
|
||||||
|
"""Detect which provider can handle the given URL"""
|
||||||
|
url_lower = url.lower()
|
||||||
|
|
||||||
|
for provider_id, provider in get_all_providers().items():
|
||||||
|
for domain in provider['domains']:
|
||||||
|
if domain in url_lower:
|
||||||
|
return provider_id
|
||||||
|
|
||||||
|
return None
|
||||||
@@ -0,0 +1,362 @@
|
|||||||
|
"""Generate personalized anime recommendations based on download history"""
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from collections import Counter
|
||||||
|
from typing import List, Dict, Set, Optional
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
import json
|
||||||
|
|
||||||
|
from app.recommendations import AnimeReleasesFetcher
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadAnalyzer:
|
||||||
|
"""Analyze download history to extract preferences"""
|
||||||
|
|
||||||
|
def __init__(self, download_dir: str = "downloads"):
|
||||||
|
self.download_dir = Path(download_dir)
|
||||||
|
self._history_cache = None
|
||||||
|
self._cache_time = None
|
||||||
|
self._cache_duration = timedelta(minutes=30)
|
||||||
|
|
||||||
|
def _parse_anime_name(self, filename: str) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Extract anime name from filename
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
"Naruto Shippuden - Episode 123.mp4" -> "Naruto Shippuden"
|
||||||
|
"One Piece S01E01.mkv" -> "One Piece"
|
||||||
|
"[FanSub] Demon Slayer - 05 [1080p].mp4" -> "Demon Slayer"
|
||||||
|
"""
|
||||||
|
# Remove extension
|
||||||
|
name = filename.rsplit('.', 1)[0] if '.' in filename else filename
|
||||||
|
|
||||||
|
# Remove common patterns
|
||||||
|
patterns_to_remove = [
|
||||||
|
r'\[.*?\]', # [Group], [1080p], etc.
|
||||||
|
r'\(.*?\)', # (Group), (Uncensored), etc.
|
||||||
|
r'[-_ ]?(E|Ep|Episode|Épisode)?[-_: ]?\d+', # Episode numbers
|
||||||
|
r'[-_ ]?S\d{2}E\d{2}', # S01E01 format
|
||||||
|
r'[-_ ]?(Saison|Season)[-_: ]?\d+', # Season indicators
|
||||||
|
r'[-_ ]?\d{3,4}p', # Quality (1080p, 720p)
|
||||||
|
r'[-_ ]?(VOSTFR|VF|MULTI|FR|SUB)', # Language tags
|
||||||
|
r'[-_ ]?(BD|BluRay|DVD|WEB)', # Source tags
|
||||||
|
r'[-_ ]?(x264|x265|H\.264|H\.265)', # Codec
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in patterns_to_remove:
|
||||||
|
name = re.sub(pattern, '', name, flags=re.IGNORECASE)
|
||||||
|
|
||||||
|
# Clean up
|
||||||
|
name = re.sub(r'[-_]+', ' ', name) # Replace hyphens/underscores with space
|
||||||
|
name = re.sub(r'\s+', ' ', name) # Multiple spaces to single space
|
||||||
|
name = name.strip()
|
||||||
|
|
||||||
|
# Only return if it looks like an anime name (has letters and reasonable length)
|
||||||
|
if len(name) >= 2 and any(c.isalpha() for c in name):
|
||||||
|
return name
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _extract_keywords(self, filename: str) -> Set[str]:
|
||||||
|
"""Extract potential genre/keyword indicators from filename"""
|
||||||
|
keywords = set()
|
||||||
|
|
||||||
|
# Common genre/keyword patterns in filenames
|
||||||
|
patterns = {
|
||||||
|
'action': r'(action|combat|fight)',
|
||||||
|
'adventure': r'(adventure|aventure)',
|
||||||
|
'comedy': r'(comedy|comédie|funny)',
|
||||||
|
'fantasy': r'(fantasy|fantastique|magie|magic)',
|
||||||
|
'romance': r'(romance|love|amour)',
|
||||||
|
'horror': r'(horror|horreur|scary)',
|
||||||
|
'sci-fi': r'(sci-fi|science\s*fiction|space|meccha)',
|
||||||
|
'slice_of_life': r'(slice\s*of\s*life|vie|school|lycée|école)',
|
||||||
|
'sports': r'(sport|football|basket|tennis)',
|
||||||
|
'supernatural': r'(supernatural|super naturel|power|pouvoir)',
|
||||||
|
'isekai': r'(isekai|another\s*world|reincarn|transport)',
|
||||||
|
'demon': r'(demon|devil|slime|ma.*ou)',
|
||||||
|
'game': r'(game|gaming|esport|rpg)',
|
||||||
|
}
|
||||||
|
|
||||||
|
filename_lower = filename.lower()
|
||||||
|
|
||||||
|
for keyword, pattern in patterns.items():
|
||||||
|
if re.search(pattern, filename_lower):
|
||||||
|
keywords.add(keyword)
|
||||||
|
|
||||||
|
return keywords
|
||||||
|
|
||||||
|
def analyze_downloads(self) -> Dict:
|
||||||
|
"""
|
||||||
|
Analyze download directory to extract preferences
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with:
|
||||||
|
- anime_list: List of downloaded anime names
|
||||||
|
- genres: Counter of extracted genres
|
||||||
|
- total_count: Total number of anime files
|
||||||
|
- recent: Most recently downloaded anime (last 10)
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
now = datetime.now()
|
||||||
|
|
||||||
|
# Check cache
|
||||||
|
if self._history_cache and self._cache_time:
|
||||||
|
if now - self._cache_time < self._cache_duration:
|
||||||
|
return self._history_cache
|
||||||
|
|
||||||
|
if not self.download_dir.exists():
|
||||||
|
logger.warning(f"Download directory does not exist: {self.download_dir}")
|
||||||
|
return {
|
||||||
|
'anime_list': [],
|
||||||
|
'genres': Counter(),
|
||||||
|
'total_count': 0,
|
||||||
|
'recent': []
|
||||||
|
}
|
||||||
|
|
||||||
|
video_extensions = {'.mp4', '.mkv', '.avi', '.mov', '.wmv', '.flv', '.webm'}
|
||||||
|
anime_names = []
|
||||||
|
all_genres = Counter()
|
||||||
|
files_with_dates = []
|
||||||
|
|
||||||
|
for file_path in self.download_dir.iterdir():
|
||||||
|
if file_path.is_file() and file_path.suffix.lower() in video_extensions:
|
||||||
|
filename = file_path.name
|
||||||
|
mtime = datetime.fromtimestamp(file_path.stat().st_mtime)
|
||||||
|
|
||||||
|
anime_name = self._parse_anime_name(filename)
|
||||||
|
if anime_name:
|
||||||
|
anime_names.append(anime_name)
|
||||||
|
genres = self._extract_keywords(filename)
|
||||||
|
all_genres.update(genres)
|
||||||
|
files_with_dates.append((anime_name, mtime, filename))
|
||||||
|
logger.debug(f"Found anime file: {filename} -> {anime_name}")
|
||||||
|
|
||||||
|
# Get recent downloads (last modified)
|
||||||
|
files_with_dates.sort(key=lambda x: x[1], reverse=True)
|
||||||
|
recent = [
|
||||||
|
{'name': name, 'date': date.isoformat(), 'filename': filename}
|
||||||
|
for name, date, filename in files_with_dates[:10]
|
||||||
|
]
|
||||||
|
|
||||||
|
result = {
|
||||||
|
'anime_list': anime_names,
|
||||||
|
'genres': all_genres,
|
||||||
|
'total_count': len(anime_names),
|
||||||
|
'recent': recent
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(f"Analyzed downloads: found {len(anime_names)} anime files, genres: {dict(all_genres.most_common(5))}")
|
||||||
|
|
||||||
|
# Update cache
|
||||||
|
self._history_cache = result
|
||||||
|
self._cache_time = now
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class RecommendationEngine:
|
||||||
|
"""Generate personalized anime recommendations"""
|
||||||
|
|
||||||
|
def __init__(self, download_dir: str = "downloads"):
|
||||||
|
self.analyzer = DownloadAnalyzer(download_dir)
|
||||||
|
self.fetcher = AnimeReleasesFetcher()
|
||||||
|
|
||||||
|
async def get_personalized_recommendations(self, limit: int = 15) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Get personalized recommendations based on download history
|
||||||
|
|
||||||
|
Strategy:
|
||||||
|
1. Analyze downloaded anime for genres and preferences
|
||||||
|
2. Search for similar anime using Jikan API
|
||||||
|
3. Get current season anime matching user's tastes
|
||||||
|
4. Rank by relevance and score
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Analyze download history
|
||||||
|
history = self.analyzer.analyze_downloads()
|
||||||
|
|
||||||
|
logger.info(f"Getting recommendations for user with {history['total_count']} downloaded anime")
|
||||||
|
|
||||||
|
if history['total_count'] == 0:
|
||||||
|
# No downloads yet, return top anime as fallback
|
||||||
|
logger.info("No downloads found, returning top anime")
|
||||||
|
try:
|
||||||
|
top_anime = await self.fetcher.get_top_anime(limit=limit)
|
||||||
|
if top_anime:
|
||||||
|
return top_anime
|
||||||
|
else:
|
||||||
|
logger.warning("Top anime API returned empty, using hardcoded fallback")
|
||||||
|
return self._get_fallback_recommendations()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching top anime: {e}, using fallback", exc_info=True)
|
||||||
|
return self._get_fallback_recommendations()
|
||||||
|
|
||||||
|
# Get top genres from user's downloads
|
||||||
|
top_genres = [genre for genre, count in history['genres'].most_common(5)]
|
||||||
|
|
||||||
|
# Get some downloaded anime names to search for similar
|
||||||
|
downloaded_anime = history['anime_list'][:5] if history['anime_list'] else []
|
||||||
|
|
||||||
|
recommendations = []
|
||||||
|
|
||||||
|
# Search for anime similar to what user downloaded
|
||||||
|
for anime_name in downloaded_anime[:3]:
|
||||||
|
try:
|
||||||
|
results = await self.fetcher.search_anime(anime_name, limit=5)
|
||||||
|
for anime in results:
|
||||||
|
# Skip if it's in user's downloads (case-insensitive check)
|
||||||
|
anime_lower = anime['title'].lower()
|
||||||
|
if not any(anime_lower == dl.lower() for dl in downloaded_anime):
|
||||||
|
recommendations.append({
|
||||||
|
**anime,
|
||||||
|
'recommendation_reason': f"Similaire à {anime_name}",
|
||||||
|
'relevance_score': 0.9
|
||||||
|
})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error searching for {anime_name}: {e}", exc_info=True)
|
||||||
|
|
||||||
|
# Get current season anime
|
||||||
|
try:
|
||||||
|
seasonal = await self.fetcher.get_seasonal_anime()
|
||||||
|
logger.info(f"Found {len(seasonal)} seasonal anime")
|
||||||
|
|
||||||
|
for anime in seasonal:
|
||||||
|
# Skip if already in recommendations or downloaded
|
||||||
|
anime_lower = anime['title'].lower()
|
||||||
|
if (anime_lower not in [r['title'].lower() for r in recommendations] and
|
||||||
|
not any(anime_lower == dl.lower() for dl in downloaded_anime)):
|
||||||
|
|
||||||
|
# Check if genres match user's preferences
|
||||||
|
anime_genres = [g.lower() for g in anime.get('genres', [])]
|
||||||
|
genre_match = any(g in anime_genres for g in top_genres)
|
||||||
|
|
||||||
|
recommendations.append({
|
||||||
|
**anime,
|
||||||
|
'recommendation_reason': 'Nouveau de la saison' + (' (vos genres!)' if genre_match else ''),
|
||||||
|
'relevance_score': 0.8 if genre_match else 0.6
|
||||||
|
})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching seasonal anime: {e}", exc_info=True)
|
||||||
|
|
||||||
|
# If still no recommendations, try top anime
|
||||||
|
if not recommendations:
|
||||||
|
logger.warning("No recommendations generated, trying top anime")
|
||||||
|
try:
|
||||||
|
recommendations = await self.fetcher.get_top_anime(limit=limit)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching top anime: {e}", exc_info=True)
|
||||||
|
recommendations = []
|
||||||
|
|
||||||
|
# If STILL no recommendations, use fallback
|
||||||
|
if not recommendations:
|
||||||
|
logger.warning("Still no recommendations, using hardcoded fallback")
|
||||||
|
recommendations = self._get_fallback_recommendations()
|
||||||
|
|
||||||
|
# Sort by relevance and score (handle None scores)
|
||||||
|
recommendations.sort(
|
||||||
|
key=lambda x: (x.get('relevance_score') or 0, x.get('score') or 0),
|
||||||
|
reverse=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# Remove duplicates by MAL ID
|
||||||
|
seen = set()
|
||||||
|
unique_recommendations = []
|
||||||
|
for rec in recommendations:
|
||||||
|
if rec.get('mal_id') not in seen:
|
||||||
|
seen.add(rec.get('mal_id'))
|
||||||
|
unique_recommendations.append(rec)
|
||||||
|
|
||||||
|
logger.info(f"Returning {len(unique_recommendations[:limit])} recommendations")
|
||||||
|
return unique_recommendations[:limit]
|
||||||
|
|
||||||
|
def _get_fallback_recommendations(self) -> List[Dict]:
|
||||||
|
"""Fallback hardcoded recommendations when API is unavailable"""
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
'title': 'Fullmetal Alchemist: Brotherhood',
|
||||||
|
'mal_id': 5114,
|
||||||
|
'score': 9.09,
|
||||||
|
'episodes': 64,
|
||||||
|
'status': 'Finished Airing',
|
||||||
|
'genres': ['Action', 'Adventure', 'Fantasy'],
|
||||||
|
'synopsis': 'Two brothers lose their mother to an incurable disease. With the power of alchemy, they use taboo knowledge to resurrect her. The process fails, and as a toll for crossing into the realm of God, they lose their bodies.',
|
||||||
|
'images': {},
|
||||||
|
'url': 'https://myanimelist.net/anime/5114/Fullmetal_Alchemist__Brotherhood',
|
||||||
|
'recommendation_reason': 'Un classique incontournable',
|
||||||
|
'relevance_score': 0.7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'title': 'Attack on Titan',
|
||||||
|
'mal_id': 16498,
|
||||||
|
'score': 8.51,
|
||||||
|
'episodes': 75,
|
||||||
|
'status': 'Finished Airing',
|
||||||
|
'genres': ['Action', 'Drama', 'Fantasy'],
|
||||||
|
'synopsis': 'Centuries ago, mankind was slaughtered to near extinction by monstrous humanoid creatures called titans. To protect what remains, humanity built walls and lived peacefully for a hundred years.',
|
||||||
|
'images': {},
|
||||||
|
'url': 'https://myanimelist.net/anime/16498/Shingeki_no_Kyojin',
|
||||||
|
'recommendation_reason': 'Shonen populaire',
|
||||||
|
'relevance_score': 0.7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'title': 'Death Note',
|
||||||
|
'mal_id': 21,
|
||||||
|
'score': 8.63,
|
||||||
|
'episodes': 37,
|
||||||
|
'status': 'Finished Airing',
|
||||||
|
'genres': ['Mystery', 'Police', 'Psychological'],
|
||||||
|
'synopsis': 'A shinigami, as a god of death, can kill any person—provided they see their victim\'s face and write their victim\'s name in a notebook called a Death Note.',
|
||||||
|
'images': {},
|
||||||
|
'url': 'https://myanimelist.net/anime/21/Death_Note',
|
||||||
|
'recommendation_reason': 'Un classique du genre',
|
||||||
|
'relevance_score': 0.7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'title': 'Demon Slayer',
|
||||||
|
'mal_id': 40028,
|
||||||
|
'score': 8.48,
|
||||||
|
'episodes': 26,
|
||||||
|
'status': 'Finished Airing',
|
||||||
|
'genres': ['Action', 'Adventure', 'Supernatural'],
|
||||||
|
'synopsis': 'It is the Taisho Period in Japan. Tanjiro, a kindhearted boy who sells charcoal for a living, finds his family slaughtered by a demon. To make matters worse, his younger sister Nezuko is turned into a demon.',
|
||||||
|
'images': {},
|
||||||
|
'url': 'https://myanimelist.net/anime/40028/Kimetsu_no_Yaiba',
|
||||||
|
'recommendation_reason': 'Animation exceptionnelle',
|
||||||
|
'relevance_score': 0.7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'title': 'Jujutsu Kaisen',
|
||||||
|
'mal_id': 38725,
|
||||||
|
'score': 8.35,
|
||||||
|
'episodes': 24,
|
||||||
|
'status': 'Finished Airing',
|
||||||
|
'genres': ['Action', 'Supernatural'],
|
||||||
|
'synopsis': 'Yuji Itadori is a boy with tremendous physical strength, though he lives a completely ordinary high school life. One day, to save a friend who has been attacked by curses, he eats the finger of a curse.',
|
||||||
|
'images': {},
|
||||||
|
'url': 'https://myanimelist.net/anime/38725/Jujutsu_Kaisen',
|
||||||
|
'recommendation_reason': 'Action intense',
|
||||||
|
'relevance_score': 0.7
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
async def get_download_stats(self) -> Dict:
|
||||||
|
"""Get statistics about user's downloads"""
|
||||||
|
history = self.analyzer.analyze_downloads()
|
||||||
|
|
||||||
|
return {
|
||||||
|
'total_anime': history['total_count'],
|
||||||
|
'top_genres': [
|
||||||
|
{'genre': genre, 'count': count}
|
||||||
|
for genre, count in history['genres'].most_common(10)
|
||||||
|
],
|
||||||
|
'recent_downloads': history['recent'][:5]
|
||||||
|
}
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
"""Close resources"""
|
||||||
|
await self.fetcher.close()
|
||||||
@@ -0,0 +1,437 @@
|
|||||||
|
"""Fetch latest anime releases from external APIs"""
|
||||||
|
import httpx
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import List, Dict, Optional
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class AnimeReleasesFetcher:
|
||||||
|
"""Fetch latest anime releases from Jikan (MAL) and other sources"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.jikan_base = "https://api.jikan.moe/v4"
|
||||||
|
self.client = httpx.AsyncClient(timeout=30.0, follow_redirects=True)
|
||||||
|
self._cache = {}
|
||||||
|
self._cache_time = {}
|
||||||
|
self._cache_duration = timedelta(hours=1) # Cache for 1 hour
|
||||||
|
self._last_request_time = None
|
||||||
|
self._min_request_interval = 0.5 # Minimum 500ms between requests
|
||||||
|
|
||||||
|
async def _rate_limited_request(self, url: str) -> httpx.Response:
|
||||||
|
"""Make a rate-limited request to Jikan API"""
|
||||||
|
# Enforce minimum delay between requests
|
||||||
|
if self._last_request_time:
|
||||||
|
elapsed = (datetime.now() - self._last_request_time).total_seconds()
|
||||||
|
if elapsed < self._min_request_interval:
|
||||||
|
await asyncio.sleep(self._min_request_interval - elapsed)
|
||||||
|
|
||||||
|
# Retry logic with exponential backoff
|
||||||
|
max_retries = 3
|
||||||
|
base_delay = 1.0
|
||||||
|
|
||||||
|
for attempt in range(max_retries):
|
||||||
|
try:
|
||||||
|
response = await self.client.get(url)
|
||||||
|
self._last_request_time = datetime.now()
|
||||||
|
|
||||||
|
# Handle rate limiting (HTTP 429)
|
||||||
|
if response.status_code == 429:
|
||||||
|
if attempt < max_retries - 1:
|
||||||
|
delay = base_delay * (2 ** attempt)
|
||||||
|
logger.warning(f"Rate limited by Jikan API, waiting {delay}s before retry {attempt + 1}/{max_retries}")
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
logger.error("Jikan API rate limit exceeded after all retries")
|
||||||
|
raise Exception(f"Jikan API rate limit exceeded after {max_retries} retries")
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
except httpx.TimeoutException as e:
|
||||||
|
if attempt < max_retries - 1:
|
||||||
|
delay = base_delay * (2 ** attempt)
|
||||||
|
logger.warning(f"Request timeout, retrying in {delay}s... (attempt {attempt + 1}/{max_retries})")
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
else:
|
||||||
|
raise Exception(f"Request timeout after {max_retries} retries") from e
|
||||||
|
except Exception as e:
|
||||||
|
# For any other exception, don't retry
|
||||||
|
raise
|
||||||
|
|
||||||
|
async def _get_cached(self, key: str, fetcher):
|
||||||
|
"""Get cached result or fetch new data"""
|
||||||
|
now = datetime.now()
|
||||||
|
|
||||||
|
if key in self._cache and key in self._cache_time:
|
||||||
|
if now - self._cache_time[key] < self._cache_duration:
|
||||||
|
return self._cache[key]
|
||||||
|
|
||||||
|
# Fetch new data
|
||||||
|
result = await fetcher()
|
||||||
|
self._cache[key] = result
|
||||||
|
self._cache_time[key] = now
|
||||||
|
return result
|
||||||
|
|
||||||
|
async def get_seasonal_anime(self, year: Optional[int] = None, season: Optional[str] = None) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Get current season anime from Jikan API
|
||||||
|
|
||||||
|
Args:
|
||||||
|
year: Year (defaults to current year)
|
||||||
|
season: Season (winter, spring, summer, fall)
|
||||||
|
"""
|
||||||
|
async def fetch():
|
||||||
|
nonlocal local_year, local_season
|
||||||
|
try:
|
||||||
|
url = f"{self.jikan_base}/seasons/{local_year}/{local_season}"
|
||||||
|
response = await self._rate_limited_request(url)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
anime_list = []
|
||||||
|
for anime in data.get('data', [])[:20]:
|
||||||
|
anime_list.append({
|
||||||
|
'title': anime.get('title', ''),
|
||||||
|
'title_japanese': anime.get('title_japanese', ''),
|
||||||
|
'episodes': anime.get('episodes'),
|
||||||
|
'status': anime.get('status', ''),
|
||||||
|
'rating': anime.get('rating', ''),
|
||||||
|
'score': anime.get('score', 0),
|
||||||
|
'genres': [g.get('name') for g in anime.get('genres', [])],
|
||||||
|
'synopsis': anime.get('synopsis', ''),
|
||||||
|
'images': anime.get('images', {}),
|
||||||
|
'url': anime.get('url', ''),
|
||||||
|
'mal_id': anime.get('mal_id')
|
||||||
|
})
|
||||||
|
|
||||||
|
return anime_list
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching seasonal anime: {e}", exc_info=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Initialize local variables
|
||||||
|
local_year = year if year else datetime.now().year
|
||||||
|
local_season = season
|
||||||
|
|
||||||
|
if not local_season:
|
||||||
|
month = datetime.now().month
|
||||||
|
if month in [12, 1, 2]:
|
||||||
|
local_season = "winter"
|
||||||
|
elif month in [3, 4, 5]:
|
||||||
|
local_season = "spring"
|
||||||
|
elif month in [6, 7, 8]:
|
||||||
|
local_season = "summer"
|
||||||
|
else:
|
||||||
|
local_season = "fall"
|
||||||
|
|
||||||
|
return await self._get_cached(f"seasonal_{local_year}_{local_season}", fetch)
|
||||||
|
|
||||||
|
async def get_scheduled_anime(self, day: Optional[str] = None) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Get anime scheduled for a specific day
|
||||||
|
|
||||||
|
Args:
|
||||||
|
day: Day of the week (monday, tuesday, etc.)
|
||||||
|
"""
|
||||||
|
async def fetch():
|
||||||
|
nonlocal local_day
|
||||||
|
try:
|
||||||
|
url = f"{self.jikan_base}/schedules/{local_day}"
|
||||||
|
response = await self._rate_limited_request(url)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
anime_list = []
|
||||||
|
for anime in data.get('data', [])[:15]:
|
||||||
|
anime_list.append({
|
||||||
|
'title': anime.get('title', ''),
|
||||||
|
'episodes': anime.get('episodes'),
|
||||||
|
'score': anime.get('score', 0),
|
||||||
|
'genres': [g.get('name') for g in anime.get('genres', [])],
|
||||||
|
'synopsis': anime.get('synopsis', ''),
|
||||||
|
'broadcast': anime.get('broadcast', {}),
|
||||||
|
'url': anime.get('url', ''),
|
||||||
|
'mal_id': anime.get('mal_id')
|
||||||
|
})
|
||||||
|
|
||||||
|
return anime_list
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching scheduled anime: {e}", exc_info=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Initialize local variable
|
||||||
|
local_day = day
|
||||||
|
if not local_day:
|
||||||
|
days = ['monday', 'tuesday', 'wednesday', 'thursday',
|
||||||
|
'friday', 'saturday', 'sunday']
|
||||||
|
local_day = days[datetime.now().weekday()]
|
||||||
|
|
||||||
|
return await self._get_cached(f"scheduled_{local_day}", fetch)
|
||||||
|
|
||||||
|
async def get_top_anime(self, type: str = "tv", limit: int = 15) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Get top anime
|
||||||
|
|
||||||
|
Args:
|
||||||
|
type: Type of anime (tv, movie, etc.)
|
||||||
|
limit: Number of results
|
||||||
|
"""
|
||||||
|
async def fetch():
|
||||||
|
try:
|
||||||
|
url = f"{self.jikan_base}/top/anime?type={type}&limit={limit}"
|
||||||
|
response = await self._rate_limited_request(url)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
anime_list = []
|
||||||
|
for anime in data.get('data', []):
|
||||||
|
anime_list.append({
|
||||||
|
'title': anime.get('title', ''),
|
||||||
|
'episodes': anime.get('episodes'),
|
||||||
|
'status': anime.get('status', ''),
|
||||||
|
'score': anime.get('score', 0),
|
||||||
|
'rank': anime.get('rank', 0),
|
||||||
|
'genres': [g.get('name') for g in anime.get('genres', [])],
|
||||||
|
'synopsis': anime.get('synopsis', ''),
|
||||||
|
'images': anime.get('images', {}),
|
||||||
|
'url': anime.get('url', ''),
|
||||||
|
'mal_id': anime.get('mal_id')
|
||||||
|
})
|
||||||
|
|
||||||
|
return anime_list
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching top anime: {e}", exc_info=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
return await self._get_cached(f"top_{type}_{limit}", fetch)
|
||||||
|
|
||||||
|
async def search_anime(self, query: str, limit: int = 10) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Search for anime by name
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query
|
||||||
|
limit: Number of results
|
||||||
|
"""
|
||||||
|
async def fetch():
|
||||||
|
try:
|
||||||
|
url = f"{self.jikan_base}/anime?q={query}&limit={limit}"
|
||||||
|
response = await self._rate_limited_request(url)
|
||||||
|
|
||||||
|
# Check HTTP status
|
||||||
|
if response.status_code != 200:
|
||||||
|
logger.error(f"Jikan API returned status {response.status_code} for query '{query}'")
|
||||||
|
return []
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
anime_list = []
|
||||||
|
for anime in data.get('data', []):
|
||||||
|
anime_list.append({
|
||||||
|
'title': anime.get('title', ''),
|
||||||
|
'episodes': anime.get('episodes'),
|
||||||
|
'status': anime.get('status', ''),
|
||||||
|
'score': anime.get('score', 0),
|
||||||
|
'genres': [g.get('name') for g in anime.get('genres', [])],
|
||||||
|
'synopsis': anime.get('synopsis', ''),
|
||||||
|
'images': anime.get('images', {}),
|
||||||
|
'url': anime.get('url', ''),
|
||||||
|
'mal_id': anime.get('mal_id')
|
||||||
|
})
|
||||||
|
|
||||||
|
return anime_list
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error searching anime for query '{query}': {e}", exc_info=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Don't cache searches
|
||||||
|
return await fetch()
|
||||||
|
|
||||||
|
async def get_anime_details(self, mal_id: int) -> Optional[Dict]:
|
||||||
|
"""
|
||||||
|
Get full details of an anime including related anime
|
||||||
|
|
||||||
|
Args:
|
||||||
|
mal_id: MyAnimeList ID of the anime
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with anime details and related anime
|
||||||
|
"""
|
||||||
|
async def fetch():
|
||||||
|
try:
|
||||||
|
# Get anime details
|
||||||
|
url = f"{self.jikan_base}/anime/{mal_id}/full"
|
||||||
|
response = await self._rate_limited_request(url)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
if 'data' not in data:
|
||||||
|
return None
|
||||||
|
|
||||||
|
anime = data['data']
|
||||||
|
|
||||||
|
# Extract basic info
|
||||||
|
anime_details = {
|
||||||
|
'mal_id': anime.get('mal_id'),
|
||||||
|
'title': anime.get('title'),
|
||||||
|
'title_japanese': anime.get('title_japanese'),
|
||||||
|
'title_english': anime.get('title_english'),
|
||||||
|
'episodes': anime.get('episodes'),
|
||||||
|
'status': anime.get('status'),
|
||||||
|
'rating': anime.get('rating'),
|
||||||
|
'score': anime.get('score'),
|
||||||
|
'scored_by': anime.get('scored_by'),
|
||||||
|
'rank': anime.get('rank'),
|
||||||
|
'popularity': anime.get('popularity'),
|
||||||
|
'members': anime.get('members'),
|
||||||
|
'favorites': anime.get('favorites'),
|
||||||
|
'synopsis': anime.get('synopsis', ''),
|
||||||
|
'background': anime.get('background', ''),
|
||||||
|
'genres': [g.get('name') for g in anime.get('genres', [])],
|
||||||
|
'themes': [t.get('name') for t in anime.get('themes', [])],
|
||||||
|
'studios': [s.get('name') for s in anime.get('studios', [])],
|
||||||
|
'producers': [p.get('name') for p in anime.get('producers', [])],
|
||||||
|
'source': anime.get('source'),
|
||||||
|
'duration': anime.get('duration'),
|
||||||
|
'season': anime.get('season'),
|
||||||
|
'year': anime.get('year'),
|
||||||
|
'broadcast': anime.get('broadcast', {}),
|
||||||
|
'images': anime.get('images', {}),
|
||||||
|
'trailer': anime.get('trailer', {}),
|
||||||
|
'url': anime.get('url', ''),
|
||||||
|
'related': []
|
||||||
|
}
|
||||||
|
|
||||||
|
# Extract related anime
|
||||||
|
relations = anime.get('relations', [])
|
||||||
|
|
||||||
|
# Collect MAL IDs that need title lookup
|
||||||
|
missing_titles = {}
|
||||||
|
|
||||||
|
for relation in relations:
|
||||||
|
for entry in relation.get('entry', []):
|
||||||
|
entry_mal_id = entry.get('mal_id')
|
||||||
|
title = entry.get('title')
|
||||||
|
|
||||||
|
if entry_mal_id and not title:
|
||||||
|
missing_titles[entry_mal_id] = None
|
||||||
|
|
||||||
|
# For better UX, extract title from URL when Jikan doesn't provide it
|
||||||
|
for relation in relations:
|
||||||
|
relation_type = relation.get('relation', '')
|
||||||
|
related_entries = []
|
||||||
|
|
||||||
|
for entry in relation.get('entry', []):
|
||||||
|
entry_mal_id = entry.get('mal_id')
|
||||||
|
entry_title = entry.get('title')
|
||||||
|
entry_url = entry.get('url')
|
||||||
|
|
||||||
|
# Jikan API sometimes returns null for title
|
||||||
|
if not entry_title and entry_mal_id:
|
||||||
|
# Try to extract title from URL
|
||||||
|
if entry_url:
|
||||||
|
# URL format: https://myanimelist.net/anime/194/Macross_Zero
|
||||||
|
# Extract the slug and convert to readable title
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
path = urlparse(entry_url).path
|
||||||
|
# path = /anime/194/Macross_Zero
|
||||||
|
parts = path.strip('/').split('/')
|
||||||
|
if len(parts) >= 3:
|
||||||
|
slug = parts[2]
|
||||||
|
# Convert slug to title: Macross_Zero -> Macross Zero
|
||||||
|
entry_title = slug.replace('_', ' ').replace('-', ' ')
|
||||||
|
else:
|
||||||
|
entry_title = f"Anime #{entry_mal_id}"
|
||||||
|
else:
|
||||||
|
# Construct URL and use ID as title
|
||||||
|
entry_url = f"https://myanimelist.net/anime/{entry_mal_id}"
|
||||||
|
entry_title = f"Anime #{entry_mal_id}"
|
||||||
|
|
||||||
|
# Construct URL if not provided
|
||||||
|
if not entry_url and entry_mal_id:
|
||||||
|
entry_url = f"https://myanimelist.net/anime/{entry_mal_id}"
|
||||||
|
|
||||||
|
related_entries.append({
|
||||||
|
'mal_id': entry_mal_id,
|
||||||
|
'title': entry_title,
|
||||||
|
'type': entry.get('type'),
|
||||||
|
'url': entry_url
|
||||||
|
})
|
||||||
|
|
||||||
|
if related_entries:
|
||||||
|
anime_details['related'].append({
|
||||||
|
'type': relation_type,
|
||||||
|
'entries': related_entries
|
||||||
|
})
|
||||||
|
|
||||||
|
return anime_details
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching anime details for MAL ID {mal_id}: {e}", exc_info=True)
|
||||||
|
return None
|
||||||
|
|
||||||
|
return await self._get_cached(f"anime_details_{mal_id}", fetch)
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
"""Close the HTTP client"""
|
||||||
|
await self.client.aclose()
|
||||||
|
|
||||||
|
|
||||||
|
async def get_latest_releases_with_info(limit: int = 20) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Get latest anime releases with detailed information
|
||||||
|
|
||||||
|
Combines seasonal anime and scheduled anime for current week
|
||||||
|
"""
|
||||||
|
fetcher = AnimeReleasesFetcher()
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Get current season anime
|
||||||
|
seasonal = await fetcher.get_seasonal_anime()
|
||||||
|
logger.info(f"Found {len(seasonal)} seasonal anime")
|
||||||
|
|
||||||
|
# Get anime scheduled for today
|
||||||
|
scheduled = await fetcher.get_scheduled_anime()
|
||||||
|
logger.info(f"Found {len(scheduled)} scheduled anime")
|
||||||
|
|
||||||
|
# Combine and deduplicate
|
||||||
|
all_anime = {}
|
||||||
|
|
||||||
|
for anime in seasonal:
|
||||||
|
all_anime[anime['mal_id']] = {
|
||||||
|
**anime,
|
||||||
|
'source': 'seasonal',
|
||||||
|
'release_type': 'current_season'
|
||||||
|
}
|
||||||
|
|
||||||
|
for anime in scheduled:
|
||||||
|
if anime['mal_id'] not in all_anime:
|
||||||
|
all_anime[anime['mal_id']] = {
|
||||||
|
**anime,
|
||||||
|
'source': 'scheduled',
|
||||||
|
'release_type': 'weekly_schedule'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Convert to list and sort by score (handle None scores)
|
||||||
|
releases = sorted(
|
||||||
|
all_anime.values(),
|
||||||
|
key=lambda x: x.get('score') or 0,
|
||||||
|
reverse=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# If no releases found, try top anime as fallback
|
||||||
|
if not releases:
|
||||||
|
logger.warning("No releases found, trying top anime")
|
||||||
|
releases = await fetcher.get_top_anime(limit=limit)
|
||||||
|
|
||||||
|
return releases[:limit]
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting latest releases: {e}", exc_info=True)
|
||||||
|
# Return empty list on error
|
||||||
|
return []
|
||||||
|
finally:
|
||||||
|
await fetcher.close()
|
||||||
@@ -1,243 +0,0 @@
|
|||||||
"""Administration : utilisateurs, activation des sources, santé, mises à jour."""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Request
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
from app import auth
|
|
||||||
from app.db import db
|
|
||||||
from app.routers.auth import AdminUser
|
|
||||||
from app.scrapers.base import (
|
|
||||||
ScrapeError,
|
|
||||||
SourceScraper,
|
|
||||||
all_sources,
|
|
||||||
get_source,
|
|
||||||
import_all_scrapers,
|
|
||||||
)
|
|
||||||
from app.services.discover import discover
|
|
||||||
from app.services.settings import (
|
|
||||||
get_sonarr_config,
|
|
||||||
get_source_health,
|
|
||||||
get_torznab_apikey,
|
|
||||||
is_source_enabled,
|
|
||||||
reset_torznab_apikey,
|
|
||||||
set_sonarr_config,
|
|
||||||
set_source_base_url,
|
|
||||||
set_source_enabled,
|
|
||||||
set_source_health,
|
|
||||||
)
|
|
||||||
from app.services.sonarr import sonarr
|
|
||||||
from app.services.update import UpdateError, fetch_latest_version, trigger_update
|
|
||||||
from app.services.update import status as update_status
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
|
||||||
|
|
||||||
import_all_scrapers()
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- utilisateurs
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/users")
|
|
||||||
async def list_users(admin: AdminUser) -> list[dict]:
|
|
||||||
rows = await db.fetchall(
|
|
||||||
"SELECT id, username, is_admin, is_active, created_at FROM users ORDER BY id"
|
|
||||||
)
|
|
||||||
return [dict(row) for row in rows]
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/users/{user_id}/toggle-active")
|
|
||||||
async def toggle_active(user_id: int, admin: AdminUser) -> dict:
|
|
||||||
if user_id == admin.id:
|
|
||||||
raise HTTPException(400, "Impossible de désactiver son propre compte")
|
|
||||||
row = await db.fetchone("SELECT is_active FROM users WHERE id = ?", (user_id,))
|
|
||||||
if row is None:
|
|
||||||
raise HTTPException(404, "Utilisateur introuvable")
|
|
||||||
new_state = 0 if row["is_active"] else 1
|
|
||||||
await db.execute("UPDATE users SET is_active = ? WHERE id = ?", (new_state, user_id))
|
|
||||||
if not new_state:
|
|
||||||
await auth.revoke_all_refresh_tokens(user_id)
|
|
||||||
return {"is_active": bool(new_state)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/users/{user_id}/toggle-admin")
|
|
||||||
async def toggle_admin(user_id: int, admin: AdminUser) -> dict:
|
|
||||||
if user_id == admin.id:
|
|
||||||
raise HTTPException(400, "Impossible de modifier ses propres droits")
|
|
||||||
row = await db.fetchone("SELECT is_admin FROM users WHERE id = ?", (user_id,))
|
|
||||||
if row is None:
|
|
||||||
raise HTTPException(404, "Utilisateur introuvable")
|
|
||||||
new_state = 0 if row["is_admin"] else 1
|
|
||||||
await db.execute("UPDATE users SET is_admin = ? WHERE id = ?", (new_state, user_id))
|
|
||||||
return {"is_admin": bool(new_state)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/users/{user_id}")
|
|
||||||
async def delete_user(user_id: int, admin: AdminUser) -> dict:
|
|
||||||
if user_id == admin.id:
|
|
||||||
raise HTTPException(400, "Impossible de supprimer son propre compte")
|
|
||||||
cursor = await db.execute("DELETE FROM users WHERE id = ?", (user_id,))
|
|
||||||
if cursor.rowcount == 0:
|
|
||||||
raise HTTPException(404, "Utilisateur introuvable")
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/stats")
|
|
||||||
async def stats(admin: AdminUser) -> dict:
|
|
||||||
users = await db.fetchone("SELECT COUNT(*) AS n FROM users")
|
|
||||||
downloads = await db.fetchall("SELECT status, COUNT(*) AS n FROM downloads GROUP BY status")
|
|
||||||
return {
|
|
||||||
"users": users["n"],
|
|
||||||
"downloads": {row["status"]: row["n"] for row in downloads},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- sources
|
|
||||||
|
|
||||||
|
|
||||||
class SourceToggle(BaseModel):
|
|
||||||
enabled: bool
|
|
||||||
|
|
||||||
|
|
||||||
def _source_or_404(name: str) -> SourceScraper:
|
|
||||||
try:
|
|
||||||
return get_source(name)
|
|
||||||
except ScrapeError as exc:
|
|
||||||
raise HTTPException(404, str(exc)) from exc
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/sources/{name}/toggle")
|
|
||||||
async def toggle_source(name: str, payload: SourceToggle, admin: AdminUser) -> dict:
|
|
||||||
_source_or_404(name)
|
|
||||||
await set_source_enabled(name, payload.enabled)
|
|
||||||
return {"name": name, "enabled": payload.enabled}
|
|
||||||
|
|
||||||
class SourceUrlUpdate(BaseModel):
|
|
||||||
url: str
|
|
||||||
|
|
||||||
|
|
||||||
@router.put("/sources/{name}/url")
|
|
||||||
async def update_source_url(name: str, payload: SourceUrlUpdate, admin: AdminUser) -> dict:
|
|
||||||
"""Change l'URL d'une source (ex. le site a changé de domaine) ; vide = défaut."""
|
|
||||||
source = _source_or_404(name)
|
|
||||||
default = type(source).base_url
|
|
||||||
url = payload.url.strip().rstrip("/")
|
|
||||||
if url and url != default:
|
|
||||||
parsed = urlparse(url)
|
|
||||||
if parsed.scheme not in ("http", "https") or not parsed.netloc:
|
|
||||||
raise HTTPException(422, "URL invalide — format attendu : https://domaine.tld")
|
|
||||||
await set_source_base_url(name, url)
|
|
||||||
else:
|
|
||||||
url = default
|
|
||||||
await set_source_base_url(name, None)
|
|
||||||
source.base_url = url
|
|
||||||
logger.info("URL de la source %s : %s", name, url)
|
|
||||||
return {
|
|
||||||
"name": name,
|
|
||||||
"base_url": url,
|
|
||||||
"default_base_url": default,
|
|
||||||
"overridden": url != default,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/sources/{name}/health")
|
|
||||||
async def health_check(name: str, admin: AdminUser) -> dict:
|
|
||||||
"""Test de santé manuel : la source doit répondre à une recherche simple."""
|
|
||||||
source = _source_or_404(name)
|
|
||||||
try:
|
|
||||||
results = await asyncio.wait_for(source.search("naruto"), timeout=30)
|
|
||||||
healthy, detail = len(results) > 0, f"{len(results)} résultats"
|
|
||||||
except (ScrapeError, TimeoutError) as exc:
|
|
||||||
healthy, detail = False, str(exc)[:200]
|
|
||||||
logger.error("Health check %s KO : %s", name, exc)
|
|
||||||
except Exception as exc:
|
|
||||||
healthy, detail = False, f"Erreur inattendue : {exc}"[:200]
|
|
||||||
logger.exception("Health check %s : erreur inattendue", name)
|
|
||||||
state = await set_source_health(name, healthy, detail)
|
|
||||||
return {"name": name, **state}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/sources")
|
|
||||||
async def sources_status(admin: AdminUser) -> list[dict]:
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"name": s.name,
|
|
||||||
"label": s.label,
|
|
||||||
"base_url": s.base_url,
|
|
||||||
"default_base_url": type(s).base_url,
|
|
||||||
"overridden": s.base_url != type(s).base_url,
|
|
||||||
"enabled": await is_source_enabled(s.name),
|
|
||||||
"health": await get_source_health(s.name),
|
|
||||||
}
|
|
||||||
for s in all_sources()
|
|
||||||
]
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- intégrations *arr
|
|
||||||
|
|
||||||
|
|
||||||
class SonarrConfig(BaseModel):
|
|
||||||
url: str
|
|
||||||
apikey: str
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/integrations")
|
|
||||||
async def integrations(admin: AdminUser, request: Request) -> dict:
|
|
||||||
"""Configuration Torznab (indexeur) et Sonarr (recommandations)."""
|
|
||||||
config = await get_sonarr_config()
|
|
||||||
base = str(request.base_url).rstrip("/")
|
|
||||||
return {
|
|
||||||
"torznab": {
|
|
||||||
"apikey": await get_torznab_apikey(),
|
|
||||||
"endpoint": f"{base}/torznab/api",
|
|
||||||
},
|
|
||||||
"sonarr": config,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/integrations/torznab/regenerate")
|
|
||||||
async def regenerate_torznab_key(admin: AdminUser) -> dict:
|
|
||||||
key = await reset_torznab_apikey()
|
|
||||||
return {"apikey": key}
|
|
||||||
|
|
||||||
|
|
||||||
@router.put("/integrations/sonarr")
|
|
||||||
async def save_sonarr(payload: SonarrConfig, admin: AdminUser) -> dict:
|
|
||||||
"""Enregistre la connexion Sonarr et recalcule les recommandations."""
|
|
||||||
await set_sonarr_config(payload.url, payload.apikey)
|
|
||||||
sonarr.invalidate()
|
|
||||||
discover.invalidate_for_you()
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/integrations/sonarr/test")
|
|
||||||
async def test_sonarr(admin: AdminUser) -> dict:
|
|
||||||
return await sonarr.test_connection()
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- mise à jour logicielle
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/update")
|
|
||||||
async def get_update(admin: AdminUser) -> dict:
|
|
||||||
"""Version courante, dernière version disponible et patchnote."""
|
|
||||||
return await update_status()
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/update/check")
|
|
||||||
async def check_update(admin: AdminUser) -> dict:
|
|
||||||
"""Force la re-vérification de la dernière version (ignore le cache)."""
|
|
||||||
await fetch_latest_version(force=True)
|
|
||||||
return await update_status()
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/update/apply")
|
|
||||||
async def apply_update(admin: AdminUser) -> dict:
|
|
||||||
"""Déclenche la mise à jour via Watchtower (le conteneur est recréé)."""
|
|
||||||
try:
|
|
||||||
return await trigger_update()
|
|
||||||
except UpdateError as exc:
|
|
||||||
raise HTTPException(502, str(exc)) from exc
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
"""Routes d'authentification (cookies httponly, adaptées à l'UI htmx)."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from typing import Annotated
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Body, Depends, Form, HTTPException, Request, Response, status
|
|
||||||
from fastapi.responses import RedirectResponse
|
|
||||||
|
|
||||||
from app import auth
|
|
||||||
from app.auth import User
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/auth", tags=["auth"])
|
|
||||||
|
|
||||||
ACCESS_COOKIE = "ohm_access"
|
|
||||||
REFRESH_COOKIE = "ohm_refresh"
|
|
||||||
|
|
||||||
|
|
||||||
def set_auth_cookies(response: Response, user: User, refresh_token: str) -> None:
|
|
||||||
response.set_cookie(
|
|
||||||
ACCESS_COOKIE, auth.create_access_token(user), httponly=True, samesite="lax"
|
|
||||||
)
|
|
||||||
response.set_cookie(REFRESH_COOKIE, refresh_token, httponly=True, samesite="lax", path="/")
|
|
||||||
|
|
||||||
|
|
||||||
def clear_auth_cookies(response: Response) -> None:
|
|
||||||
response.delete_cookie(ACCESS_COOKIE)
|
|
||||||
response.delete_cookie(REFRESH_COOKIE, path="/")
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- dépendances
|
|
||||||
|
|
||||||
|
|
||||||
async def current_user(request: Request, response: Response) -> User:
|
|
||||||
"""Utilisateur courant via cookie d'accès ; tente un refresh si expiré."""
|
|
||||||
token = request.cookies.get(ACCESS_COOKIE)
|
|
||||||
if token:
|
|
||||||
payload = auth.decode_access_token(token)
|
|
||||||
if payload:
|
|
||||||
user = await auth.get_user(int(payload["sub"]))
|
|
||||||
if user:
|
|
||||||
return user
|
|
||||||
|
|
||||||
refresh = request.cookies.get(REFRESH_COOKIE)
|
|
||||||
if refresh:
|
|
||||||
user = await auth.use_refresh_token(refresh)
|
|
||||||
if user:
|
|
||||||
new_refresh = await auth.create_refresh_token(user.id)
|
|
||||||
set_auth_cookies(response, user, new_refresh)
|
|
||||||
return user
|
|
||||||
|
|
||||||
raise HTTPException(status.HTTP_303_SEE_OTHER, headers={"Location": "/login"})
|
|
||||||
|
|
||||||
|
|
||||||
async def require_admin(user: Annotated[User, Depends(current_user)]) -> User:
|
|
||||||
if not user.is_admin:
|
|
||||||
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="Droits administrateur requis")
|
|
||||||
return user
|
|
||||||
|
|
||||||
|
|
||||||
CurrentUser = Annotated[User, Depends(current_user)]
|
|
||||||
AdminUser = Annotated[User, Depends(require_admin)]
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- routes
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/register")
|
|
||||||
async def register(
|
|
||||||
response: Response,
|
|
||||||
username: Annotated[str, Form(min_length=3, max_length=32)],
|
|
||||||
password: Annotated[str, Form(min_length=6)],
|
|
||||||
) -> RedirectResponse:
|
|
||||||
row = await auth.db.fetchone("SELECT id FROM users WHERE username = ?", (username.strip(),))
|
|
||||||
if row is not None:
|
|
||||||
raise HTTPException(status.HTTP_409_CONFLICT, detail="Nom d'utilisateur déjà pris")
|
|
||||||
user = await auth.create_user(username, password)
|
|
||||||
refresh = await auth.create_refresh_token(user.id)
|
|
||||||
redirect = RedirectResponse("/", status.HTTP_303_SEE_OTHER)
|
|
||||||
set_auth_cookies(redirect, user, refresh)
|
|
||||||
return redirect
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/login")
|
|
||||||
async def login(
|
|
||||||
response: Response,
|
|
||||||
username: Annotated[str, Form()],
|
|
||||||
password: Annotated[str, Form()],
|
|
||||||
) -> RedirectResponse:
|
|
||||||
user = await auth.authenticate(username, password)
|
|
||||||
if user is None:
|
|
||||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED, detail="Identifiants invalides")
|
|
||||||
refresh = await auth.create_refresh_token(user.id)
|
|
||||||
redirect = RedirectResponse("/", status.HTTP_303_SEE_OTHER)
|
|
||||||
set_auth_cookies(redirect, user, refresh)
|
|
||||||
return redirect
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/logout")
|
|
||||||
async def logout(user: CurrentUser) -> RedirectResponse:
|
|
||||||
await auth.revoke_all_refresh_tokens(user.id)
|
|
||||||
redirect = RedirectResponse("/login", status.HTTP_303_SEE_OTHER)
|
|
||||||
clear_auth_cookies(redirect)
|
|
||||||
return redirect
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/me")
|
|
||||||
async def me(user: CurrentUser) -> dict:
|
|
||||||
return {
|
|
||||||
"id": user.id,
|
|
||||||
"username": user.username,
|
|
||||||
"is_admin": user.is_admin,
|
|
||||||
"content_preference": user.content_preference,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.put("/preferences")
|
|
||||||
async def update_preferences(
|
|
||||||
user: CurrentUser,
|
|
||||||
content_preference: Annotated[str, Body(embed=True)],
|
|
||||||
) -> dict:
|
|
||||||
"""Préférence de contenus : animés, séries, ou les deux."""
|
|
||||||
try:
|
|
||||||
await auth.set_content_preference(user.id, content_preference)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise HTTPException(422, detail=str(exc)) from exc
|
|
||||||
return {"content_preference": content_preference}
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
"""Découverte : nouveautés par type, incontournables, recommandations, exploration."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from typing import Annotated
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Query
|
|
||||||
|
|
||||||
from app.routers.auth import CurrentUser, current_user
|
|
||||||
from app.routers.search import _allowed_media_types
|
|
||||||
from app.scrapers.base import ScrapeError, get_source
|
|
||||||
from app.services.discover import ANIME_GENRES, discover
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api", tags=["discover"], dependencies=[Depends(current_user)])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/discover")
|
|
||||||
async def get_discover(
|
|
||||||
user: CurrentUser,
|
|
||||||
latest_limit: Annotated[int, Query(ge=1, le=50)] = 24,
|
|
||||||
must_watch_limit: Annotated[int, Query(ge=1, le=20)] = 20,
|
|
||||||
for_you_limit: Annotated[int, Query(ge=1, le=20)] = 20,
|
|
||||||
) -> dict:
|
|
||||||
"""Rails de découverte en une requête (sections vides si source KO).
|
|
||||||
|
|
||||||
« latest_anime » et « latest_serie » (séries + films réels) sont des rails
|
|
||||||
indépendants, filtrés par la préférence de contenu du compte.
|
|
||||||
"""
|
|
||||||
allowed = _allowed_media_types(user.content_preference)
|
|
||||||
rails = await discover.latest_by_type(latest_limit)
|
|
||||||
latest_anime = rails["anime"] if "anime" in allowed else []
|
|
||||||
latest_serie = (rails["serie"] + rails["film"]) if {"serie", "film"} & allowed else []
|
|
||||||
# Incontournables et Pour toi sont issus de Kitsu (catalogue animés) :
|
|
||||||
# en mode séries, elles n'ont pas de sens — on ne les calcule même pas.
|
|
||||||
if "anime" in allowed:
|
|
||||||
must_watch = await discover.must_watch(must_watch_limit)
|
|
||||||
for_you = await discover.for_you(user.id, for_you_limit)
|
|
||||||
else:
|
|
||||||
must_watch = []
|
|
||||||
for_you = {"based_on": [], "items": []}
|
|
||||||
return {
|
|
||||||
"latest_anime": latest_anime,
|
|
||||||
"latest_serie": latest_serie,
|
|
||||||
"must_watch": must_watch,
|
|
||||||
"for_you": for_you,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/discover/genres")
|
|
||||||
async def get_genres(user: CurrentUser) -> dict:
|
|
||||||
"""Catalogue des genres parcourables (page Explorer), selon la préférence."""
|
|
||||||
allowed = _allowed_media_types(user.content_preference)
|
|
||||||
catalog: dict[str, list[dict]] = {}
|
|
||||||
if "anime" in allowed:
|
|
||||||
catalog["anime"] = [{"key": key, "label": label} for key, label in ANIME_GENRES.items()]
|
|
||||||
try:
|
|
||||||
french_stream = get_source("french_stream")
|
|
||||||
available = french_stream.browse_catalog()
|
|
||||||
except ScrapeError:
|
|
||||||
available = {}
|
|
||||||
for kind in ("serie", "film"):
|
|
||||||
if kind in allowed and kind in available:
|
|
||||||
catalog[kind] = [
|
|
||||||
{"key": key, "label": label} for key, label in available[kind].items()
|
|
||||||
]
|
|
||||||
return catalog
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/discover/browse")
|
|
||||||
async def get_browse(
|
|
||||||
user: CurrentUser,
|
|
||||||
type: Annotated[str, Query(pattern="^(anime|serie|film)$")],
|
|
||||||
genre: Annotated[str, Query(min_length=1, max_length=40)],
|
|
||||||
limit: Annotated[int, Query(ge=1, le=40)] = 24,
|
|
||||||
) -> dict:
|
|
||||||
"""Titres d'un genre : animés via Kitsu, séries/films via French-Stream."""
|
|
||||||
if type not in _allowed_media_types(user.content_preference):
|
|
||||||
return {"items": []} # hors préférence du compte
|
|
||||||
if type == "anime":
|
|
||||||
return {"items": await discover.browse_anime(genre, limit)}
|
|
||||||
return {"items": await discover.browse_serie_film(type, genre, limit)}
|
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
"""Gestion de la file de téléchargements + progression temps réel (SSE)."""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from sse_starlette.sse import EventSourceResponse
|
|
||||||
|
|
||||||
from app.routers.auth import current_user
|
|
||||||
from app.scrapers.base import decode_internal_url
|
|
||||||
from app.services.downloads import download_manager
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(
|
|
||||||
prefix="/api/downloads", tags=["downloads"], dependencies=[Depends(current_user)]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class EnqueueRequest(BaseModel):
|
|
||||||
internal_url: str # format `video_url|page_url|titre`
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
|
||||||
async def list_downloads() -> list[dict]:
|
|
||||||
return await download_manager.list_all()
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("", status_code=201)
|
|
||||||
async def enqueue(payload: EnqueueRequest) -> dict:
|
|
||||||
try:
|
|
||||||
video_url, page_url, title = decode_internal_url(payload.internal_url)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise HTTPException(422, detail=str(exc)) from exc
|
|
||||||
return await download_manager.enqueue(video_url, page_url, title)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{download_id}/pause")
|
|
||||||
async def pause(download_id: int) -> dict:
|
|
||||||
await download_manager.pause(download_id)
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{download_id}/resume")
|
|
||||||
async def resume(download_id: int) -> dict:
|
|
||||||
if not await download_manager.resume(download_id):
|
|
||||||
raise HTTPException(409, detail="Ce téléchargement n'est pas en pause")
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{download_id}/retry")
|
|
||||||
async def retry(download_id: int) -> dict:
|
|
||||||
if not await download_manager.retry(download_id):
|
|
||||||
raise HTTPException(
|
|
||||||
409, detail="Seules les tâches en échec/annulées peuvent être relancées"
|
|
||||||
)
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{download_id}/cancel")
|
|
||||||
async def cancel(download_id: int) -> dict:
|
|
||||||
await download_manager.cancel(download_id)
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
@router.delete("/{download_id}")
|
|
||||||
async def delete_download(download_id: int, delete_file: bool = False) -> dict:
|
|
||||||
"""Retire une tâche de la file ; delete_file=true efface aussi le fichier."""
|
|
||||||
if not await download_manager.delete(download_id, delete_file=delete_file):
|
|
||||||
raise HTTPException(404, detail="Téléchargement introuvable")
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/cancel-all")
|
|
||||||
async def cancel_all() -> dict:
|
|
||||||
return {"cancelled": await download_manager.cancel_all()}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/clear-finished")
|
|
||||||
async def clear_finished() -> dict:
|
|
||||||
return {"removed": await download_manager.clear_finished()}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/events")
|
|
||||||
async def events() -> EventSourceResponse:
|
|
||||||
"""Flux SSE : progression de tous les téléchargements en temps réel."""
|
|
||||||
|
|
||||||
async def stream() -> AsyncIterator[dict]:
|
|
||||||
yield {"data": json.dumps({"type": "snapshot", "items": await download_manager.list_all()})}
|
|
||||||
async for message in download_manager.subscribe():
|
|
||||||
yield {"data": json.dumps(message, default=str)}
|
|
||||||
|
|
||||||
return EventSourceResponse(stream())
|
|
||||||
@@ -1,201 +0,0 @@
|
|||||||
"""Bibliothèque locale, streaming de fichiers (range requests) et favoris."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import mimetypes
|
|
||||||
import re
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
|
|
||||||
import aiosqlite
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
||||||
from fastapi.responses import StreamingResponse
|
|
||||||
from pydantic import BaseModel, Field
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.db import db
|
|
||||||
from app.routers.auth import CurrentUser, current_user
|
|
||||||
from app.services.downloads import download_manager
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api", tags=["library"], dependencies=[Depends(current_user)])
|
|
||||||
|
|
||||||
CHUNK_SIZE = 1 << 20 # 1 Mio
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- bibliothèque
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/library")
|
|
||||||
async def library(user: CurrentUser) -> list[dict]:
|
|
||||||
"""Fichiers téléchargés, streamables, avec progression de visionnage."""
|
|
||||||
rows = await db.fetchall(
|
|
||||||
"SELECT d.*, wp.position_seconds FROM downloads d "
|
|
||||||
"LEFT JOIN watch_progress wp ON wp.download_id = d.id AND wp.user_id = ? "
|
|
||||||
"WHERE d.status = 'done' ORDER BY d.updated_at DESC",
|
|
||||||
(user.id,),
|
|
||||||
)
|
|
||||||
return [dict(row) for row in rows]
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/library/{download_id}/neighbors")
|
|
||||||
async def neighbors(download_id: int) -> dict:
|
|
||||||
"""Épisode précédent/suivant : heuristique sur les titres (même série, N±1)."""
|
|
||||||
rows = await db.fetchall("SELECT id, title FROM downloads WHERE status = 'done' ORDER BY title")
|
|
||||||
|
|
||||||
def parse(title: str) -> tuple[str, float | None]:
|
|
||||||
match = re.search(r"(?:épisode|episode|ep|e)\s*(\d+(?:\.\d+)?)", title, re.IGNORECASE)
|
|
||||||
if not match:
|
|
||||||
return title, None
|
|
||||||
return title[: match.start()].strip(), float(match.group(1))
|
|
||||||
|
|
||||||
current = await db.fetchone("SELECT id, title FROM downloads WHERE id = ?", (download_id,))
|
|
||||||
if current is None:
|
|
||||||
raise HTTPException(404, "Fichier introuvable")
|
|
||||||
base, number = parse(current["title"])
|
|
||||||
prev_ep = next_ep = None
|
|
||||||
for row in rows:
|
|
||||||
other_base, other_num = parse(row["title"])
|
|
||||||
if other_base != base or other_num is None or row["id"] == download_id:
|
|
||||||
continue
|
|
||||||
if number is not None and other_num == number - 1:
|
|
||||||
prev_ep = row["id"]
|
|
||||||
if number is not None and other_num == number + 1:
|
|
||||||
next_ep = row["id"]
|
|
||||||
return {"previous": prev_ep, "next": next_ep}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- streaming
|
|
||||||
|
|
||||||
_RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)")
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/stream/{download_id}")
|
|
||||||
async def stream(download_id: int, request: Request) -> StreamingResponse:
|
|
||||||
"""Streaming d'un fichier local avec support des requêtes Range (206)."""
|
|
||||||
row = await db.fetchone(
|
|
||||||
"SELECT file_path FROM downloads WHERE id = ? AND status = 'done'", (download_id,)
|
|
||||||
)
|
|
||||||
if row is None:
|
|
||||||
raise HTTPException(404, "Fichier introuvable ou téléchargement incomplet")
|
|
||||||
path = get_settings().download_dir / row["file_path"]
|
|
||||||
if not path.is_file():
|
|
||||||
logger.error("Fichier manquant sur disque : %s", path)
|
|
||||||
raise HTTPException(404, "Fichier absent du disque")
|
|
||||||
|
|
||||||
size = path.stat().st_size
|
|
||||||
content_type = mimetypes.guess_type(path.name)[0] or "video/mp4"
|
|
||||||
start, end = 0, size - 1
|
|
||||||
status_code = 200
|
|
||||||
|
|
||||||
range_header = request.headers.get("range")
|
|
||||||
if range_header:
|
|
||||||
match = _RANGE_RE.fullmatch(range_header)
|
|
||||||
if match:
|
|
||||||
if match.group(1):
|
|
||||||
start = int(match.group(1))
|
|
||||||
if match.group(2):
|
|
||||||
end = min(int(match.group(2)), size - 1)
|
|
||||||
if start >= size:
|
|
||||||
raise HTTPException(416, "Plage invalide")
|
|
||||||
status_code = 206
|
|
||||||
|
|
||||||
length = end - start + 1
|
|
||||||
|
|
||||||
async def iter_file() -> AsyncIterator[bytes]:
|
|
||||||
with path.open("rb") as fh:
|
|
||||||
fh.seek(start)
|
|
||||||
remaining = length
|
|
||||||
while remaining > 0:
|
|
||||||
chunk = fh.read(min(CHUNK_SIZE, remaining))
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
remaining -= len(chunk)
|
|
||||||
yield chunk
|
|
||||||
|
|
||||||
headers = {
|
|
||||||
"Content-Range": f"bytes {start}-{end}/{size}",
|
|
||||||
"Accept-Ranges": "bytes",
|
|
||||||
"Content-Length": str(length),
|
|
||||||
}
|
|
||||||
return StreamingResponse(
|
|
||||||
iter_file(), status_code=status_code, headers=headers, media_type=content_type
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ProgressRequest(BaseModel):
|
|
||||||
position_seconds: float = Field(ge=0)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/stream/{download_id}/progress")
|
|
||||||
async def save_progress(download_id: int, payload: ProgressRequest, user: CurrentUser) -> dict:
|
|
||||||
"""Reprise de lecture : mémorise la position de visionnage."""
|
|
||||||
await db.execute(
|
|
||||||
"INSERT INTO watch_progress (user_id, download_id, position_seconds, updated_at) "
|
|
||||||
"VALUES (?, ?, ?, datetime('now')) "
|
|
||||||
"ON CONFLICT(user_id, download_id) DO UPDATE SET "
|
|
||||||
"position_seconds = excluded.position_seconds, updated_at = excluded.updated_at",
|
|
||||||
(user.id, download_id, payload.position_seconds),
|
|
||||||
)
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- favoris
|
|
||||||
|
|
||||||
|
|
||||||
class FavoriteRequest(BaseModel):
|
|
||||||
source: str
|
|
||||||
source_id: str
|
|
||||||
title: str
|
|
||||||
image_url: str | None = None
|
|
||||||
payload: dict | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/favorites")
|
|
||||||
async def list_favorites(user: CurrentUser, offset: int = 0, limit: int = 24) -> dict:
|
|
||||||
total = await db.fetchone("SELECT COUNT(*) AS n FROM favorites WHERE user_id = ?", (user.id,))
|
|
||||||
rows = await db.fetchall(
|
|
||||||
"SELECT * FROM favorites WHERE user_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?",
|
|
||||||
(user.id, limit, offset),
|
|
||||||
)
|
|
||||||
return {"total": total["n"], "items": [dict(row) for row in rows]}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/favorites", status_code=201)
|
|
||||||
async def add_favorite(payload: FavoriteRequest, user: CurrentUser) -> dict:
|
|
||||||
import json
|
|
||||||
|
|
||||||
try:
|
|
||||||
await db.execute(
|
|
||||||
"INSERT INTO favorites (user_id, source, source_id, title, image_url, payload) "
|
|
||||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
|
||||||
(
|
|
||||||
user.id,
|
|
||||||
payload.source,
|
|
||||||
payload.source_id,
|
|
||||||
payload.title,
|
|
||||||
payload.image_url,
|
|
||||||
json.dumps(payload.payload, ensure_ascii=False) if payload.payload else None,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
except aiosqlite.IntegrityError as exc:
|
|
||||||
raise HTTPException(409, "Déjà dans les favoris") from exc
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/favorites/{favorite_id}")
|
|
||||||
async def remove_favorite(favorite_id: int, user: CurrentUser) -> dict:
|
|
||||||
cursor = await db.execute(
|
|
||||||
"DELETE FROM favorites WHERE id = ? AND user_id = ?", (favorite_id, user.id)
|
|
||||||
)
|
|
||||||
if cursor.rowcount == 0:
|
|
||||||
raise HTTPException(404, "Favori introuvable")
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
|
|
||||||
# raccourci pratique pour l'UI
|
|
||||||
@router.get("/downloads/{download_id}")
|
|
||||||
async def get_download(download_id: int) -> dict:
|
|
||||||
data = await download_manager.get(download_id)
|
|
||||||
if data is None:
|
|
||||||
raise HTTPException(404, "Téléchargement introuvable")
|
|
||||||
return data
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
"""Pages HTML (Jinja2 + htmx). Toutes protégées sauf /login."""
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request
|
|
||||||
from fastapi.responses import HTMLResponse
|
|
||||||
from fastapi.templating import Jinja2Templates
|
|
||||||
|
|
||||||
from app.config import BASE_DIR
|
|
||||||
from app.routers.auth import current_user
|
|
||||||
|
|
||||||
router = APIRouter(tags=["pages"], include_in_schema=False)
|
|
||||||
protected = APIRouter(tags=["pages"], include_in_schema=False, dependencies=[Depends(current_user)])
|
|
||||||
|
|
||||||
templates = Jinja2Templates(directory=BASE_DIR / "app" / "templates")
|
|
||||||
|
|
||||||
# Cache-busting des assets : version dérivée de la date des fichiers statiques —
|
|
||||||
# tout changement de CSS/JS invalide le cache navigateur sans intervention.
|
|
||||||
_static_root = BASE_DIR / "app" / "static"
|
|
||||||
templates.env.globals["asset_v"] = str(
|
|
||||||
max(int(p.stat().st_mtime) for p in _static_root.rglob("*") if p.is_file())
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/login", response_class=HTMLResponse)
|
|
||||||
async def login_page(request: Request) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(request, "login.html")
|
|
||||||
|
|
||||||
|
|
||||||
@protected.get("/", response_class=HTMLResponse)
|
|
||||||
async def index(request: Request) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(request, "index.html")
|
|
||||||
|
|
||||||
|
|
||||||
@protected.get("/discover", response_class=HTMLResponse)
|
|
||||||
async def discover_page(request: Request) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(request, "discover.html")
|
|
||||||
|
|
||||||
|
|
||||||
@protected.get("/title/{source}/{source_id:path}", response_class=HTMLResponse)
|
|
||||||
async def title_page(request: Request, source: str, source_id: str) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(
|
|
||||||
request, "title.html", {"source": source, "source_id": source_id}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@protected.get("/downloads", response_class=HTMLResponse)
|
|
||||||
async def downloads_page(request: Request) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(request, "downloads.html")
|
|
||||||
|
|
||||||
|
|
||||||
@protected.get("/library", response_class=HTMLResponse)
|
|
||||||
async def library_page(request: Request) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(request, "library.html")
|
|
||||||
|
|
||||||
|
|
||||||
@protected.get("/watch/{download_id}", response_class=HTMLResponse)
|
|
||||||
async def watch_page(request: Request, download_id: int) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(request, "watch.html", {"download_id": download_id})
|
|
||||||
|
|
||||||
|
|
||||||
@protected.get("/favorites", response_class=HTMLResponse)
|
|
||||||
async def favorites_page(request: Request) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(request, "favorites.html")
|
|
||||||
|
|
||||||
|
|
||||||
@protected.get("/admin", response_class=HTMLResponse)
|
|
||||||
async def admin_page(request: Request) -> HTMLResponse:
|
|
||||||
return templates.TemplateResponse(request, "admin.html")
|
|
||||||
@@ -1,129 +0,0 @@
|
|||||||
"""Proxy vidéo : sert les flux distants (mp4/HLS) via le serveur.
|
|
||||||
|
|
||||||
Indispensable pour les hébergeurs dont les tokens sont liés à l'IP qui a
|
|
||||||
résolu le lien (le navigateur de l'utilisateur a une IP différente du serveur).
|
|
||||||
Les playlists m3u8 sont réécrites pour que segments et variantes passent aussi
|
|
||||||
par le proxy. Accès réservé aux utilisateurs connectés.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from urllib.parse import quote, urljoin
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
|
||||||
from fastapi.responses import Response, StreamingResponse
|
|
||||||
|
|
||||||
from app.routers.auth import current_user
|
|
||||||
from app.scrapers.http import get_client
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api", tags=["proxy"], dependencies=[Depends(current_user)])
|
|
||||||
|
|
||||||
HLS_CONTENT_TYPES = ("application/vnd.apple.mpegurl", "application/x-mpegurl", "audio/mpegurl")
|
|
||||||
HOP_HEADERS = (
|
|
||||||
"content-type", "content-length", "content-range", "accept-ranges", "cache-control"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _check_url(url: str) -> None:
|
|
||||||
if not url.startswith(("http://", "https://")):
|
|
||||||
raise HTTPException(422, "URL invalide")
|
|
||||||
|
|
||||||
|
|
||||||
def proxy_url(remote_url: str, referer: str | None = None) -> str:
|
|
||||||
out = "/api/proxy?url=" + quote(remote_url, safe="")
|
|
||||||
if referer:
|
|
||||||
out += "&ref=" + quote(referer, safe="")
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/proxy")
|
|
||||||
async def proxy(
|
|
||||||
request: Request,
|
|
||||||
url: str = Query(),
|
|
||||||
ref: str | None = Query(None),
|
|
||||||
) -> Response:
|
|
||||||
_check_url(url)
|
|
||||||
headers: dict[str, str] = {}
|
|
||||||
if ref:
|
|
||||||
_check_url(ref)
|
|
||||||
headers["Referer"] = ref
|
|
||||||
if range_header := request.headers.get("range"):
|
|
||||||
headers["Range"] = range_header
|
|
||||||
|
|
||||||
client = get_client()
|
|
||||||
try:
|
|
||||||
upstream = await client.send(
|
|
||||||
client.build_request("GET", url, headers=headers), stream=True
|
|
||||||
)
|
|
||||||
except httpx.HTTPError as exc:
|
|
||||||
logger.error("Proxy : échec de connexion à %s : %s", url, exc)
|
|
||||||
raise HTTPException(502, f"Hébergeur injoignable : {exc}") from exc
|
|
||||||
|
|
||||||
content_type = upstream.headers.get("content-type", "")
|
|
||||||
is_hls = any(t in content_type for t in HLS_CONTENT_TYPES) or url.endswith(".m3u8")
|
|
||||||
|
|
||||||
if is_hls:
|
|
||||||
# Playlist : lecture intégrale puis réécriture des URIs via le proxy
|
|
||||||
try:
|
|
||||||
body = (await upstream.aread()).decode("utf-8", errors="replace")
|
|
||||||
finally:
|
|
||||||
await upstream.aclose()
|
|
||||||
if upstream.status_code >= 400:
|
|
||||||
logger.error("Proxy HLS : %s → HTTP %s", url, upstream.status_code)
|
|
||||||
raise HTTPException(502, f"Hébergeur : HTTP {upstream.status_code}")
|
|
||||||
rewritten = _rewrite_playlist(body, url, ref)
|
|
||||||
return Response(
|
|
||||||
rewritten,
|
|
||||||
media_type="application/vnd.apple.mpegurl",
|
|
||||||
headers={"Cache-Control": "no-store"},
|
|
||||||
)
|
|
||||||
|
|
||||||
if upstream.status_code >= 400:
|
|
||||||
await upstream.aclose()
|
|
||||||
logger.error("Proxy : %s → HTTP %s", url, upstream.status_code)
|
|
||||||
raise HTTPException(502, f"Hébergeur : HTTP {upstream.status_code}")
|
|
||||||
|
|
||||||
async def byte_stream() -> AsyncIterator[bytes]:
|
|
||||||
try:
|
|
||||||
async for chunk in upstream.aiter_bytes(1 << 16):
|
|
||||||
yield chunk
|
|
||||||
finally:
|
|
||||||
await upstream.aclose()
|
|
||||||
|
|
||||||
passthrough = {
|
|
||||||
h: upstream.headers[h] for h in HOP_HEADERS if h in upstream.headers
|
|
||||||
}
|
|
||||||
passthrough.setdefault("accept-ranges", "bytes")
|
|
||||||
return StreamingResponse(
|
|
||||||
byte_stream(),
|
|
||||||
status_code=upstream.status_code,
|
|
||||||
headers=passthrough,
|
|
||||||
media_type=content_type.split(";")[0] or "application/octet-stream",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
_KEY_URI_RE = re.compile(r'URI="([^"]+)"')
|
|
||||||
|
|
||||||
|
|
||||||
def _rewrite_playlist(body: str, base_url: str, referer: str | None) -> str:
|
|
||||||
"""Réécrit toutes les URIs d'une playlist m3u8 pour passer par le proxy."""
|
|
||||||
|
|
||||||
def absolutize(uri: str) -> str:
|
|
||||||
return proxy_url(urljoin(base_url, uri), referer)
|
|
||||||
|
|
||||||
lines = []
|
|
||||||
for line in body.splitlines():
|
|
||||||
stripped = line.strip()
|
|
||||||
if not stripped:
|
|
||||||
continue
|
|
||||||
if stripped.startswith("#"):
|
|
||||||
lines.append(_KEY_URI_RE.sub(
|
|
||||||
lambda m: f'URI="{absolutize(m.group(1))}"', line
|
|
||||||
))
|
|
||||||
else:
|
|
||||||
lines.append(absolutize(stripped))
|
|
||||||
return "\n".join(lines) + "\n"
|
|
||||||
@@ -1,263 +0,0 @@
|
|||||||
"""API compatible qBittorrent Web API v2 — Ohm comme client de téléchargement Sonarr.
|
|
||||||
|
|
||||||
Sonarr sait piloter un qBittorrent ; Ohm implémente le sous-set suffisant pour
|
|
||||||
être vu comme un client de téléchargement « torrent » :
|
|
||||||
|
|
||||||
- login (`/auth/login`, mot de passe = clé API Torznab) + session SID
|
|
||||||
- `torrents/add` : Sonarr renvoie le .torrent de service servi par l'indexeur
|
|
||||||
Torznab → le grab est rejoué (dédupliqué par infohash) dans la file interne
|
|
||||||
- `torrents/info` / `properties` : progression temps réel, `content_path`
|
|
||||||
pointant vers le fichier dans /downloads (mapper en chemin hôte côté Sonarr
|
|
||||||
via « Remote Path Mapping » si besoin)
|
|
||||||
- `torrents/delete` : retrait de la file, avec ou sans le fichier
|
|
||||||
- `pause`/`resume` : branchés sur le gestionnaire de téléchargements
|
|
||||||
"""
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import logging
|
|
||||||
import secrets
|
|
||||||
import time
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
from urllib.parse import parse_qsl
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, Response, UploadFile
|
|
||||||
|
|
||||||
from app.db import db
|
|
||||||
from app.scrapers.base import ScrapeError
|
|
||||||
from app.services.downloads import download_manager
|
|
||||||
from app.services.settings import get_torznab_apikey
|
|
||||||
from app.services.torznab import _bencode, bdecode, torznab
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def _plain(text: str) -> Response:
|
|
||||||
"""qBittorrent répond en texte brut, pas en JSON."""
|
|
||||||
return Response(text, media_type="text/plain")
|
|
||||||
router = APIRouter(tags=["qbit"])
|
|
||||||
_OPTIONAL_FILE = File(None)
|
|
||||||
|
|
||||||
_SID_TTL = 3600.0
|
|
||||||
_sessions: dict[str, float] = {}
|
|
||||||
|
|
||||||
_QBIT_VERSION = "v4.6.0"
|
|
||||||
_WEBAPI_VERSION = "2.9.3"
|
|
||||||
|
|
||||||
# status Ohm → état qBittorrent (noms compris par Sonarr v3/v4)
|
|
||||||
_QBIT_STATES = {
|
|
||||||
"pending": "queuedDL",
|
|
||||||
"downloading": "downloading",
|
|
||||||
"paused": "pausedDL",
|
|
||||||
"done": "pausedUP", # terminé → Sonarr importe
|
|
||||||
"failed": "error",
|
|
||||||
"cancelled": "error",
|
|
||||||
}
|
|
||||||
|
|
||||||
_PREFIX = "sonarr:"
|
|
||||||
|
|
||||||
|
|
||||||
async def _require_sid(request: Request) -> None:
|
|
||||||
sid = request.cookies.get("SID")
|
|
||||||
if not sid or _sessions.get(sid, 0.0) < time.time():
|
|
||||||
_sessions.pop(sid, None)
|
|
||||||
raise HTTPException(403, "Session invalide — (re)connecte-toi via /api/v2/auth/login")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/v2/auth/login")
|
|
||||||
async def login(username: str = Form(""), password: str = Form("")) -> Response:
|
|
||||||
if password != await get_torznab_apikey():
|
|
||||||
logger.warning("Login qBittorrent refusé (utilisateur %r)", username)
|
|
||||||
raise HTTPException(403, "Fails.")
|
|
||||||
sid = secrets.token_hex(16)
|
|
||||||
_sessions[sid] = time.time() + _SID_TTL
|
|
||||||
response = _plain("Ok.")
|
|
||||||
response.set_cookie("SID", sid, httponly=True)
|
|
||||||
logger.info("Client qBittorrent authentifié (utilisateur %r)", username)
|
|
||||||
return response
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v2/app/version", dependencies=[Depends(_require_sid)])
|
|
||||||
async def app_version() -> Response:
|
|
||||||
return _plain(_QBIT_VERSION)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v2/app/webapiVersion", dependencies=[Depends(_require_sid)])
|
|
||||||
async def webapi_version() -> Response:
|
|
||||||
return _plain(_WEBAPI_VERSION)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v2/app/preferences", dependencies=[Depends(_require_sid)])
|
|
||||||
async def app_preferences() -> dict:
|
|
||||||
return {"save_path": "/downloads"}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v2/transfer/info", dependencies=[Depends(_require_sid)])
|
|
||||||
async def transfer_info() -> dict:
|
|
||||||
return {"dl_info_speed": 0, "dl_info_data": 0, "up_info_speed": 0, "up_info_data": 0}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- torrents
|
|
||||||
|
|
||||||
|
|
||||||
def _ts(sqlite_dt: str | None) -> int:
|
|
||||||
if not sqlite_dt:
|
|
||||||
return -1
|
|
||||||
try:
|
|
||||||
return int(
|
|
||||||
datetime.strptime(sqlite_dt, "%Y-%m-%d %H:%M:%S")
|
|
||||||
.replace(tzinfo=UTC)
|
|
||||||
.timestamp()
|
|
||||||
)
|
|
||||||
except ValueError:
|
|
||||||
return -1
|
|
||||||
|
|
||||||
|
|
||||||
async def _sonarr_rows() -> dict[str, dict]:
|
|
||||||
"""Téléchargements d'origine Sonarr, indexés par infohash (dernier par hash)."""
|
|
||||||
rows = await db.fetchall(
|
|
||||||
f"SELECT * FROM downloads WHERE source_key LIKE '{_PREFIX}%' ORDER BY id"
|
|
||||||
)
|
|
||||||
by_hash: dict[str, dict] = {}
|
|
||||||
for row in rows:
|
|
||||||
prefix_end = row["source_key"].find("|")
|
|
||||||
infohash = row["source_key"][len(_PREFIX) : prefix_end]
|
|
||||||
by_hash[infohash] = dict(row) # le plus grand id écrase les précédents
|
|
||||||
return by_hash
|
|
||||||
|
|
||||||
|
|
||||||
def _content_path(row: dict) -> str:
|
|
||||||
return "/downloads/" + (row["file_path"] or row["title"])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v2/torrents/info", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_info() -> list[dict]:
|
|
||||||
items = []
|
|
||||||
for infohash, row in (await _sonarr_rows()).items():
|
|
||||||
live = await download_manager.get(row["id"])
|
|
||||||
downloaded = live["downloaded_bytes"]
|
|
||||||
total = live["total_bytes"]
|
|
||||||
items.append(
|
|
||||||
{
|
|
||||||
"hash": infohash,
|
|
||||||
"name": row["title"],
|
|
||||||
"state": _QBIT_STATES.get(row["status"], "error"),
|
|
||||||
"progress": round(downloaded / total, 4) if total else 0.0,
|
|
||||||
"dlspeed": live["speed_bps"],
|
|
||||||
"eta": live["eta_seconds"] or 0,
|
|
||||||
"total_size": total or 0,
|
|
||||||
"completed": downloaded,
|
|
||||||
"amount_left": max(0, (total or 0) - downloaded),
|
|
||||||
"category": "",
|
|
||||||
"tags": "",
|
|
||||||
"save_path": _content_path(row).rsplit("/", 1)[0],
|
|
||||||
"content_path": _content_path(row),
|
|
||||||
"added_on": _ts(row["created_at"]),
|
|
||||||
"completion_on": _ts(row["updated_at"]) if row["status"] == "done" else -1,
|
|
||||||
"ratio": 1,
|
|
||||||
"num_seeds": 0,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return items
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v2/torrents/properties", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_properties(hash: str) -> dict:
|
|
||||||
rows = await _sonarr_rows()
|
|
||||||
row = rows.get(hash.lower())
|
|
||||||
if row is None:
|
|
||||||
raise HTTPException(404, "Torrent introuvable")
|
|
||||||
live = await download_manager.get(row["id"])
|
|
||||||
return {
|
|
||||||
"name": row["title"],
|
|
||||||
"content_path": _content_path(row),
|
|
||||||
"save_path": _content_path(row).rsplit("/", 1)[0],
|
|
||||||
"total_size": live["total_bytes"] or 0,
|
|
||||||
"total_downloaded": live["downloaded_bytes"],
|
|
||||||
"addition_date": _ts(row["created_at"]),
|
|
||||||
"completion_date": _ts(row["updated_at"]) if row["status"] == "done" else -1,
|
|
||||||
"seeding_time": 0,
|
|
||||||
"share_ratio": 1,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
_categories: dict[str, dict] = {}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v2/torrents/categories", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_categories() -> dict:
|
|
||||||
return _categories
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/v2/torrents/createCategory", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_create_category(category: str = Form(""), savePath: str = Form("")) -> Response:
|
|
||||||
if category:
|
|
||||||
_categories[category] = {"name": category, "savePath": savePath}
|
|
||||||
return _plain("Ok.")
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/v2/torrents/tags", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_tags() -> list:
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/v2/torrents/setCategory", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_set_category() -> Response:
|
|
||||||
return _plain("Ok.")
|
|
||||||
|
|
||||||
|
|
||||||
async def _add_stub(stub: bytes) -> Response:
|
|
||||||
"""Rejoue le grab encodé dans un .torrent de service."""
|
|
||||||
try:
|
|
||||||
parsed = bdecode(stub)
|
|
||||||
info = parsed[b"info"]
|
|
||||||
announce = parsed[b"announce"].decode()
|
|
||||||
params = dict(parse_qsl(announce.split("?", 1)[1]))
|
|
||||||
infohash = hashlib.sha1(_bencode(info)).hexdigest()
|
|
||||||
await torznab.grab(
|
|
||||||
params["source"],
|
|
||||||
params["sid"],
|
|
||||||
int(params["season"]),
|
|
||||||
int(params["ep"]),
|
|
||||||
params["series"],
|
|
||||||
sonarr_hash=infohash,
|
|
||||||
)
|
|
||||||
except (ValueError, KeyError, ScrapeError) as exc:
|
|
||||||
logger.error("Ajout qBittorrent refusé : %s", exc)
|
|
||||||
return _plain("Fals.")
|
|
||||||
return _plain("Ok.")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/v2/torrents/add", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_add(torrents: UploadFile | None = _OPTIONAL_FILE) -> Response:
|
|
||||||
if torrents is None or not torrents.filename:
|
|
||||||
return _plain("Fals.")
|
|
||||||
return await _add_stub(await torrents.read())
|
|
||||||
|
|
||||||
|
|
||||||
async def _ids_for_hashes(hashes: str) -> list[int]:
|
|
||||||
wanted = {h.lower() for h in hashes.split("|") if h}
|
|
||||||
rows = await _sonarr_rows()
|
|
||||||
return [row["id"] for infohash, row in rows.items() if infohash in wanted]
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/v2/torrents/delete", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_delete(
|
|
||||||
hashes: str = Form(...), deleteFiles: str = Form("false")
|
|
||||||
) -> Response:
|
|
||||||
for download_id in await _ids_for_hashes(hashes):
|
|
||||||
await download_manager.delete(download_id, delete_file=deleteFiles == "true")
|
|
||||||
return _plain("Ok.")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/v2/torrents/pause", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_pause(hashes: str = Form(...)) -> Response:
|
|
||||||
for download_id in await _ids_for_hashes(hashes):
|
|
||||||
await download_manager.pause(download_id)
|
|
||||||
return _plain("Ok.")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/v2/torrents/resume", dependencies=[Depends(_require_sid)])
|
|
||||||
async def torrents_resume(hashes: str = Form(...)) -> Response:
|
|
||||||
for download_id in await _ids_for_hashes(hashes):
|
|
||||||
await download_manager.resume(download_id)
|
|
||||||
return _plain("Ok.")
|
|
||||||
@@ -1,155 +0,0 @@
|
|||||||
"""Recherche multi-sources, fiches de titres et extraction de liens vidéo."""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import dataclasses
|
|
||||||
import logging
|
|
||||||
from typing import Annotated
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
||||||
|
|
||||||
from app.routers.auth import CurrentUser, current_user
|
|
||||||
from app.scrapers.base import (
|
|
||||||
ScrapeError,
|
|
||||||
SourceScraper,
|
|
||||||
VideoLink,
|
|
||||||
all_sources,
|
|
||||||
get_source,
|
|
||||||
import_all_scrapers,
|
|
||||||
resolve_hoster,
|
|
||||||
)
|
|
||||||
from app.services.kitsu import KitsuService
|
|
||||||
from app.services.settings import is_source_enabled
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api", tags=["search"], dependencies=[Depends(current_user)])
|
|
||||||
|
|
||||||
import_all_scrapers()
|
|
||||||
kitsu = KitsuService()
|
|
||||||
|
|
||||||
# Types de médias gardés selon la préférence. Les films réels (French-Stream)
|
|
||||||
# accompagnent les séries — le mode animés reste sur l'animation.
|
|
||||||
_PREFERENCE_MEDIA_TYPES = {
|
|
||||||
"anime": {"anime"},
|
|
||||||
"serie": {"serie", "film"},
|
|
||||||
"both": {"anime", "serie", "film"},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _allowed_media_types(preference: str) -> set[str]:
|
|
||||||
return _PREFERENCE_MEDIA_TYPES.get(preference, _PREFERENCE_MEDIA_TYPES["both"])
|
|
||||||
|
|
||||||
|
|
||||||
async def enabled_sources(allowed: set[str] | None = None) -> list[SourceScraper]:
|
|
||||||
"""Sources activées, limitées à celles pouvant servir les types de médias autorisés."""
|
|
||||||
sources = []
|
|
||||||
for source in all_sources():
|
|
||||||
if not await is_source_enabled(source.name):
|
|
||||||
continue
|
|
||||||
if allowed is not None and not set(source.media_types) & allowed:
|
|
||||||
continue
|
|
||||||
sources.append(source)
|
|
||||||
return sources
|
|
||||||
|
|
||||||
|
|
||||||
sources = []
|
|
||||||
for source in all_sources():
|
|
||||||
if await is_source_enabled(source.name):
|
|
||||||
sources.append(source)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/sources")
|
|
||||||
async def list_sources() -> list[dict]:
|
|
||||||
"""Sources disponibles avec leur état d'activation."""
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"name": source.name,
|
|
||||||
"label": source.label,
|
|
||||||
"base_url": source.base_url,
|
|
||||||
"media_types": list(source.media_types),
|
|
||||||
"enabled": await is_source_enabled(source.name),
|
|
||||||
}
|
|
||||||
for source in all_sources()
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/search")
|
|
||||||
async def search(q: Annotated[str, Query(min_length=2)], user: CurrentUser) -> dict:
|
|
||||||
"""Recherche unifiée : une requête interroge toutes les sources activées."""
|
|
||||||
allowed = _allowed_media_types(user.content_preference)
|
|
||||||
sources = await enabled_sources(allowed)
|
|
||||||
|
|
||||||
async def safe_search(source: SourceScraper) -> tuple[list, str | None]:
|
|
||||||
try:
|
|
||||||
results = [dataclasses.asdict(r) for r in await source.search(q)]
|
|
||||||
return results, None
|
|
||||||
except ScrapeError as exc:
|
|
||||||
logger.error("Recherche échouée sur %s : %s", source.name, exc)
|
|
||||||
return [], source.name
|
|
||||||
|
|
||||||
outcomes = await asyncio.gather(*(safe_search(s) for s in sources))
|
|
||||||
results = [item for items, _ in outcomes for item in items]
|
|
||||||
results = [r for r in results if r.get("media_type", "anime") in allowed]
|
|
||||||
failed = [name for _, name in outcomes if name]
|
|
||||||
return {"query": q, "count": len(results), "results": results, "failed_sources": failed}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/titles/{source}/{source_id:path}")
|
|
||||||
async def title_details(source: str, source_id: str, enrich: bool = True) -> dict:
|
|
||||||
"""Fiche détaillée d'un titre (+ enrichissement Kitsu des champs manquants)."""
|
|
||||||
scraper = get_source(source)
|
|
||||||
try:
|
|
||||||
details = await scraper.get_details(source_id)
|
|
||||||
except ScrapeError as exc:
|
|
||||||
raise HTTPException(502, detail=str(exc)) from exc
|
|
||||||
if enrich:
|
|
||||||
details = await kitsu.enrich(details)
|
|
||||||
return dataclasses.asdict(details)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/episodes/{source}/{source_id:path}")
|
|
||||||
async def list_episodes(source: str, source_id: str) -> dict:
|
|
||||||
scraper = get_source(source)
|
|
||||||
try:
|
|
||||||
episodes = await scraper.list_episodes(source_id)
|
|
||||||
except ScrapeError as exc:
|
|
||||||
raise HTTPException(502, detail=str(exc)) from exc
|
|
||||||
return {"episodes": [dataclasses.asdict(e) for e in episodes]}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/extract")
|
|
||||||
async def extract(episode_url: Annotated[str, Query()]) -> dict:
|
|
||||||
"""Résout la chaîne complète : page d'épisode → embeds → URL vidéo directe."""
|
|
||||||
try:
|
|
||||||
source = _find_source_for_url(episode_url)
|
|
||||||
embeds = await source.extract_embed_links(episode_url)
|
|
||||||
except ScrapeError as exc:
|
|
||||||
raise HTTPException(502, detail=str(exc)) from exc
|
|
||||||
|
|
||||||
links: list[dict] = []
|
|
||||||
errors: list[str] = []
|
|
||||||
for embed in embeds:
|
|
||||||
extractor = resolve_hoster(embed)
|
|
||||||
if extractor is None:
|
|
||||||
errors.append(f"Hébergeur non supporté : {embed}")
|
|
||||||
logger.warning("Aucun extracteur pour %s", embed)
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
link: VideoLink = await extractor.extract(embed)
|
|
||||||
data = dataclasses.asdict(link)
|
|
||||||
data["embed_url"] = embed
|
|
||||||
links.append(data)
|
|
||||||
except ScrapeError as exc:
|
|
||||||
errors.append(str(exc))
|
|
||||||
logger.error("Extraction échouée pour %s : %s", embed, exc)
|
|
||||||
|
|
||||||
if not links and not embeds:
|
|
||||||
raise HTTPException(502, detail="Aucun lecteur trouvé sur la page de l'épisode")
|
|
||||||
return {"episode_url": episode_url, "links": links, "errors": errors}
|
|
||||||
|
|
||||||
|
|
||||||
def _find_source_for_url(url: str) -> SourceScraper:
|
|
||||||
for source in all_sources():
|
|
||||||
if source.base_url.split("//")[-1].split("/")[0] in url:
|
|
||||||
return source
|
|
||||||
raise ScrapeError(f"Aucune source ne correspond à l'URL : {url}")
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
"""Endpoints système publics : version (utilisée par le frontend pour détecter
|
|
||||||
une mise à jour et recharger la page automatiquement)."""
|
|
||||||
|
|
||||||
from fastapi import APIRouter
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.version import get_version
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api", tags=["system"])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/version")
|
|
||||||
async def version() -> dict[str, str]:
|
|
||||||
return {"name": get_settings().app_name, "version": get_version()}
|
|
||||||
@@ -1,139 +0,0 @@
|
|||||||
"""API Torznab/Newznab consommée par Sonarr, Prowlarr (et les *arr en général).
|
|
||||||
|
|
||||||
Authentification par clé API (`?apikey=…` ou en-tête `X-Api-Key`), indépendante
|
|
||||||
des sessions utilisateurs — aucun cookie requis.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import secrets
|
|
||||||
from urllib.parse import urlencode
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Request, Response
|
|
||||||
|
|
||||||
from app.scrapers.base import ScrapeError
|
|
||||||
from app.services.settings import get_torznab_apikey
|
|
||||||
from app.services.torznab import build_stub, torznab
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/torznab", tags=["torznab"])
|
|
||||||
|
|
||||||
|
|
||||||
def _base_url(request: Request) -> str:
|
|
||||||
return str(request.base_url).rstrip("/")
|
|
||||||
|
|
||||||
|
|
||||||
async def _auth_error(request: Request, apikey: str | None) -> Response | None:
|
|
||||||
"""Réponse d'erreur XML si la clé est invalide (None = authentifié)."""
|
|
||||||
expected = await get_torznab_apikey()
|
|
||||||
provided = apikey or request.headers.get("X-Api-Key") or ""
|
|
||||||
if secrets.compare_digest(provided, expected):
|
|
||||||
return None
|
|
||||||
return Response(
|
|
||||||
torznab.error_xml(100, "Invalid API key"),
|
|
||||||
media_type="application/xml",
|
|
||||||
status_code=401,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api")
|
|
||||||
@router.post("/api")
|
|
||||||
async def torznab_api(
|
|
||||||
request: Request,
|
|
||||||
t: str = "caps",
|
|
||||||
q: str | None = None,
|
|
||||||
season: int | None = None,
|
|
||||||
ep: int | None = None,
|
|
||||||
apikey: str | None = None,
|
|
||||||
) -> Response:
|
|
||||||
"""Point d'entrée Torznab : caps / tvsearch / search."""
|
|
||||||
error = await _auth_error(request, apikey)
|
|
||||||
if error is not None:
|
|
||||||
return error
|
|
||||||
key = await get_torznab_apikey()
|
|
||||||
base = _base_url(request)
|
|
||||||
|
|
||||||
if t == "caps":
|
|
||||||
return Response(torznab.caps_xml(base), media_type="application/xml")
|
|
||||||
|
|
||||||
if t in ("tvsearch", "search"):
|
|
||||||
if not q or len(q) < 2:
|
|
||||||
releases = await torznab.latest_releases()
|
|
||||||
return Response(
|
|
||||||
torznab.results_xml(base, key, releases), media_type="application/rss+xml"
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
releases = await torznab.tvsearch(q, season=season, ep=ep)
|
|
||||||
except Exception as exc: # noqa: BLE001 — Sonarr attend du XML, pas un traceback
|
|
||||||
logger.error("Torznab %s %r KO : %s", t, q, exc)
|
|
||||||
return Response(
|
|
||||||
torznab.error_xml(300, "Recherche indisponible"),
|
|
||||||
media_type="application/xml",
|
|
||||||
status_code=502,
|
|
||||||
)
|
|
||||||
return Response(
|
|
||||||
torznab.results_xml(base, key, releases), media_type="application/rss+xml"
|
|
||||||
)
|
|
||||||
|
|
||||||
return Response(
|
|
||||||
torznab.error_xml(203, f"Fonction non supportée : {t}"),
|
|
||||||
media_type="application/xml",
|
|
||||||
status_code=400,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/download")
|
|
||||||
async def torznab_download(
|
|
||||||
request: Request,
|
|
||||||
source: str | None = None,
|
|
||||||
sid: str | None = None,
|
|
||||||
season: int | None = None,
|
|
||||||
ep: int | None = None,
|
|
||||||
series: str | None = None,
|
|
||||||
apikey: str | None = None,
|
|
||||||
) -> Response:
|
|
||||||
"""Grab : Sonarr récupère le « .torrent » ; Ohm télécharge l'épisode.
|
|
||||||
|
|
||||||
Le flux retourne un .torrent de service : avec un client « Torrent
|
|
||||||
Blackhole » c'est un simple accusé de réception ; avec le client
|
|
||||||
qBittumber (l'API /api/v2 d'Ohm), Sonarr le renvoie et le grab est
|
|
||||||
rejoué/dédupliqué, puis suivi comme un téléchargement classique.
|
|
||||||
"""
|
|
||||||
error = await _auth_error(request, apikey)
|
|
||||||
if error is not None:
|
|
||||||
return error
|
|
||||||
if None in (source, sid, season, ep, series):
|
|
||||||
return Response(
|
|
||||||
torznab.error_xml(201, "Paramètres manquants : source, sid, season, ep, series"),
|
|
||||||
media_type="application/xml",
|
|
||||||
status_code=400,
|
|
||||||
)
|
|
||||||
name = f"{series} S{season:02d}E{ep:02d}"
|
|
||||||
announce = (
|
|
||||||
_base_url(request)
|
|
||||||
+ "/torznab/api?"
|
|
||||||
+ urlencode(
|
|
||||||
{"source": source, "sid": sid, "season": season, "ep": ep, "series": series}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
stub, infohash = build_stub(announce, name)
|
|
||||||
try:
|
|
||||||
result = await torznab.grab(source, sid, season, ep, series, sonarr_hash=infohash)
|
|
||||||
logger.info("Torznab grab OK : %s → download %s", series, result.get("id"))
|
|
||||||
except ScrapeError as exc:
|
|
||||||
logger.error("Torznab grab KO : %s", exc)
|
|
||||||
return Response(
|
|
||||||
torznab.error_xml(300, str(exc)),
|
|
||||||
media_type="application/xml",
|
|
||||||
status_code=502,
|
|
||||||
)
|
|
||||||
return Response(
|
|
||||||
stub,
|
|
||||||
media_type="application/x-bittorrent",
|
|
||||||
headers={
|
|
||||||
"Content-Disposition": (
|
|
||||||
f'attachment; filename="ohm-{series.replace("/", "-")}'
|
|
||||||
f"-S{season:02d}E{ep:02d}.torrent"
|
|
||||||
)
|
|
||||||
},
|
|
||||||
)
|
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
"""Contrats et registres des scrapers.
|
|
||||||
|
|
||||||
Architecture à 2 registres :
|
|
||||||
- sources (sites de catalogues animes/séries) : recherche, détails, épisodes, liens embed
|
|
||||||
- hébergeurs vidéo : résolution d'une URL embed → URL directe du fichier
|
|
||||||
|
|
||||||
Ajouter une source = écrire une classe qui implémente le contrat et la décorer
|
|
||||||
@register_source / @register_hoster.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class ScrapeError(Exception):
|
|
||||||
"""Échec de scraping/extraction — jamais silencieux, toujours journalisé et remonté."""
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- modèles
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class SearchResult:
|
|
||||||
source: str
|
|
||||||
source_id: str # identifiant stable chez la source (slug, id…)
|
|
||||||
title: str
|
|
||||||
url: str
|
|
||||||
image_url: str | None = None
|
|
||||||
media_type: str = "anime" # anime | serie | film
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Episode:
|
|
||||||
number: float # 1, 2, 2.5 pour les OAV intermédiaires
|
|
||||||
title: str | None
|
|
||||||
url: str # page de l'épisode chez la source
|
|
||||||
season: int = 1
|
|
||||||
version: str | None = None # langue ("vf" / "vostfr") quand la source la distingue
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class TitleDetails:
|
|
||||||
source: str
|
|
||||||
source_id: str
|
|
||||||
title: str
|
|
||||||
url: str
|
|
||||||
synopsis: str | None = None
|
|
||||||
image_url: str | None = None
|
|
||||||
banner_url: str | None = None
|
|
||||||
genres: list[str] = field(default_factory=list)
|
|
||||||
rating: float | None = None
|
|
||||||
year: int | None = None
|
|
||||||
episode_count: int | None = None
|
|
||||||
episodes: list[Episode] = field(default_factory=list)
|
|
||||||
media_type: str = "anime"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class VideoLink:
|
|
||||||
"""Lien vidéo résolu (URL directe du fichier, lisible par un lecteur)."""
|
|
||||||
|
|
||||||
url: str
|
|
||||||
hoster: str
|
|
||||||
quality: str | None = None
|
|
||||||
headers: dict[str, str] = field(default_factory=dict) # ex. Referer obligatoire
|
|
||||||
is_hls: bool = False
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- contrats
|
|
||||||
|
|
||||||
|
|
||||||
class SourceScraper(ABC):
|
|
||||||
"""Contrat d'un site catalogue (animes, séries…)."""
|
|
||||||
|
|
||||||
name: str # identifiant unique, ex. "vostfree"
|
|
||||||
label: str # nom affiché
|
|
||||||
base_url: str
|
|
||||||
media_types: tuple[str, ...] = ("anime",)
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
async def search(self, query: str) -> list[SearchResult]: ...
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
async def get_details(self, source_id: str) -> TitleDetails: ...
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
async def list_episodes(self, source_id: str) -> list[Episode]: ...
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
|
||||||
"""URLs des lecteurs embarqués trouvés sur la page d'un épisode."""
|
|
||||||
...
|
|
||||||
|
|
||||||
async def latest(self) -> list[SearchResult]:
|
|
||||||
"""Ajouts récents du catalogue (découverte). Vide si la source ne le supporte pas."""
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
class HosterExtractor(ABC):
|
|
||||||
"""Contrat d'un hébergeur vidéo : embed URL → URL directe."""
|
|
||||||
|
|
||||||
name: str
|
|
||||||
domains: tuple[str, ...] = ()
|
|
||||||
|
|
||||||
def can_handle(self, url: str) -> bool:
|
|
||||||
return any(d in url for d in self.domains)
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink: ...
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- registres
|
|
||||||
|
|
||||||
_source_registry: dict[str, type[SourceScraper]] = {}
|
|
||||||
_hoster_registry: list[type[HosterExtractor]] = []
|
|
||||||
|
|
||||||
|
|
||||||
def register_source(cls: type[SourceScraper]) -> type[SourceScraper]:
|
|
||||||
if cls.name in _source_registry:
|
|
||||||
raise ValueError(f"Source déjà enregistrée : {cls.name}")
|
|
||||||
_source_registry[cls.name] = cls
|
|
||||||
logger.debug("Source enregistrée : %s", cls.name)
|
|
||||||
return cls
|
|
||||||
|
|
||||||
|
|
||||||
def register_hoster(cls: type[HosterExtractor]) -> type[HosterExtractor]:
|
|
||||||
_hoster_registry.append(cls)
|
|
||||||
logger.debug("Hébergeur enregistré : %s", cls.name)
|
|
||||||
return cls
|
|
||||||
|
|
||||||
|
|
||||||
_source_instances: dict[str, SourceScraper] = {}
|
|
||||||
_hoster_instances: list[HosterExtractor] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def get_source(name: str) -> SourceScraper:
|
|
||||||
if name not in _source_instances:
|
|
||||||
cls = _source_registry.get(name)
|
|
||||||
if cls is None:
|
|
||||||
raise ScrapeError(f"Source inconnue : {name}")
|
|
||||||
_source_instances[name] = cls()
|
|
||||||
return _source_instances[name]
|
|
||||||
|
|
||||||
|
|
||||||
def all_sources() -> list[SourceScraper]:
|
|
||||||
return [get_source(name) for name in _source_registry]
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_hoster(url: str) -> HosterExtractor | None:
|
|
||||||
"""Trouve l'extracteur capable de traiter une URL embed (None → générique)."""
|
|
||||||
global _hoster_instances
|
|
||||||
if _hoster_instances is None:
|
|
||||||
_hoster_instances = [cls() for cls in _hoster_registry]
|
|
||||||
for extractor in _hoster_instances:
|
|
||||||
if extractor.can_handle(url):
|
|
||||||
return extractor
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def import_all_scrapers() -> None:
|
|
||||||
"""Importe tous les modules pour déclencher les décorateurs d'enregistrement."""
|
|
||||||
import importlib
|
|
||||||
import pkgutil
|
|
||||||
|
|
||||||
import app.scrapers.hosters as hosters_pkg
|
|
||||||
import app.scrapers.sources as sources_pkg
|
|
||||||
|
|
||||||
for pkg in (sources_pkg, hosters_pkg):
|
|
||||||
for mod in pkgutil.iter_modules(pkg.__path__):
|
|
||||||
importlib.import_module(f"{pkg.__name__}.{mod.name}")
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- format interne
|
|
||||||
|
|
||||||
INTERNAL_SEP = "|"
|
|
||||||
|
|
||||||
|
|
||||||
def encode_internal_url(video_url: str, page_url: str, title: str) -> str:
|
|
||||||
"""Format interne `video_url|page_url|titre` pour transporter le contexte."""
|
|
||||||
for part in (video_url, page_url, title):
|
|
||||||
if INTERNAL_SEP in part:
|
|
||||||
raise ValueError(f"Caractère interdit '{INTERNAL_SEP}' dans : {part!r}")
|
|
||||||
return INTERNAL_SEP.join([video_url, page_url, title])
|
|
||||||
|
|
||||||
|
|
||||||
def decode_internal_url(value: str) -> tuple[str, str, str]:
|
|
||||||
parts = value.split(INTERNAL_SEP)
|
|
||||||
if len(parts) != 3 or not parts[0]:
|
|
||||||
raise ValueError(f"URL interne invalide : {value!r}")
|
|
||||||
return parts[0], parts[1], parts[2]
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
"""Chargement des configurations de scraping externalisées (sélecteurs YAML).
|
|
||||||
|
|
||||||
Permet de réparer un site cassé en modifiant un fichier de config sans toucher
|
|
||||||
au code. Chaque source peut avoir un fichier `configs/<nom_source>.yaml`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from functools import lru_cache
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
@lru_cache
|
|
||||||
def load_scraper_config(source_name: str) -> dict[str, Any]:
|
|
||||||
path = get_settings().scrapers_config_dir / f"{source_name}.yaml"
|
|
||||||
if not path.exists():
|
|
||||||
logger.debug("Pas de config externe pour %s (%s)", source_name, path)
|
|
||||||
return {}
|
|
||||||
try:
|
|
||||||
with path.open(encoding="utf-8") as fh:
|
|
||||||
config = yaml.safe_load(fh) or {}
|
|
||||||
logger.info("Config de scraping chargée pour %s", source_name)
|
|
||||||
return config
|
|
||||||
except yaml.YAMLError as exc:
|
|
||||||
logger.error("Config %s invalide : %s", path, exc)
|
|
||||||
return {}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
# Sélecteurs de scraping French-Manga (DataLife Engine + API AJAX maison)
|
|
||||||
# Structure vérifiée en live sur https://w16.french-manga.net
|
|
||||||
|
|
||||||
search:
|
|
||||||
endpoint: "/engine/ajax/search.php" # POST query=<q>&page=1 (la recherche DLE native est cassée)
|
|
||||||
result: ".search-item" # lien dans l'attribut onclick
|
|
||||||
title: ".search-title"
|
|
||||||
image: ".search-poster img"
|
|
||||||
|
|
||||||
details:
|
|
||||||
config_el: "#serie-config" # data-title / data-news-id
|
|
||||||
poster: ".fposter img"
|
|
||||||
synopsis: ".flist .fdesc p"
|
|
||||||
release: ".facts .release" # année
|
|
||||||
genres: ".facts .genres a"
|
|
||||||
episode_badge: ".short-meta.short-label" # "Ep 32 sur 32"
|
|
||||||
|
|
||||||
episodes:
|
|
||||||
api: "/engine/ajax/manga_episodes_api.php" # ?id=<newsid> → JSON {vf, vostfr, info}
|
|
||||||
versions: ["vostfr", "vf"] # priorité des versions
|
|
||||||
fragment_prefix: "ep" # URL épisode : <fiche>#ep=<version>-<numéro>
|
|
||||||
|
|
||||||
# Nouveautés — section « Récemment mises à jour » de la page d'accueil épinglée
|
|
||||||
latest:
|
|
||||||
path: "/manga-streaming-1/"
|
|
||||||
item: ".short"
|
|
||||||
link: "a.short-poster" # href = index.php?newsid=<id>, alt = titre
|
|
||||||
title: ".short-title"
|
|
||||||
image: "a.short-poster img"
|
|
||||||
@@ -1,60 +0,0 @@
|
|||||||
# Sélecteurs/endpoints French-Stream (french-stream.lat, DataLife Engine) — surchargeables
|
|
||||||
# sans toucher au code. Structure vérifiée en live (films & séries VF/VOSTFR).
|
|
||||||
|
|
||||||
endpoints:
|
|
||||||
search: "/engine/ajax/search.php" # POST query=<q>&page=1
|
|
||||||
episodes: "/ep-data.php?id={newsid}&format=js" # JSON {vf, vostfr, vo, info}
|
|
||||||
film: "/engine/ajax/film_api.php?id={newsid}" # JSON {players: {hoster: {version: url}}}
|
|
||||||
|
|
||||||
search:
|
|
||||||
item: "div.search-item" # bloc résultat (lien dans onclick="location.href='...'")
|
|
||||||
title: ".search-title"
|
|
||||||
poster: ".search-poster img"
|
|
||||||
|
|
||||||
details:
|
|
||||||
title: "h1#s-title"
|
|
||||||
synopsis: "div.fdesc" # le boilerplate p.desc-text est retiré
|
|
||||||
synopsis_boilerplate: "p.desc-text"
|
|
||||||
genres: ".facts .genres"
|
|
||||||
year: ".facts .release"
|
|
||||||
poster_serie: ".fposter img"
|
|
||||||
poster_film: "#film-data" # attribut data-affiche
|
|
||||||
serie_marker: "#serie-config" # présent = fiche série (sinon film)
|
|
||||||
|
|
||||||
latest:
|
|
||||||
path: "/series/" # mix films/séries ; « Saison » dans le titre = série
|
|
||||||
item: "div.short"
|
|
||||||
link: "a.short-poster" # href = fiche (/index.php?newsid=N), alt = titre
|
|
||||||
image: "img"
|
|
||||||
|
|
||||||
browse: # parcours par genre (page Explorer) — chemins vérifiés en live
|
|
||||||
serie: # pages /<genre>-series-/ (9 genres exposés par le site)
|
|
||||||
aventure: {path: "/aventure-series-/", label: "Aventure"}
|
|
||||||
familles: {path: "/familles-series-/", label: "Famille"}
|
|
||||||
fantastique: {path: "/fantastique-series-/", label: "Fantastique"}
|
|
||||||
judiciaire: {path: "/judiciare-series-/", label: "Judiciaire"} # coquille du site
|
|
||||||
medical: {path: "/medical-series-/", label: "Médical"}
|
|
||||||
romance: {path: "/romance-series-/", label: "Romance"}
|
|
||||||
science-fiction: {path: "/science-fiction-series-/", label: "Science-Fiction"}
|
|
||||||
thriller: {path: "/thriller-series-/", label: "Thriller"}
|
|
||||||
western: {path: "/western-series-/", label: "Western"}
|
|
||||||
film: # pages /films/<genre>/
|
|
||||||
actions: {path: "/films/actions/", label: "Action"}
|
|
||||||
animations: {path: "/films/animations/", label: "Animation"}
|
|
||||||
aventures: {path: "/films/aventures/", label: "Aventure"}
|
|
||||||
biopics: {path: "/films/biopics/", label: "Biopic"}
|
|
||||||
comedies: {path: "/films/comedies/", label: "Comédie"}
|
|
||||||
cultes: {path: "/films/cultes/", label: "Culte"}
|
|
||||||
documentaires: {path: "/films/documentaires/", label: "Documentaire"}
|
|
||||||
drames: {path: "/films/drames/", label: "Drame"}
|
|
||||||
epouvante-horreurs: {path: "/films/epouvante-horreurs/", label: "Épouvante-Horreur"}
|
|
||||||
espionnages: {path: "/films/espionnages/", label: "Espionnage"}
|
|
||||||
familles: {path: "/films/familles/", label: "Famille"}
|
|
||||||
fantastiques: {path: "/films/fantastiques/", label: "Fantastique"}
|
|
||||||
guerres: {path: "/films/guerres/", label: "Guerre"}
|
|
||||||
historiques: {path: "/films/historiques/", label: "Historique"}
|
|
||||||
policiers: {path: "/films/policiers/", label: "Policier"}
|
|
||||||
romances: {path: "/films/romances/", label: "Romance"}
|
|
||||||
science-fictions: {path: "/films/science-fictions/", label: "Science-Fiction"}
|
|
||||||
thrillers: {path: "/films/thrillers/", label: "Thriller"}
|
|
||||||
westerns: {path: "/films/westerns/", label: "Western"}
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
# Sélecteurs de scraping Vostfree (DataLife Engine) — surchargeables sans toucher au code.
|
|
||||||
# Structure vérifiée en live sur https://ipv4.vostfree.ws
|
|
||||||
|
|
||||||
search:
|
|
||||||
result: "div.search-result" # bloc d'un résultat de recherche
|
|
||||||
link: "div.title a" # lien + titre
|
|
||||||
image: "span.image img" # poster
|
|
||||||
genres: "ul.additional li" # "Genre:...", "Anneé:..." (détection film)
|
|
||||||
|
|
||||||
details:
|
|
||||||
title: "h1"
|
|
||||||
poster: ".slide-poster img"
|
|
||||||
synopsis: ".slide-desc" # les .cast internes sont retirés
|
|
||||||
genres: '.slide-top li.right a[href*="/genre/"]'
|
|
||||||
episode_badge: ".slide-poster .year" # ex. "Episode 293"
|
|
||||||
season_li: "ul.slide-top li" # ex. "Saison: 01"
|
|
||||||
|
|
||||||
episodes:
|
|
||||||
option: "select.new_player_selector option" # value="buttons_N" → Episode NN
|
|
||||||
button: "div.button_box" # repli si pas de sélecteur
|
|
||||||
content_prefix: "content_" # #player_M → #content_player_M
|
|
||||||
|
|
||||||
# Nouveautés — page « Animes VOSTFR récemment ajoutés »
|
|
||||||
latest:
|
|
||||||
path: "/animes-vostfr-recement-ajoutees.html"
|
|
||||||
item: "div.movie-poster"
|
|
||||||
link: ".play a" # href = fiche, alt = titre
|
|
||||||
image: "span.image img"
|
|
||||||
# class CSS du lecteur → template d'URL ({} = valeur de #content_player_M)
|
|
||||||
# chaîne vide = la valeur est déjà une URL complète
|
|
||||||
players:
|
|
||||||
new_player_vip: ""
|
|
||||||
new_player_moevideo: ""
|
|
||||||
new_player_sibnet: "https://video.sibnet.ru/shell.php?videoid={}"
|
|
||||||
new_player_netu: "https://video.sibnet.ru/shell.php?videoid={}"
|
|
||||||
new_player_uqload: "https://uqload.com/embed-{}.html"
|
|
||||||
new_player_mp4: "https://www.mp4upload.com/embed-{}.html"
|
|
||||||
new_player_fembed: "https://www.fembed.com/v/{}"
|
|
||||||
new_player_mytv: "https://www.myvi.top/embed/{}"
|
|
||||||
new_player_myvi: "https://myvi.ru/player/embed/html/{}"
|
|
||||||
new_player_rutube: "https://rutube.ru/play/embed/{}"
|
|
||||||
new_player_ok: "https://ok.ru/video/{}"
|
|
||||||
new_player_mail2: "https://my.mail.ru/video/embed/{}"
|
|
||||||
new_player_rapids: "https://rapidstream.co/embed-{}.html"
|
|
||||||
new_player_gtv: "https://iframedream.com/embed/{}.html"
|
|
||||||
new_player_cloudvideo: "https://cloudvideo.tv/embed-{}.html"
|
|
||||||
new_player_uptostream: "https://uptostream.com/iframe/{}"
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
"""Décompresseur pour le JavaScript packé « p,a,c,k,e,d » (Dean Edwards).
|
|
||||||
|
|
||||||
Utilisé par plusieurs hébergeurs vidéo (Uqload, VidMoly…) pour masquer
|
|
||||||
l'URL du lecteur : le HTML contient `eval(function(p,a,c,k,e,d){...}(...))`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
_DIGITS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
|
||||||
|
|
||||||
_PACKED_RE = re.compile(
|
|
||||||
r"\}\('(?P<payload>.*?)',(?P<a>\d+),(?P<c>\d+),'(?P<keys>.*?)'\.split\('\|'\)",
|
|
||||||
re.DOTALL,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _base_n(number: int, base: int) -> str:
|
|
||||||
if number == 0:
|
|
||||||
return "0"
|
|
||||||
chars = []
|
|
||||||
while number:
|
|
||||||
number, rest = divmod(number, base)
|
|
||||||
chars.append(_DIGITS[rest])
|
|
||||||
return "".join(reversed(chars))
|
|
||||||
|
|
||||||
|
|
||||||
def unpack_packed_js(text: str) -> str | None:
|
|
||||||
"""Retourne le JS dépacké si `text` contient un bloc packé, sinon None."""
|
|
||||||
match = _PACKED_RE.search(text)
|
|
||||||
if not match:
|
|
||||||
return None
|
|
||||||
payload = match.group("payload").replace("\\\\", "\\").replace("\\'", "'")
|
|
||||||
base, count = int(match.group("a")), int(match.group("c"))
|
|
||||||
keys = match.group("keys").split("|")
|
|
||||||
for index in range(count - 1, -1, -1):
|
|
||||||
token = _base_n(index, base)
|
|
||||||
if keys[index]:
|
|
||||||
replacement = keys[index].replace("\\", "\\\\")
|
|
||||||
payload = re.sub(rf"\b{re.escape(token)}\b", replacement, payload)
|
|
||||||
return payload
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
"""Extracteur Luluvid/Luluvdo (luluvdo.com, luluvid.com, lulustream.com) — embed → m3u8.
|
|
||||||
|
|
||||||
Famille StreamSB : la page embed configure jwplayer dans du JS packé
|
|
||||||
(p,a,c,k,e,d) avec `sources:[{file:"https://.../master.m3u8?t=<token>"}]`.
|
|
||||||
Le token CDN (tnmr.org…) est lié à l'IP **et à l'User-Agent** de la requête
|
|
||||||
d'embed : la VideoLink doit donc renvoyer le User-Agent du socle pour que le
|
|
||||||
téléchargement passe — sans lui, le CDN répond 403.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
|
||||||
from app.scrapers.hosters._packer import unpack_packed_js
|
|
||||||
from app.scrapers.http import fetch
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_PATTERNS = (
|
|
||||||
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?file\s*:\s*["\'](?P<url>[^"\']+\.m3u8[^"\']*)["\']'),
|
|
||||||
re.compile(r'file\s*:\s*["\'](?P<url>https?://[^"\']+?\.m3u8[^"\']*)["\']'),
|
|
||||||
re.compile(r'["\'](?P<url>https?://[^"\']*?\.m3u8(?:\?[^"\']*)?)["\']'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@register_hoster
|
|
||||||
class LuluvdoExtractor(HosterExtractor):
|
|
||||||
name = "luluvdo"
|
|
||||||
domains = ("luluvdo.com", "luluvid.com", "lulustream.com")
|
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
|
||||||
html = await fetch(embed_url)
|
|
||||||
for content in (html, unpack_packed_js(html) or ""):
|
|
||||||
for pattern in _PATTERNS:
|
|
||||||
for match in pattern.finditer(content):
|
|
||||||
url = match.group("url")
|
|
||||||
if url.startswith("http"):
|
|
||||||
return VideoLink(
|
|
||||||
url=url,
|
|
||||||
hoster=self.name,
|
|
||||||
headers={
|
|
||||||
"Referer": f"https://{urlparse(embed_url).hostname}/",
|
|
||||||
"User-Agent": get_settings().user_agent,
|
|
||||||
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
|
||||||
},
|
|
||||||
is_hls=".m3u8" in url,
|
|
||||||
)
|
|
||||||
raise ScrapeError(f"luluvdo : URL vidéo introuvable dans {embed_url}")
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
"""Extracteur SendVid (sendvid.com) — page embed → mp4 direct.
|
|
||||||
|
|
||||||
La page embed expose soit une balise `<source src="...">` ( lecteur HTML5),
|
|
||||||
soit une variable JS `video_source`. Extraction par regex, sans dépendance JS.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
|
||||||
from app.scrapers.hosters._packer import unpack_packed_js
|
|
||||||
from app.scrapers.http import fetch
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_PATTERNS = (
|
|
||||||
re.compile(r'<source[^>]+src=["\'](?P<url>[^"\']+)["\']'),
|
|
||||||
re.compile(r'video_source\s*[:=]\s*["\'](?P<url>[^"\']+)["\']'),
|
|
||||||
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:mp4|m3u8)(?:\?[^"\']*)?)["\']'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@register_hoster
|
|
||||||
class SendvidExtractor(HosterExtractor):
|
|
||||||
name = "sendvid"
|
|
||||||
domains = ("sendvid.com",)
|
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
|
||||||
html = await fetch(embed_url)
|
|
||||||
candidates: list[str] = []
|
|
||||||
for content in (html, unpack_packed_js(html) or ""):
|
|
||||||
for pattern in _PATTERNS:
|
|
||||||
for match in pattern.finditer(content):
|
|
||||||
candidates.append(match.group("url"))
|
|
||||||
url = self._pick(candidates)
|
|
||||||
if not url:
|
|
||||||
raise ScrapeError(f"sendvid : URL vidéo introuvable dans {embed_url}")
|
|
||||||
return VideoLink(
|
|
||||||
url=url,
|
|
||||||
hoster=self.name,
|
|
||||||
headers={"Referer": embed_url},
|
|
||||||
is_hls=".m3u8" in url,
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _pick(candidates: list[str]) -> str | None:
|
|
||||||
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
|
|
||||||
for url in video:
|
|
||||||
if ".mp4" in url:
|
|
||||||
return url
|
|
||||||
return video[0] if video else None
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
"""Extracteur Sibnet (video.sibnet.ru) — page shell/watch → mp4 direct.
|
|
||||||
|
|
||||||
La page embed `shell.php?videoid=N` (ou la page publique `/videoN`) contient
|
|
||||||
une config jwplayer du type `player.src([{src: "/v/<hash>/<id>.mp4"}])`.
|
|
||||||
`shell.php` répond 403 depuis certains réseaux : on retombe alors sur la page
|
|
||||||
publique de la vidéo, qui expose la même config.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from urllib.parse import urljoin
|
|
||||||
|
|
||||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
|
||||||
from app.scrapers.http import fetch
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_BASE = "https://video.sibnet.ru"
|
|
||||||
_VIDEO_ID_RE = re.compile(r"(?:videoid=|/video)(\d+)")
|
|
||||||
_MP4_PATTERNS = (
|
|
||||||
re.compile(r'player\.src\(\[\{src:\s*"(?P<url>/v/[^"]+?\.mp4)"'),
|
|
||||||
re.compile(r'["\'](?P<url>/v/[0-9a-f]{32}/\d+\.mp4)["\']'),
|
|
||||||
re.compile(r'["\'](?P<url>https?://[^"\']*?/v/[^"\']+?\.mp4)["\']'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@register_hoster
|
|
||||||
class SibnetExtractor(HosterExtractor):
|
|
||||||
name = "sibnet"
|
|
||||||
domains = ("video.sibnet.ru",)
|
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
|
||||||
match = _VIDEO_ID_RE.search(embed_url)
|
|
||||||
if not match:
|
|
||||||
raise ScrapeError(f"sibnet : identifiant vidéo introuvable dans {embed_url}")
|
|
||||||
video_id = match.group(1)
|
|
||||||
watch_url = f"{_BASE}/video{video_id}"
|
|
||||||
|
|
||||||
pages: list[tuple[str, str]] = []
|
|
||||||
try:
|
|
||||||
pages.append(("embed", await fetch(embed_url, retries=0)))
|
|
||||||
except ScrapeError as exc:
|
|
||||||
logger.info(
|
|
||||||
"sibnet : page embed %s inaccessible (%s), essai page publique", embed_url, exc
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
pages.append(("watch", await fetch(watch_url, retries=1)))
|
|
||||||
except ScrapeError as exc:
|
|
||||||
logger.error("sibnet : page publique %s inaccessible : %s", watch_url, exc)
|
|
||||||
|
|
||||||
for source, html in pages:
|
|
||||||
url = self._find_mp4(html, source)
|
|
||||||
if url:
|
|
||||||
return VideoLink(
|
|
||||||
url=url,
|
|
||||||
hoster=self.name,
|
|
||||||
headers={"Referer": watch_url},
|
|
||||||
is_hls=False,
|
|
||||||
)
|
|
||||||
raise ScrapeError(f"sibnet : URL mp4 introuvable pour la vidéo {video_id}")
|
|
||||||
|
|
||||||
def _find_mp4(self, html: str, source: str) -> str | None:
|
|
||||||
for pattern in _MP4_PATTERNS:
|
|
||||||
match = pattern.search(html)
|
|
||||||
if match:
|
|
||||||
url = urljoin(_BASE + "/", match.group("url"))
|
|
||||||
logger.debug("sibnet : mp4 trouvé via %s → %s", source, url)
|
|
||||||
return url
|
|
||||||
return None
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
"""Extracteur Uqload (uqload.to/.co/.com/.io/.vc) — page embed → mp4/m3u8.
|
|
||||||
|
|
||||||
La page embed contient un jwplayer configuré dans du JS packé
|
|
||||||
(p,a,c,k,e,d) : `sources:[{file:"https://.../master.m3u8?..."}]`.
|
|
||||||
On dépacke puis on extrait par regex.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
|
||||||
from app.scrapers.hosters._packer import unpack_packed_js
|
|
||||||
from app.scrapers.http import fetch
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_PATTERNS = (
|
|
||||||
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?(?:file|src)\s*:\s*["\'](?P<url>[^"\']+)["\']'),
|
|
||||||
re.compile(r'(?:file|src)\s*:\s*["\'](?P<url>https?://[^"\']+?\.(?:m3u8|mp4)[^"\']*)["\']'),
|
|
||||||
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:m3u8|mp4)(?:\?[^"\']*)?)["\']'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@register_hoster
|
|
||||||
class UqloadExtractor(HosterExtractor):
|
|
||||||
name = "uqload"
|
|
||||||
domains = ("uqload.to", "uqload.co", "uqload.com", "uqload.io", "uqload.vc")
|
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
|
||||||
html = await fetch(embed_url)
|
|
||||||
candidates: list[str] = []
|
|
||||||
for content in (html, unpack_packed_js(html) or ""):
|
|
||||||
for pattern in _PATTERNS:
|
|
||||||
for match in pattern.finditer(content):
|
|
||||||
candidates.append(match.group("url"))
|
|
||||||
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
|
|
||||||
url = next((u for u in video if ".mp4" in u), None) or (video[0] if video else None)
|
|
||||||
if not url:
|
|
||||||
raise ScrapeError(f"uqload : URL vidéo introuvable dans {embed_url}")
|
|
||||||
return VideoLink(
|
|
||||||
url=url,
|
|
||||||
hoster=self.name,
|
|
||||||
headers={"Referer": embed_url},
|
|
||||||
is_hls=".m3u8" in url,
|
|
||||||
)
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
"""Extracteur VidMoly (vidmoly.to / vidmoly.me) — page embed → mp4/m3u8.
|
|
||||||
|
|
||||||
Le lecteur jwplayer est configuré dans du JS parfois packé (p,a,c,k,e,d) :
|
|
||||||
`sources: [{file: "...m3u8"}]`. On dépacke si besoin puis on extrait par regex.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
|
||||||
from app.scrapers.hosters._packer import unpack_packed_js
|
|
||||||
from app.scrapers.http import fetch
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_PATTERNS = (
|
|
||||||
re.compile(r'sources\s*:\s*\[\s*\{[^}]*?file\s*:\s*["\'](?P<url>[^"\']+)["\']'),
|
|
||||||
re.compile(r'file\s*:\s*["\'](?P<url>https?://[^"\']+?\.(?:m3u8|mp4)[^"\']*)["\']'),
|
|
||||||
re.compile(r'["\'](?P<url>https?://[^"\']*?\.(?:m3u8|mp4)(?:\?[^"\']*)?)["\']'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@register_hoster
|
|
||||||
class VidmolyExtractor(HosterExtractor):
|
|
||||||
name = "vidmoly"
|
|
||||||
domains = ("vidmoly.to", "vidmoly.me", "vidmoly.biz", "vidmoly.com")
|
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
|
||||||
html = await fetch(embed_url)
|
|
||||||
candidates: list[str] = []
|
|
||||||
for content in (html, unpack_packed_js(html) or ""):
|
|
||||||
for pattern in _PATTERNS:
|
|
||||||
for match in pattern.finditer(content):
|
|
||||||
candidates.append(match.group("url"))
|
|
||||||
video = [u for u in candidates if re.search(r"\.(mp4|m3u8)(\?|$)", u)]
|
|
||||||
url = next((u for u in video if ".mp4" in u), None) or (video[0] if video else None)
|
|
||||||
if not url:
|
|
||||||
raise ScrapeError(f"vidmoly : URL vidéo introuvable dans {embed_url}")
|
|
||||||
return VideoLink(
|
|
||||||
url=url,
|
|
||||||
hoster=self.name,
|
|
||||||
headers={"Referer": embed_url},
|
|
||||||
is_hls=".m3u8" in url,
|
|
||||||
)
|
|
||||||
@@ -1,78 +0,0 @@
|
|||||||
"""Extracteur Vidzy (vidzy.org/.cc/.live) — page embed videojs → m3u8.
|
|
||||||
|
|
||||||
La page embed ne contient pas l'URL vidéo en clair : le script videojs appelle
|
|
||||||
une fonction de décodage inline `atob(s)` + reverse + XOR, avec une graine
|
|
||||||
dérivée du hostname (`somme des codes des caractères & 0xFF`). Si le décodage
|
|
||||||
échoue, la page sert un leurre `https://s1.fsvid.lol/troll/master.m3u8`
|
|
||||||
(même valeur pour tous les épisodes) — on le rejette explicitement.
|
|
||||||
|
|
||||||
Algorithme (reproduit fidèlement depuis le JS de la page) :
|
|
||||||
b = base64decode(s) ; a = b[::-1]
|
|
||||||
r[i] = chr(a[i] ^ ((0x3D + i*89 + H) & 0xFF)) avec H = sum(ord(hostname)) & 0xFF
|
|
||||||
"""
|
|
||||||
|
|
||||||
import base64
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
|
||||||
from app.scrapers.http import fetch
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_B64_RE = re.compile(
|
|
||||||
r"atob\(s\)(?s:.){0,800}?\}\)\(\"(?P<b64>[A-Za-z0-9+/=]{50,})\"",
|
|
||||||
)
|
|
||||||
_CLEAR_PATTERNS = (
|
|
||||||
re.compile(r'sources\s*:\s*\[\s*\{\s*src\s*:\s*["\'](?P<url>https?://[^"\']+?\.m3u8[^"\']*)["\']'),
|
|
||||||
re.compile(r'_fsvHls\s*=\s*"(?P<url>https?://[^"]+?\.m3u8[^"]*)"'),
|
|
||||||
)
|
|
||||||
_TROLL = "/troll/"
|
|
||||||
|
|
||||||
|
|
||||||
def _decode(b64: str, hostname: str) -> str | None:
|
|
||||||
try:
|
|
||||||
data = base64.b64decode(b64)
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
return None
|
|
||||||
seed = sum(ord(c) for c in hostname) & 0xFF
|
|
||||||
decoded = "".join(chr(b ^ ((0x3D + i * 89 + seed) & 0xFF)) for i, b in enumerate(data[::-1]))
|
|
||||||
return decoded if decoded.startswith(("http://", "https://")) else None
|
|
||||||
|
|
||||||
|
|
||||||
@register_hoster
|
|
||||||
class VidzyExtractor(HosterExtractor):
|
|
||||||
name = "vidzy"
|
|
||||||
domains = ("vidzy.org", "vidzy.cc", "vidzy.live")
|
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
|
||||||
html = await fetch(embed_url)
|
|
||||||
hostname = urlparse(embed_url).hostname or ""
|
|
||||||
|
|
||||||
for match in _B64_RE.finditer(html):
|
|
||||||
url = _decode(match.group("b64"), hostname)
|
|
||||||
if url and _TROLL not in url:
|
|
||||||
return self._video_link(url, embed_url)
|
|
||||||
logger.warning("vidzy : décodage atob+XOR sans résultat pour %s", embed_url)
|
|
||||||
|
|
||||||
for pattern in _CLEAR_PATTERNS:
|
|
||||||
for match in pattern.finditer(html):
|
|
||||||
url = match.group("url")
|
|
||||||
if _TROLL not in url:
|
|
||||||
return self._video_link(url, embed_url)
|
|
||||||
raise ScrapeError(
|
|
||||||
f"vidzy : URL vidéo introuvable dans {embed_url} (leurre anti-bot ou page modifiée)"
|
|
||||||
)
|
|
||||||
|
|
||||||
def _video_link(self, url: str, embed_url: str) -> VideoLink:
|
|
||||||
return VideoLink(
|
|
||||||
url=url,
|
|
||||||
hoster=self.name,
|
|
||||||
headers={
|
|
||||||
"Referer": f"https://{urlparse(embed_url).hostname}/",
|
|
||||||
"User-Agent": get_settings().user_agent,
|
|
||||||
},
|
|
||||||
is_hls=".m3u8" in url,
|
|
||||||
)
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
"""Extracteur VoirAnime — endpoint « prepare » → URL lisible (directe ou proxy site).
|
|
||||||
|
|
||||||
Chaque source d'un épisode voiranime.xyz se résout via
|
|
||||||
`/lecteur/prepare/<source_id>?content=..&episode=..` (AJAX : en-têtes Accept JSON et
|
|
||||||
X-Requested-With obligatoires) qui renvoie :
|
|
||||||
`{"success": true, "media_type": "mp4"|"hls", "stream_url": "/proxy/media?payload=<b64>"}`.
|
|
||||||
|
|
||||||
Le payload base64 est un JSON `{"url": ..., "referer": ..., "kind": ...}` où `url` est
|
|
||||||
l'URL directe chez l'hébergeur d'origine. Deux cas (vérifiés en live) :
|
|
||||||
- mp4 : l'URL d'origine est libre d'accès avec son Referer (Sibnet…) → on la retourne
|
|
||||||
directement (téléchargement/Range natifs, sans double saut).
|
|
||||||
- hls : les CDN utilisés (SmoothPre & co) signent leurs playlists pour le backend du
|
|
||||||
site — accès direct 403. Le proxy `/proxy/media?payload=…` du site, lui, sert la
|
|
||||||
playlist ET ses segments (chemins re-hostés) → on retourne l'URL proxy absolue.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import base64
|
|
||||||
import binascii
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
from urllib.parse import parse_qs, urlparse
|
|
||||||
|
|
||||||
from app.scrapers.base import HosterExtractor, ScrapeError, VideoLink, register_hoster
|
|
||||||
from app.scrapers.http import fetch
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
@register_hoster
|
|
||||||
class VoirAnimeExtractor(HosterExtractor):
|
|
||||||
name = "voiranime"
|
|
||||||
domains = ("voiranime.xyz",)
|
|
||||||
|
|
||||||
async def extract(self, embed_url: str) -> VideoLink:
|
|
||||||
origin = self._origin_of(embed_url)
|
|
||||||
raw = await fetch(
|
|
||||||
embed_url,
|
|
||||||
referer=self._referer_for(embed_url),
|
|
||||||
headers={
|
|
||||||
"Accept": "application/json",
|
|
||||||
"X-Requested-With": "XMLHttpRequest",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
data = json.loads(raw)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ScrapeError(f"voiranime : réponse prepare invalide : {raw[:120]}") from exc
|
|
||||||
if not data.get("success"):
|
|
||||||
raise ScrapeError(f"voiranime : {data.get('error') or 'échec de préparation du lecteur'}")
|
|
||||||
|
|
||||||
media_type = str(data.get("media_type") or "").lower()
|
|
||||||
stream_url = str(data.get("stream_url") or "")
|
|
||||||
|
|
||||||
if media_type == "hls":
|
|
||||||
if not stream_url:
|
|
||||||
raise ScrapeError(f"voiranime : flux HLS sans stream_url dans {embed_url}")
|
|
||||||
return VideoLink(
|
|
||||||
url=f"{origin}/{stream_url.lstrip('/')}",
|
|
||||||
hoster=self.name,
|
|
||||||
headers={"Referer": f"{origin}/"},
|
|
||||||
is_hls=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
payload = self._extract_payload(stream_url)
|
|
||||||
url = str(payload.get("url") or "")
|
|
||||||
if not url:
|
|
||||||
raise ScrapeError(f"voiranime : payload sans URL de flux dans {embed_url}")
|
|
||||||
referer = str(payload.get("referer") or "")
|
|
||||||
return VideoLink(
|
|
||||||
url=url,
|
|
||||||
hoster=self.name,
|
|
||||||
headers={"Referer": referer} if referer else {},
|
|
||||||
is_hls=url.split("?")[0].endswith(".m3u8"),
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _origin_of(embed_url: str) -> str:
|
|
||||||
parsed = urlparse(embed_url)
|
|
||||||
return f"{parsed.scheme}://{parsed.netloc}"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _referer_for(embed_url: str) -> str:
|
|
||||||
"""Reconstruit la page lecteur d'où provient la requête prepare."""
|
|
||||||
origin = VoirAnimeExtractor._origin_of(embed_url)
|
|
||||||
query = parse_qs(urlparse(embed_url).query)
|
|
||||||
content = query.get("content", [""])[0]
|
|
||||||
episode = query.get("episode", [""])[0]
|
|
||||||
if content and episode:
|
|
||||||
return f"{origin}/lecteur/{content}/{episode}"
|
|
||||||
return f"{origin}/"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _extract_payload(stream_url: str) -> dict:
|
|
||||||
payload_b64 = parse_qs(urlparse(stream_url).query).get("payload", [""])[0]
|
|
||||||
if not payload_b64:
|
|
||||||
raise ScrapeError(f"voiranime : pas de payload dans stream_url ({stream_url[:80]})")
|
|
||||||
padded = payload_b64 + "=" * (-len(payload_b64) % 4)
|
|
||||||
try:
|
|
||||||
decoded = base64.b64decode(padded)
|
|
||||||
except (binascii.Error, ValueError):
|
|
||||||
try:
|
|
||||||
decoded = base64.urlsafe_b64decode(padded)
|
|
||||||
except (binascii.Error, ValueError) as exc:
|
|
||||||
raise ScrapeError(f"voiranime : payload illisible : {payload_b64[:60]}") from exc
|
|
||||||
try:
|
|
||||||
data = json.loads(decoded)
|
|
||||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
|
||||||
raise ScrapeError(f"voiranime : payload non-JSON : {payload_b64[:60]}") from exc
|
|
||||||
if not isinstance(data, dict):
|
|
||||||
raise ScrapeError("voiranime : payload inattendu (pas un objet JSON)")
|
|
||||||
return data
|
|
||||||
@@ -1,78 +0,0 @@
|
|||||||
"""Client HTTP partagé pour le scraping (httpx async, headers navigateur, retries)."""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.scrapers.base import ScrapeError
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_client: httpx.AsyncClient | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def get_client() -> httpx.AsyncClient:
|
|
||||||
global _client
|
|
||||||
if _client is None:
|
|
||||||
settings = get_settings()
|
|
||||||
_client = httpx.AsyncClient(
|
|
||||||
timeout=settings.http_timeout,
|
|
||||||
follow_redirects=True,
|
|
||||||
headers={
|
|
||||||
"User-Agent": settings.user_agent,
|
|
||||||
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
return _client
|
|
||||||
|
|
||||||
|
|
||||||
async def close_client() -> None:
|
|
||||||
global _client
|
|
||||||
if _client is not None:
|
|
||||||
await _client.aclose()
|
|
||||||
_client = None
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch(
|
|
||||||
url: str,
|
|
||||||
*,
|
|
||||||
referer: str | None = None,
|
|
||||||
retries: int = 2,
|
|
||||||
headers: dict[str, str] | None = None,
|
|
||||||
data: dict[str, str] | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""GET (ou POST si `data` est fourni) avec retries ; lève ScrapeError en cas d'échec définitif."""
|
|
||||||
request_headers = dict(headers) if headers else {}
|
|
||||||
if referer:
|
|
||||||
request_headers.setdefault("Referer", referer)
|
|
||||||
last_error: Exception | None = None
|
|
||||||
for attempt in range(retries + 1):
|
|
||||||
try:
|
|
||||||
if data is None:
|
|
||||||
response = await get_client().get(url, headers=request_headers)
|
|
||||||
else:
|
|
||||||
response = await get_client().post(url, headers=request_headers, data=data)
|
|
||||||
response.raise_for_status()
|
|
||||||
return response.text
|
|
||||||
except httpx.HTTPStatusError as exc:
|
|
||||||
status = exc.response.status_code
|
|
||||||
if 400 <= status < 500 and status not in (408, 429):
|
|
||||||
raise ScrapeError(f"Échec de récupération de {url} : HTTP {status} (définitif)") from exc
|
|
||||||
last_error = exc
|
|
||||||
logger.warning("fetch %s — HTTP %d, tentative %d/%d", url, status, attempt + 1, retries + 1)
|
|
||||||
if attempt < retries:
|
|
||||||
await asyncio.sleep(1.0 * (attempt + 1))
|
|
||||||
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
|
||||||
last_error = exc
|
|
||||||
logger.warning("fetch %s — tentative %d/%d : %s", url, attempt + 1, retries + 1, exc)
|
|
||||||
if attempt < retries:
|
|
||||||
await asyncio.sleep(1.0 * (attempt + 1))
|
|
||||||
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_soup(url: str, *, referer: str | None = None) -> BeautifulSoup:
|
|
||||||
html = await fetch(url, referer=referer)
|
|
||||||
return BeautifulSoup(html, "lxml")
|
|
||||||
@@ -1,312 +0,0 @@
|
|||||||
"""Source French-Manga (w16.french-manga.net) — animes VF/VOSTFR, moteur DataLife Engine.
|
|
||||||
|
|
||||||
Faits structurels (vérifiés en live) :
|
|
||||||
- La recherche native DLE est cassée (erreur MySQL en GET, « moins de 4 caractères » en POST) ;
|
|
||||||
le site utilise un endpoint AJAX : POST /engine/ajax/search.php avec `query=<q>&page=1`
|
|
||||||
→ blocs `.search-item` (lien dans l'attribut onclick).
|
|
||||||
- Nouveautés : /manga-streaming-1/ (« Récemment mises à jour ») → blocs `.short`
|
|
||||||
(lien `a.short-poster` href = `index.php?newsid=<id>`, titre `.short-title`, poster img).
|
|
||||||
- Fiche : `/<id>-<slug>.html` (alias canonique `index.php?newsid=<id>`), métadonnées dans
|
|
||||||
`.facts`, `.fdesc`, `.fposter` et `#serie-config` (data-title, data-news-id).
|
|
||||||
- Épisodes et lecteurs : GET /engine/ajax/manga_episodes_api.php?id=<newsid> → JSON
|
|
||||||
{"vf": {ep: {hoster: url}}, "vostfr": {...}, "info": {ep: {title, poster}}}.
|
|
||||||
- Pas de page par épisode : l'URL d'épisode est `<fiche>#ep=<version>-<numéro>`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from copy import deepcopy
|
|
||||||
from urllib.parse import urljoin
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
from bs4 import BeautifulSoup, Tag
|
|
||||||
|
|
||||||
from app.scrapers.base import (
|
|
||||||
Episode,
|
|
||||||
ScrapeError,
|
|
||||||
SearchResult,
|
|
||||||
SourceScraper,
|
|
||||||
TitleDetails,
|
|
||||||
register_source,
|
|
||||||
)
|
|
||||||
from app.scrapers.config_loader import load_scraper_config
|
|
||||||
from app.scrapers.http import fetch, fetch_soup, get_client
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
DEFAULT_CONFIG: dict = {
|
|
||||||
"search": {
|
|
||||||
"endpoint": "/engine/ajax/search.php",
|
|
||||||
"result": ".search-item",
|
|
||||||
"title": ".search-title",
|
|
||||||
"image": ".search-poster img",
|
|
||||||
},
|
|
||||||
"details": {
|
|
||||||
"config_el": "#serie-config",
|
|
||||||
"poster": ".fposter img",
|
|
||||||
"synopsis": ".flist .fdesc p",
|
|
||||||
"release": ".facts .release",
|
|
||||||
"genres": ".facts .genres a",
|
|
||||||
"episode_badge": ".short-meta.short-label",
|
|
||||||
"episodes": {
|
|
||||||
"api": "/engine/ajax/manga_episodes_api.php",
|
|
||||||
"versions": ["vostfr", "vf"],
|
|
||||||
"fragment_prefix": "ep",
|
|
||||||
},
|
|
||||||
"latest": {
|
|
||||||
"path": "/manga-streaming-1/", # « Récemment mises à jour »
|
|
||||||
"item": ".short",
|
|
||||||
"link": "a.short-poster", # href = index.php?newsid=<id>, alt = titre
|
|
||||||
"title": ".short-title",
|
|
||||||
"image": "a.short-poster img",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_ID_RE = re.compile(r"newsid=(\d+)|/(\d+)-[^/]+\.html?")
|
|
||||||
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
|
|
||||||
_SEASON_RE = re.compile(r"Saison\s+(\d+)", re.IGNORECASE)
|
|
||||||
_YEAR_RE = re.compile(r"\((\d{4})\)\s*$")
|
|
||||||
|
|
||||||
|
|
||||||
def _merged_config() -> dict:
|
|
||||||
merged = deepcopy(DEFAULT_CONFIG)
|
|
||||||
for key, values in load_scraper_config("french_manga").items():
|
|
||||||
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
|
||||||
merged[key].update(values)
|
|
||||||
else:
|
|
||||||
merged[key] = values
|
|
||||||
return merged
|
|
||||||
|
|
||||||
|
|
||||||
@register_source
|
|
||||||
class FrenchMangaScraper(SourceScraper):
|
|
||||||
name = "french_manga"
|
|
||||||
label = "French-Manga"
|
|
||||||
base_url = "https://w16.french-manga.net"
|
|
||||||
media_types = ("anime",)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- helpers
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _id_from_url(url: str) -> str | None:
|
|
||||||
match = _ID_RE.search(url)
|
|
||||||
if not match:
|
|
||||||
return None
|
|
||||||
return match.group(1) or match.group(2)
|
|
||||||
|
|
||||||
def _title_url(self, source_id: str) -> str:
|
|
||||||
return f"{self.base_url}/index.php?newsid={source_id}"
|
|
||||||
|
|
||||||
async def _post(self, url: str, data: dict[str, str]) -> str:
|
|
||||||
"""POST avec retries (fetch du socle est GET uniquement)."""
|
|
||||||
last_error: Exception | None = None
|
|
||||||
for attempt in range(3):
|
|
||||||
try:
|
|
||||||
response = await get_client().post(url, data=data)
|
|
||||||
response.raise_for_status()
|
|
||||||
return response.text
|
|
||||||
except (httpx.HTTPError, httpx.InvalidURL) as exc:
|
|
||||||
last_error = exc
|
|
||||||
logger.warning("french_manga POST %s — tentative %d/3 : %s", url, attempt + 1, exc)
|
|
||||||
await asyncio.sleep(1.0 * (attempt + 1))
|
|
||||||
raise ScrapeError(f"Échec de récupération de {url} : {last_error}")
|
|
||||||
|
|
||||||
async def _fetch_episodes_api(self, source_id: str) -> dict:
|
|
||||||
config = _merged_config()
|
|
||||||
api_url = urljoin(self.base_url + "/", config["episodes"]["api"])
|
|
||||||
html = await fetch(f"{api_url}?id={source_id}", referer=self._title_url(source_id))
|
|
||||||
try:
|
|
||||||
data = json.loads(html)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ScrapeError(
|
|
||||||
f"french_manga : réponse JSON invalide de l'API épisodes ({source_id}) : {exc}"
|
|
||||||
) from exc
|
|
||||||
if not isinstance(data, dict):
|
|
||||||
raise ScrapeError(f"french_manga : réponse inattendue de l'API épisodes ({source_id})")
|
|
||||||
return data
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- search
|
|
||||||
|
|
||||||
async def search(self, query: str) -> list[SearchResult]:
|
|
||||||
config = _merged_config()
|
|
||||||
search_cfg = config["search"]
|
|
||||||
endpoint = urljoin(self.base_url + "/", search_cfg["endpoint"])
|
|
||||||
html = await self._post(endpoint, {"query": query, "page": "1"})
|
|
||||||
soup = BeautifulSoup(html, "lxml")
|
|
||||||
results: list[SearchResult] = []
|
|
||||||
for item in soup.select(search_cfg["result"]):
|
|
||||||
onclick = item.get("onclick", "")
|
|
||||||
match = re.search(r"location\.href='([^']+)'", onclick)
|
|
||||||
if not match:
|
|
||||||
logger.warning("french_manga : search-item sans lien (onclick=%r)", onclick[:80])
|
|
||||||
continue
|
|
||||||
url = urljoin(self.base_url + "/", match.group(1))
|
|
||||||
source_id = self._id_from_url(url)
|
|
||||||
if not source_id:
|
|
||||||
logger.warning("french_manga : identifiant introuvable dans %s", url)
|
|
||||||
continue
|
|
||||||
title_el = item.select_one(search_cfg["title"])
|
|
||||||
title = title_el.get_text(strip=True) if title_el else url
|
|
||||||
image = item.select_one(search_cfg["image"])
|
|
||||||
results.append(
|
|
||||||
SearchResult(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=_YEAR_RE.sub("", title).strip() or title,
|
|
||||||
url=url,
|
|
||||||
image_url=image.get("src") if image else None,
|
|
||||||
media_type="film" if re.search(r"\bfilm\b", title, re.IGNORECASE) else "anime",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
logger.info("french_manga : %d résultats pour %r", len(results), query)
|
|
||||||
return results
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- latest
|
|
||||||
|
|
||||||
async def latest(self) -> list[SearchResult]:
|
|
||||||
"""Ajouts récents — section « Récemment mises à jour »."""
|
|
||||||
config = _merged_config()
|
|
||||||
latest_cfg = config["latest"]
|
|
||||||
url = urljoin(self.base_url + "/", latest_cfg["path"])
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
results: list[SearchResult] = []
|
|
||||||
for item in soup.select(latest_cfg["item"]):
|
|
||||||
link = item.select_one(latest_cfg["link"])
|
|
||||||
href = link.get("href") if link else None
|
|
||||||
if not href:
|
|
||||||
logger.warning("french_manga : bloc nouveauté sans lien, ignoré")
|
|
||||||
continue
|
|
||||||
absolute = urljoin(self.base_url + "/", href)
|
|
||||||
source_id = self._id_from_url(absolute)
|
|
||||||
if not source_id:
|
|
||||||
logger.warning("french_manga : identifiant introuvable dans %s", absolute)
|
|
||||||
continue
|
|
||||||
image = item.select_one(latest_cfg["image"])
|
|
||||||
title_el = item.select_one(latest_cfg["title"])
|
|
||||||
title = title_el.get_text(strip=True) if title_el else (link.get("alt") or absolute)
|
|
||||||
title = _YEAR_RE.sub("", title).strip() or title
|
|
||||||
results.append(
|
|
||||||
SearchResult(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=title,
|
|
||||||
url=self._title_url(source_id),
|
|
||||||
image_url=image.get("src") if image else None,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
logger.info("french_manga : %d nouveautés récupérées", len(results))
|
|
||||||
return results
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- details
|
|
||||||
|
|
||||||
async def get_details(self, source_id: str) -> TitleDetails:
|
|
||||||
config = _merged_config()
|
|
||||||
url = self._title_url(source_id)
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
details_cfg = config["details"]
|
|
||||||
|
|
||||||
config_el = soup.select_one(details_cfg["config_el"])
|
|
||||||
news_id = config_el.get("data-news-id") if config_el else None
|
|
||||||
if news_id != source_id:
|
|
||||||
raise ScrapeError(f"french_manga : fiche introuvable pour {source_id} ({url})")
|
|
||||||
title = config_el.get("data-title") or (
|
|
||||||
soup.title.get_text(strip=True) if soup.title else source_id
|
|
||||||
)
|
|
||||||
|
|
||||||
release = self._text(soup.select_one(details_cfg["release"]))
|
|
||||||
release_match = re.search(r"(\d{4})", release)
|
|
||||||
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
|
|
||||||
badge_match = _NUMBER_RE.search(badge)
|
|
||||||
|
|
||||||
episodes = await self._episodes_from_api(source_id, title, url)
|
|
||||||
|
|
||||||
return TitleDetails(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=title,
|
|
||||||
url=url,
|
|
||||||
synopsis=self._text(soup.select_one(details_cfg["synopsis"])) or None,
|
|
||||||
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
|
|
||||||
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
|
|
||||||
year=int(release_match.group(1)) if release_match else None,
|
|
||||||
episode_count=int(float(badge_match.group(1).replace(",", ".")))
|
|
||||||
if badge_match
|
|
||||||
else None,
|
|
||||||
episodes=episodes,
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _text(element: Tag | None) -> str:
|
|
||||||
return element.get_text(" ", strip=True) if element else ""
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ episodes
|
|
||||||
|
|
||||||
async def list_episodes(self, source_id: str) -> list[Episode]:
|
|
||||||
return await self._episodes_from_api(source_id, None, self._title_url(source_id))
|
|
||||||
|
|
||||||
async def _episodes_from_api(
|
|
||||||
self, source_id: str, title: str | None, page_url: str
|
|
||||||
) -> list[Episode]:
|
|
||||||
config = _merged_config()
|
|
||||||
data = await self._fetch_episodes_api(source_id)
|
|
||||||
versions: list[str] = config["episodes"]["versions"]
|
|
||||||
season = 1
|
|
||||||
if title:
|
|
||||||
season_match = _SEASON_RE.search(title)
|
|
||||||
if season_match:
|
|
||||||
season = int(season_match.group(1))
|
|
||||||
|
|
||||||
info = data.get("info") or {}
|
|
||||||
episodes: list[Episode] = []
|
|
||||||
for version in versions:
|
|
||||||
numbers: set[float] = set()
|
|
||||||
for number_text in data.get(version) or {}:
|
|
||||||
try:
|
|
||||||
numbers.add(float(number_text))
|
|
||||||
except ValueError:
|
|
||||||
logger.warning("french_manga : numéro d'épisode invalide %r", number_text)
|
|
||||||
for number in sorted(numbers):
|
|
||||||
number_text = str(int(number)) if number.is_integer() else str(number)
|
|
||||||
episode_info = info.get(number_text) or info.get(str(int(number)))
|
|
||||||
label = episode_info.get("title") if isinstance(episode_info, dict) else None
|
|
||||||
episodes.append(
|
|
||||||
Episode(
|
|
||||||
number=number,
|
|
||||||
title=label,
|
|
||||||
url=f"{page_url}#{config['episodes']['fragment_prefix']}={version}-{number_text}",
|
|
||||||
season=season,
|
|
||||||
version=version,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if not episodes:
|
|
||||||
raise ScrapeError(f"french_manga : aucun épisode trouvé pour {source_id}")
|
|
||||||
return episodes
|
|
||||||
|
|
||||||
# -------------------------------------------------------------- embeds
|
|
||||||
|
|
||||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
|
||||||
config = _merged_config()
|
|
||||||
page_url, _, fragment = episode_url.partition("#")
|
|
||||||
prefix = config["episodes"]["fragment_prefix"] + "="
|
|
||||||
match = re.match(rf"{re.escape(prefix)}([A-Za-z0-9]+)-([\d.]+)$", fragment)
|
|
||||||
if not match:
|
|
||||||
raise ScrapeError(f"french_manga : fragment d'épisode invalide dans {episode_url}")
|
|
||||||
version, number_text = match.group(1), match.group(2)
|
|
||||||
if number_text.isdigit():
|
|
||||||
number_text = str(int(number_text))
|
|
||||||
|
|
||||||
source_id = self._id_from_url(page_url)
|
|
||||||
if not source_id:
|
|
||||||
raise ScrapeError(f"french_manga : identifiant introuvable dans {episode_url}")
|
|
||||||
data = await self._fetch_episodes_api(source_id)
|
|
||||||
hosters = (data.get(version) or {}).get(number_text)
|
|
||||||
if not hosters:
|
|
||||||
raise ScrapeError(
|
|
||||||
f"french_manga : aucun lecteur pour {version} épisode {number_text} ({episode_url})"
|
|
||||||
)
|
|
||||||
links = list(hosters.values())
|
|
||||||
logger.info("french_manga : %d liens embed pour %s", len(links), episode_url)
|
|
||||||
return links
|
|
||||||
@@ -1,429 +0,0 @@
|
|||||||
"""Source French-Stream (french-stream.lat) — films & séries VF/VOSTFR, moteur DataLife Engine.
|
|
||||||
|
|
||||||
Faits structurels (vérifiés en live) :
|
|
||||||
- Recherche : POST /engine/ajax/search.php (query, page) → blocs `div.search-item`
|
|
||||||
(lien dans onclick="location.href='...'", poster `.search-poster img`).
|
|
||||||
- Fiche : `/index.php?newsid=<id>` (ou la jolie URL `/<id>-<slug>.html`).
|
|
||||||
`source_id` = newsid numérique.
|
|
||||||
- Séries : une fiche par saison (« Titre - Saison N »). Épisodes via
|
|
||||||
GET /ep-data.php?id=<newsid>&format=js → JSON
|
|
||||||
`{"vf": {"1": {"vidzy": url, "uqload": url, ...}}, "vostfr": {...}, "info": {...}}`.
|
|
||||||
- Films : lecteurs via GET /engine/ajax/film_api.php?id=<newsid> → JSON
|
|
||||||
`{"players": {"vidzy": {"default": url, "vff": url, "vostfr": url, ...}}}`.
|
|
||||||
- URL d'épisode interne : `<fiche>#vf-3` (série) ou `<fiche>#film` (film).
|
|
||||||
- Nouveautés : /series/ → blocs `div.short` (mix films/séries, « Saison » dans le
|
|
||||||
titre = série).
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from copy import deepcopy
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup, Tag
|
|
||||||
|
|
||||||
from app.scrapers.base import (
|
|
||||||
Episode,
|
|
||||||
ScrapeError,
|
|
||||||
SearchResult,
|
|
||||||
SourceScraper,
|
|
||||||
TitleDetails,
|
|
||||||
register_source,
|
|
||||||
)
|
|
||||||
from app.scrapers.config_loader import load_scraper_config
|
|
||||||
from app.scrapers.http import fetch, fetch_soup
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
DEFAULT_CONFIG: dict = {
|
|
||||||
"endpoints": {
|
|
||||||
"search": "/engine/ajax/search.php",
|
|
||||||
"episodes": "/ep-data.php?id={newsid}&format=js",
|
|
||||||
"film": "/engine/ajax/film_api.php?id={newsid}",
|
|
||||||
},
|
|
||||||
"search": {
|
|
||||||
"item": "div.search-item",
|
|
||||||
"title": ".search-title",
|
|
||||||
"poster": ".search-poster img",
|
|
||||||
},
|
|
||||||
"details": {
|
|
||||||
"title": "h1#s-title",
|
|
||||||
"synopsis": "div.fdesc",
|
|
||||||
"synopsis_boilerplate": "p.desc-text",
|
|
||||||
"genres": ".facts .genres",
|
|
||||||
"year": ".facts .release",
|
|
||||||
"poster_serie": ".fposter img",
|
|
||||||
"poster_film": "#film-data",
|
|
||||||
"serie_marker": "#serie-config",
|
|
||||||
},
|
|
||||||
"latest": {
|
|
||||||
"path": "/series/",
|
|
||||||
"item": "div.short",
|
|
||||||
"link": "a.short-poster",
|
|
||||||
"image": "img",
|
|
||||||
},
|
|
||||||
# Parcours par genre (page Explorer) : type → clé → {path, label}.
|
|
||||||
# Chemins vérifiés en live : films = /films/<genre>/, séries = /<genre>-series-/.
|
|
||||||
"browse": {
|
|
||||||
"serie": {
|
|
||||||
"aventure": {"path": "/aventure-series-/", "label": "Aventure"},
|
|
||||||
"familles": {"path": "/familles-series-/", "label": "Famille"},
|
|
||||||
"fantastique": {"path": "/fantastique-series-/", "label": "Fantastique"},
|
|
||||||
"judiciaire": {"path": "/judiciare-series-/", "label": "Judiciaire"}, # coquille du site
|
|
||||||
"medical": {"path": "/medical-series-/", "label": "Médical"},
|
|
||||||
"romance": {"path": "/romance-series-/", "label": "Romance"},
|
|
||||||
"science-fiction": {"path": "/science-fiction-series-/", "label": "Science-Fiction"},
|
|
||||||
"thriller": {"path": "/thriller-series-/", "label": "Thriller"},
|
|
||||||
"western": {"path": "/western-series-/", "label": "Western"},
|
|
||||||
},
|
|
||||||
"film": {
|
|
||||||
"actions": {"path": "/films/actions/", "label": "Action"},
|
|
||||||
"animations": {"path": "/films/animations/", "label": "Animation"},
|
|
||||||
"aventures": {"path": "/films/aventures/", "label": "Aventure"},
|
|
||||||
"biopics": {"path": "/films/biopics/", "label": "Biopic"},
|
|
||||||
"comedies": {"path": "/films/comedies/", "label": "Comédie"},
|
|
||||||
"cultes": {"path": "/films/cultes/", "label": "Culte"},
|
|
||||||
"documentaires": {"path": "/films/documentaires/", "label": "Documentaire"},
|
|
||||||
"drames": {"path": "/films/drames/", "label": "Drame"},
|
|
||||||
"epouvante-horreurs": {"path": "/films/epouvante-horreurs/", "label": "Épouvante-Horreur"},
|
|
||||||
"espionnages": {"path": "/films/espionnages/", "label": "Espionnage"},
|
|
||||||
"familles": {"path": "/films/familles/", "label": "Famille"},
|
|
||||||
"fantastiques": {"path": "/films/fantastiques/", "label": "Fantastique"},
|
|
||||||
"guerres": {"path": "/films/guerres/", "label": "Guerre"},
|
|
||||||
"historiques": {"path": "/films/historiques/", "label": "Historique"},
|
|
||||||
"policiers": {"path": "/films/policiers/", "label": "Policier"},
|
|
||||||
"romances": {"path": "/films/romances/", "label": "Romance"},
|
|
||||||
"science-fictions": {"path": "/films/science-fictions/", "label": "Science-Fiction"},
|
|
||||||
"thrillers": {"path": "/films/thrillers/", "label": "Thriller"},
|
|
||||||
"westerns": {"path": "/films/westerns/", "label": "Western"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_NEWSID_RE = re.compile(r"(?:newsid=|/)(\d+)(?:-[^/]*)?\.?html?$|(?:newsid=)(\d+)")
|
|
||||||
_ONCLICK_URL_RE = re.compile(r"location\.href='([^']+)'")
|
|
||||||
_SEASON_RE = re.compile(r"saison\s*:?\s*(\d+)", re.IGNORECASE)
|
|
||||||
_YEAR_RE = re.compile(r"(\d{4})")
|
|
||||||
_FRAGMENT_RE = re.compile(r"^(vf|vostfr|vo)-(\d+(?:\.\d+)?)$")
|
|
||||||
_VERSION_ORDER = {"vf": 0, "vostfr": 1, "vo": 2}
|
|
||||||
|
|
||||||
|
|
||||||
def _merged_config() -> dict:
|
|
||||||
merged = deepcopy(DEFAULT_CONFIG)
|
|
||||||
for key, values in load_scraper_config("french_stream").items():
|
|
||||||
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
|
||||||
merged[key].update(values)
|
|
||||||
else:
|
|
||||||
merged[key] = values
|
|
||||||
return merged
|
|
||||||
|
|
||||||
|
|
||||||
def _newsid_from_url(url: str) -> str | None:
|
|
||||||
match = _NEWSID_RE.search(url)
|
|
||||||
if not match:
|
|
||||||
return None
|
|
||||||
return match.group(1) or match.group(2)
|
|
||||||
|
|
||||||
|
|
||||||
def _media_type(title: str, url: str) -> str:
|
|
||||||
if _SEASON_RE.search(title) or "-saison-" in url:
|
|
||||||
return "serie"
|
|
||||||
return "film"
|
|
||||||
|
|
||||||
|
|
||||||
@register_source
|
|
||||||
class FrenchStreamScraper(SourceScraper):
|
|
||||||
name = "french_stream"
|
|
||||||
label = "French-Stream"
|
|
||||||
base_url = "https://french-stream.lat"
|
|
||||||
media_types = ("serie", "film")
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- helpers
|
|
||||||
|
|
||||||
def _title_url(self, source_id: str) -> str:
|
|
||||||
return f"{self.base_url}/index.php?newsid={source_id}"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _text(element: Tag | None) -> str:
|
|
||||||
return element.get_text(" ", strip=True) if element else ""
|
|
||||||
|
|
||||||
def _is_serie(self, soup: BeautifulSoup, title: str, url: str) -> bool:
|
|
||||||
config = _merged_config()
|
|
||||||
if soup.select_one(config["details"]["serie_marker"]):
|
|
||||||
return True
|
|
||||||
return _media_type(title, url) == "serie"
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- search
|
|
||||||
|
|
||||||
async def search(self, query: str) -> list[SearchResult]:
|
|
||||||
config = _merged_config()
|
|
||||||
url = f"{self.base_url}{config['endpoints']['search']}"
|
|
||||||
html = await fetch(
|
|
||||||
url, referer=f"{self.base_url}/", data={"query": query, "page": "1"}
|
|
||||||
)
|
|
||||||
soup = BeautifulSoup(html, "lxml")
|
|
||||||
results: list[SearchResult] = []
|
|
||||||
for block in soup.select(config["search"]["item"]):
|
|
||||||
onclick = block.get("onclick", "")
|
|
||||||
url_match = _ONCLICK_URL_RE.search(onclick)
|
|
||||||
if not url_match:
|
|
||||||
logger.warning("french_stream : résultat sans lien, ignoré")
|
|
||||||
continue
|
|
||||||
href = url_match.group(1)
|
|
||||||
source_id = _newsid_from_url(href)
|
|
||||||
if not source_id:
|
|
||||||
logger.warning("french_stream : newsid introuvable dans %s", href)
|
|
||||||
continue
|
|
||||||
title = self._text(block.select_one(config["search"]["title"]))
|
|
||||||
image = block.select_one(config["search"]["poster"])
|
|
||||||
results.append(
|
|
||||||
SearchResult(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=title,
|
|
||||||
url=f"{self.base_url}{href}" if href.startswith("/") else href,
|
|
||||||
image_url=image.get("src") if image else None,
|
|
||||||
media_type=_media_type(title, href),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
logger.info("french_stream : %d résultats pour %r", len(results), query)
|
|
||||||
return results
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- latest
|
|
||||||
|
|
||||||
async def latest(self) -> list[SearchResult]:
|
|
||||||
"""Ajouts récents — page /series/ (mix films & séries)."""
|
|
||||||
config = _merged_config()
|
|
||||||
soup = await fetch_soup(f"{self.base_url}{config['latest']['path']}")
|
|
||||||
results = self._short_blocks(soup)
|
|
||||||
logger.info("french_stream : %d nouveautés récupérées", len(results))
|
|
||||||
return results
|
|
||||||
|
|
||||||
async def browse(self, media_type: str, category: str) -> list[SearchResult]:
|
|
||||||
"""Parcours par genre — pages /films/<genre>/ et /<genre>-series-/.
|
|
||||||
|
|
||||||
``media_type`` (« serie » | « film ») et ``category`` (clé du catalogue
|
|
||||||
YAML, ex. « thriller »). Le type est forcé sur les résultats : une page
|
|
||||||
genre séries ne liste que des séries, une page genre films que des films.
|
|
||||||
"""
|
|
||||||
catalog = _merged_config().get("browse", {}).get(media_type, {})
|
|
||||||
entry = catalog.get(category)
|
|
||||||
if entry is None:
|
|
||||||
raise ScrapeError(f"Catégorie inconnue : {media_type}/{category}")
|
|
||||||
soup = await fetch_soup(f"{self.base_url}{entry['path']}")
|
|
||||||
results = self._short_blocks(soup, force_type=media_type)
|
|
||||||
logger.info("french_stream : %d titres dans %s/%s", len(results), media_type, category)
|
|
||||||
return results
|
|
||||||
|
|
||||||
def _short_blocks(self, soup: BeautifulSoup, force_type: str | None = None) -> list[SearchResult]:
|
|
||||||
"""Bloc `div.short` → SearchResult (structure partagée nouveautés/genres)."""
|
|
||||||
config = _merged_config()
|
|
||||||
latest_cfg = config["latest"]
|
|
||||||
results: list[SearchResult] = []
|
|
||||||
for block in soup.select(latest_cfg["item"]):
|
|
||||||
link = block.select_one(latest_cfg["link"])
|
|
||||||
href = link.get("href") if link else None
|
|
||||||
if not href:
|
|
||||||
continue
|
|
||||||
source_id = _newsid_from_url(href)
|
|
||||||
if not source_id:
|
|
||||||
logger.warning("french_stream : newsid introuvable dans %s", href)
|
|
||||||
continue
|
|
||||||
image = link.select_one(latest_cfg["image"])
|
|
||||||
title = link.get("alt") or self._text(link)
|
|
||||||
results.append(
|
|
||||||
SearchResult(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=title,
|
|
||||||
url=f"{self.base_url}{href}" if href.startswith("/") else href,
|
|
||||||
image_url=image.get("src") if image else None,
|
|
||||||
media_type=force_type or _media_type(title, href),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return results
|
|
||||||
|
|
||||||
def browse_catalog(self) -> dict[str, dict[str, str]]:
|
|
||||||
"""Catalogue des genres parcourables : ``{type: {clé: libellé}}``."""
|
|
||||||
catalog = _merged_config().get("browse", {})
|
|
||||||
return {
|
|
||||||
media_type: {key: entry.get("label", key) for key, entry in genres.items()}
|
|
||||||
for media_type, genres in catalog.items()
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def genres_of_page(self, page_url: str) -> list[str]:
|
|
||||||
"""Genres (libellés FR) de la fiche pointée par l'URL d'un téléchargement.
|
|
||||||
|
|
||||||
Utilisé par les recommandations « Pour toi » pour les séries/films,
|
|
||||||
absents de Kitsu (et donc sans signal de genre sans Sonarr).
|
|
||||||
"""
|
|
||||||
if self.base_url.split("//")[-1].split("/")[0] not in page_url:
|
|
||||||
return [] # fiche d'une autre source (animé) : rien à lire ici
|
|
||||||
source_id = _newsid_from_url(page_url)
|
|
||||||
try:
|
|
||||||
details = await self.get_details(source_id)
|
|
||||||
except ScrapeError:
|
|
||||||
return []
|
|
||||||
return details.genres or []
|
|
||||||
# ------------------------------------------------------------- details
|
|
||||||
|
|
||||||
async def get_details(self, source_id: str) -> TitleDetails:
|
|
||||||
config = _merged_config()
|
|
||||||
details_cfg = config["details"]
|
|
||||||
url = self._title_url(source_id)
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
|
|
||||||
title = " ".join(self._text(soup.select_one(details_cfg["title"])).split())
|
|
||||||
if not title:
|
|
||||||
raise ScrapeError(f"french_stream : fiche introuvable pour {source_id} ({url})")
|
|
||||||
|
|
||||||
synopsis_el = soup.select_one(details_cfg["synopsis"])
|
|
||||||
if synopsis_el is not None:
|
|
||||||
for boilerplate in synopsis_el.select(details_cfg["synopsis_boilerplate"]):
|
|
||||||
boilerplate.decompose()
|
|
||||||
synopsis = self._text(synopsis_el) or None
|
|
||||||
|
|
||||||
genres_el = soup.select_one(details_cfg["genres"])
|
|
||||||
if genres_el and genres_el.select("a"):
|
|
||||||
genres = [a.get_text(strip=True) for a in genres_el.select("a") if a.get_text(strip=True)]
|
|
||||||
else:
|
|
||||||
genres = [g.strip() for g in self._text(genres_el).split(",") if g.strip()]
|
|
||||||
|
|
||||||
year_match = _YEAR_RE.search(self._text(soup.select_one(details_cfg["year"])))
|
|
||||||
if year_match is None:
|
|
||||||
year_match = _YEAR_RE.search(title)
|
|
||||||
|
|
||||||
poster_el = soup.select_one(details_cfg["poster_serie"])
|
|
||||||
poster = poster_el.get("src") if poster_el else None
|
|
||||||
if not poster:
|
|
||||||
film_data = soup.select_one(details_cfg["poster_film"])
|
|
||||||
poster = film_data.get("data-affiche") if film_data else None
|
|
||||||
|
|
||||||
is_serie = self._is_serie(soup, title, url)
|
|
||||||
episodes = await self._fetch_episodes(source_id, url, title) if is_serie else []
|
|
||||||
|
|
||||||
return TitleDetails(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=title,
|
|
||||||
url=url,
|
|
||||||
synopsis=synopsis,
|
|
||||||
image_url=poster,
|
|
||||||
genres=genres,
|
|
||||||
year=int(year_match.group(1)) if year_match else None,
|
|
||||||
episode_count=len(episodes) if is_serie else 1,
|
|
||||||
episodes=episodes if is_serie else [self._film_episode(url)],
|
|
||||||
media_type="serie" if is_serie else "film",
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ episodes
|
|
||||||
|
|
||||||
async def list_episodes(self, source_id: str) -> list[Episode]:
|
|
||||||
url = self._title_url(source_id)
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
title = " ".join(
|
|
||||||
self._text(soup.select_one(_merged_config()["details"]["title"])).split()
|
|
||||||
)
|
|
||||||
if not self._is_serie(soup, title, url):
|
|
||||||
return [self._film_episode(url)]
|
|
||||||
episodes = await self._fetch_episodes(source_id, url, title)
|
|
||||||
if not episodes:
|
|
||||||
raise ScrapeError(f"french_stream : aucun épisode trouvé pour {source_id} ({url})")
|
|
||||||
return episodes
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _film_episode(page_url: str) -> Episode:
|
|
||||||
return Episode(number=1, title="Film", url=f"{page_url}#film", season=1)
|
|
||||||
|
|
||||||
async def _fetch_episodes(
|
|
||||||
self, source_id: str, page_url: str, title: str
|
|
||||||
) -> list[Episode]:
|
|
||||||
"""Épisodes d'une saison via l'API JSON du site (versions vf/vostfr/vo)."""
|
|
||||||
config = _merged_config()
|
|
||||||
endpoint = config["endpoints"]["episodes"].format(newsid=source_id)
|
|
||||||
raw = await fetch(f"{self.base_url}{endpoint}", referer=page_url)
|
|
||||||
try:
|
|
||||||
data = json.loads(raw)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ScrapeError(
|
|
||||||
f"french_stream : JSON d'épisodes invalide pour {source_id}"
|
|
||||||
) from exc
|
|
||||||
if not isinstance(data, dict):
|
|
||||||
return []
|
|
||||||
|
|
||||||
season_match = _SEASON_RE.search(title)
|
|
||||||
season = int(season_match.group(1)) if season_match else 1
|
|
||||||
info = data.get("info") if isinstance(data.get("info"), dict) else {}
|
|
||||||
|
|
||||||
episodes: list[Episode] = []
|
|
||||||
for version, eps in data.items():
|
|
||||||
if version == "info" or not isinstance(eps, dict):
|
|
||||||
continue
|
|
||||||
for number_key in eps:
|
|
||||||
number_match = re.match(r"^(\d+(?:\.\d+)?)$", number_key)
|
|
||||||
if not number_match:
|
|
||||||
continue
|
|
||||||
number = float(number_key)
|
|
||||||
ep_info = info.get(number_key) or info.get(str(int(number))) or {}
|
|
||||||
episodes.append(
|
|
||||||
Episode(
|
|
||||||
number=number,
|
|
||||||
title=ep_info.get("title") if isinstance(ep_info, dict) else None,
|
|
||||||
url=f"{page_url}#{version}-{number_key}",
|
|
||||||
season=season,
|
|
||||||
version=version,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
episodes.sort(
|
|
||||||
key=lambda e: (e.number, _VERSION_ORDER.get(e.version or "", 99))
|
|
||||||
)
|
|
||||||
return episodes
|
|
||||||
|
|
||||||
# -------------------------------------------------------------- embeds
|
|
||||||
|
|
||||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
|
||||||
config = _merged_config()
|
|
||||||
page_url, _, fragment = episode_url.partition("#")
|
|
||||||
source_id = _newsid_from_url(page_url)
|
|
||||||
if not source_id:
|
|
||||||
raise ScrapeError(f"french_stream : newsid introuvable dans {episode_url}")
|
|
||||||
|
|
||||||
if fragment == "film":
|
|
||||||
endpoint = config["endpoints"]["film"].format(newsid=source_id)
|
|
||||||
raw = await fetch(f"{self.base_url}{endpoint}", referer=page_url)
|
|
||||||
try:
|
|
||||||
data = json.loads(raw)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ScrapeError(
|
|
||||||
f"french_stream : JSON film invalide pour {source_id}"
|
|
||||||
) from exc
|
|
||||||
links: list[str] = []
|
|
||||||
for hoster_urls in (data.get("players") or {}).values():
|
|
||||||
if not isinstance(hoster_urls, dict):
|
|
||||||
continue
|
|
||||||
for embed in hoster_urls.values():
|
|
||||||
if isinstance(embed, str) and embed.startswith("http") and embed not in links:
|
|
||||||
links.append(embed)
|
|
||||||
else:
|
|
||||||
fragment_match = _FRAGMENT_RE.match(fragment)
|
|
||||||
if not fragment_match:
|
|
||||||
raise ScrapeError(
|
|
||||||
f"french_stream : fragment d'épisode invalide dans {episode_url}"
|
|
||||||
)
|
|
||||||
version, number = fragment_match.group(1), fragment_match.group(2)
|
|
||||||
endpoint = config["endpoints"]["episodes"].format(newsid=source_id)
|
|
||||||
raw = await fetch(f"{self.base_url}{endpoint}", referer=page_url)
|
|
||||||
try:
|
|
||||||
data = json.loads(raw)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ScrapeError(
|
|
||||||
f"french_stream : JSON d'épisodes invalide pour {source_id}"
|
|
||||||
) from exc
|
|
||||||
hosters = (data.get(version) or {}).get(number) or {}
|
|
||||||
links = [u for u in hosters.values() if isinstance(u, str) and u.startswith("http")]
|
|
||||||
|
|
||||||
if not links:
|
|
||||||
raise ScrapeError(f"french_stream : aucun lien embed extrait de {episode_url}")
|
|
||||||
logger.info("french_stream : %d liens embed pour %s", len(links), episode_url)
|
|
||||||
return links
|
|
||||||
@@ -1,258 +0,0 @@
|
|||||||
"""Source VoirAnime (voiranime.xyz) — plateforme VODSPHERE, animes VOSTFR.
|
|
||||||
|
|
||||||
Faits structurels (vérifiés en live) :
|
|
||||||
- Recherche : GET /catalogue?q=<q> → `article.catalogue-card` (lien `a.catalogue-poster`,
|
|
||||||
poster et titre dans `img[src]` / `img[alt]`).
|
|
||||||
- Fiche : /catalogue/<slug> — titre `h1`, poster `img[alt^="Affiche de"]`, compteur
|
|
||||||
d'épisodes dans `.media-detail-meta span` (« 1192 épisodes »), synopsis `.media-detail-story p`.
|
|
||||||
- Épisodes : `a.detail-episode-card` avec `data-season` et libellé `<small>S1 · E1</small>`.
|
|
||||||
- Lecteur : /lecteur/<content>/<episode> — attributs data-prepare-url (résolution AJAX)
|
|
||||||
et data-fallback-source-url (source suivante) ; en fin de chaîne le fallback repointe
|
|
||||||
vers une source déjà vue → dédoublonnage par id de source obligatoire.
|
|
||||||
- Accueil : carrousels #carousel-new (nouveautés) et #carousel-added (ajouts), 24 cartes.
|
|
||||||
- La résolution d'un lien direct se fait dans l'extracteur hoster dédié (voir
|
|
||||||
app/scrapers/hosters/voiranime.py) : cette source ne renvoie que les URLs prepare.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from copy import deepcopy
|
|
||||||
from urllib.parse import quote_plus, urljoin
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup, Tag
|
|
||||||
|
|
||||||
from app.scrapers.base import (
|
|
||||||
Episode,
|
|
||||||
ScrapeError,
|
|
||||||
SearchResult,
|
|
||||||
SourceScraper,
|
|
||||||
TitleDetails,
|
|
||||||
register_source,
|
|
||||||
)
|
|
||||||
from app.scrapers.config_loader import load_scraper_config
|
|
||||||
from app.scrapers.http import fetch_soup
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
DEFAULT_CONFIG: dict = {
|
|
||||||
"search": {
|
|
||||||
"endpoint": "/catalogue?q={query}",
|
|
||||||
"result": "article.catalogue-card",
|
|
||||||
"link": "a.catalogue-poster",
|
|
||||||
},
|
|
||||||
"details": {
|
|
||||||
"title": "h1",
|
|
||||||
"poster": 'img[alt^="Affiche de"]',
|
|
||||||
"meta_count": ".media-detail-meta span",
|
|
||||||
"synopsis": ".media-detail-story p",
|
|
||||||
},
|
|
||||||
"episodes": {
|
|
||||||
"card": "a.detail-episode-card",
|
|
||||||
},
|
|
||||||
"player": {
|
|
||||||
"root": "[data-prepare-url]",
|
|
||||||
"prepare_attr": "data-prepare-url",
|
|
||||||
"fallback_attr": "data-fallback-source-url",
|
|
||||||
"max_sources": 4,
|
|
||||||
},
|
|
||||||
"latest": {
|
|
||||||
"path": "/",
|
|
||||||
"carousels": ("#carousel-new", "#carousel-added"),
|
|
||||||
"item": "article.content-card",
|
|
||||||
"link": "a.card-poster",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_SLUG_RE = re.compile(r"/catalogue/([a-z0-9-]+)", re.IGNORECASE)
|
|
||||||
_EPISODE_LABEL_RE = re.compile(r"S\s*(\d+)\s*·\s*E\s*(\d+(?:[.,]\d+)?)", re.IGNORECASE)
|
|
||||||
_EPISODE_COUNT_RE = re.compile(r"(\d+)\s*épisodes", re.IGNORECASE)
|
|
||||||
_SOURCE_ID_RE = re.compile(r"/lecteur/prepare/(\d+)")
|
|
||||||
|
|
||||||
_FALLBACK_SOURCE_RE = re.compile(r"[?&]source=(\d+)")
|
|
||||||
def _merged_config() -> dict:
|
|
||||||
merged = deepcopy(DEFAULT_CONFIG)
|
|
||||||
for key, values in load_scraper_config("voiranime").items():
|
|
||||||
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
|
||||||
merged[key].update(values)
|
|
||||||
else:
|
|
||||||
merged[key] = values
|
|
||||||
return merged
|
|
||||||
|
|
||||||
|
|
||||||
@register_source
|
|
||||||
class VoirAnimeScraper(SourceScraper):
|
|
||||||
name = "voiranime"
|
|
||||||
label = "VoirAnime"
|
|
||||||
base_url = "https://voiranime.xyz"
|
|
||||||
media_types = ("anime",)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- helpers
|
|
||||||
|
|
||||||
def _title_url(self, source_id: str) -> str:
|
|
||||||
return f"{self.base_url}/catalogue/{source_id}"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _slug_from_url(url: str) -> str | None:
|
|
||||||
match = _SLUG_RE.search(url)
|
|
||||||
return match.group(1) if match else None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _text(element: Tag | None) -> str:
|
|
||||||
return element.get_text(" ", strip=True) if element else ""
|
|
||||||
|
|
||||||
def _result_from_card(self, link: Tag) -> SearchResult | None:
|
|
||||||
href = link.get("href")
|
|
||||||
if not href:
|
|
||||||
return None
|
|
||||||
source_id = self._slug_from_url(str(href))
|
|
||||||
if not source_id:
|
|
||||||
return None
|
|
||||||
img = link.select_one("img")
|
|
||||||
title = (img.get("alt") if img else None) or source_id.replace("-", " ")
|
|
||||||
image = img.get("src") if img else None
|
|
||||||
return SearchResult(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=str(title).strip(),
|
|
||||||
url=self._title_url(source_id),
|
|
||||||
image_url=str(image) if image else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- search
|
|
||||||
|
|
||||||
async def search(self, query: str) -> list[SearchResult]:
|
|
||||||
config = _merged_config()
|
|
||||||
search_cfg = config["search"]
|
|
||||||
url = self.base_url + search_cfg["endpoint"].format(query=quote_plus(query))
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
results: list[SearchResult] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
for card in soup.select(search_cfg["result"]):
|
|
||||||
link = card.select_one(search_cfg["link"])
|
|
||||||
if link is None:
|
|
||||||
logger.warning("voiranime : carte de résultat sans lien, ignorée")
|
|
||||||
continue
|
|
||||||
result = self._result_from_card(link)
|
|
||||||
if result and result.source_id not in seen:
|
|
||||||
seen.add(result.source_id)
|
|
||||||
results.append(result)
|
|
||||||
return results
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- détails
|
|
||||||
|
|
||||||
async def get_details(self, source_id: str) -> TitleDetails:
|
|
||||||
config = _merged_config()
|
|
||||||
details_cfg = config["details"]
|
|
||||||
url = self._title_url(source_id)
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
|
|
||||||
title = self._text(soup.select_one(details_cfg["title"])) or source_id.replace("-", " ")
|
|
||||||
poster = soup.select_one(details_cfg["poster"])
|
|
||||||
image = str(poster["src"]) if poster and poster.get("src") else None
|
|
||||||
synopsis = self._text(soup.select_one(details_cfg["synopsis"])) or None
|
|
||||||
|
|
||||||
episode_count: int | None = None
|
|
||||||
count_el = soup.select_one(details_cfg["meta_count"])
|
|
||||||
if count_el:
|
|
||||||
match = _EPISODE_COUNT_RE.search(self._text(count_el))
|
|
||||||
if match:
|
|
||||||
episode_count = int(match.group(1))
|
|
||||||
|
|
||||||
episodes = self._parse_episodes(soup, config)
|
|
||||||
return TitleDetails(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=title,
|
|
||||||
url=url,
|
|
||||||
synopsis=synopsis,
|
|
||||||
image_url=image,
|
|
||||||
episode_count=episode_count if episode_count is not None else len(episodes),
|
|
||||||
episodes=episodes,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- épisodes
|
|
||||||
|
|
||||||
async def list_episodes(self, source_id: str) -> list[Episode]:
|
|
||||||
config = _merged_config()
|
|
||||||
soup = await fetch_soup(self._title_url(source_id))
|
|
||||||
return self._parse_episodes(soup, config)
|
|
||||||
|
|
||||||
def _parse_episodes(self, soup: BeautifulSoup, config: dict) -> list[Episode]:
|
|
||||||
episodes: list[Episode] = []
|
|
||||||
for card in soup.select(config["episodes"]["card"]):
|
|
||||||
href = card.get("href")
|
|
||||||
if not href:
|
|
||||||
continue
|
|
||||||
label = self._text(card)
|
|
||||||
match = _EPISODE_LABEL_RE.search(label)
|
|
||||||
if not match:
|
|
||||||
logger.warning("voiranime : libellé d'épisode illisible : %r", label[:60])
|
|
||||||
continue
|
|
||||||
episodes.append(
|
|
||||||
Episode(
|
|
||||||
number=float(match.group(2).replace(",", ".")),
|
|
||||||
title=None,
|
|
||||||
url=urljoin(self.base_url + "/", str(href)),
|
|
||||||
season=int(match.group(1)),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return episodes
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- lecteurs
|
|
||||||
|
|
||||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
|
||||||
"""Parcourt la chaîne Lecteur 1 → 2 → … et renvoie les URLs prepare absolues."""
|
|
||||||
config = _merged_config()
|
|
||||||
player_cfg = config["player"]
|
|
||||||
prepare_urls: list[str] = []
|
|
||||||
seen_sources: set[str] = set()
|
|
||||||
page_url = episode_url
|
|
||||||
for _ in range(int(player_cfg["max_sources"])):
|
|
||||||
soup = await fetch_soup(page_url)
|
|
||||||
root = soup.select_one(player_cfg["root"])
|
|
||||||
if root is None:
|
|
||||||
break
|
|
||||||
prepare = root.get(player_cfg["prepare_attr"])
|
|
||||||
if not prepare:
|
|
||||||
break
|
|
||||||
prepare_url = urljoin(self.base_url + "/", str(prepare))
|
|
||||||
match = _SOURCE_ID_RE.search(prepare_url)
|
|
||||||
source_key = match.group(1) if match else prepare_url
|
|
||||||
if source_key in seen_sources:
|
|
||||||
break
|
|
||||||
seen_sources.add(source_key)
|
|
||||||
prepare_urls.append(prepare_url)
|
|
||||||
fallback = root.get(player_cfg["fallback_attr"])
|
|
||||||
if not fallback:
|
|
||||||
break
|
|
||||||
page_url = urljoin(self.base_url + "/", str(fallback))
|
|
||||||
next_source = _FALLBACK_SOURCE_RE.search(page_url)
|
|
||||||
if next_source and next_source.group(1) in seen_sources:
|
|
||||||
break
|
|
||||||
if not prepare_urls:
|
|
||||||
raise ScrapeError(f"voiranime : aucun lecteur trouvé sur {episode_url}")
|
|
||||||
return prepare_urls
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- découverte
|
|
||||||
|
|
||||||
async def latest(self) -> list[SearchResult]:
|
|
||||||
"""Nouveautés + ajouts récents — carrousels de la page d'accueil."""
|
|
||||||
config = _merged_config()
|
|
||||||
latest_cfg = config["latest"]
|
|
||||||
soup = await fetch_soup(self.base_url + latest_cfg["path"])
|
|
||||||
results: list[SearchResult] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
for carousel in latest_cfg["carousels"]:
|
|
||||||
section = soup.select_one(carousel)
|
|
||||||
if section is None:
|
|
||||||
logger.debug("voiranime : carrousel %s absent de l'accueil", carousel)
|
|
||||||
continue
|
|
||||||
for card in section.select(latest_cfg["item"]):
|
|
||||||
link = card.select_one(latest_cfg["link"])
|
|
||||||
if link is None:
|
|
||||||
continue
|
|
||||||
result = self._result_from_card(link)
|
|
||||||
if result and result.source_id not in seen:
|
|
||||||
seen.add(result.source_id)
|
|
||||||
results.append(result)
|
|
||||||
return results
|
|
||||||
@@ -1,339 +0,0 @@
|
|||||||
"""Source Vostfree (ipv4.vostfree.ws) — moteur DataLife Engine, animes/films VF & VOSTFR.
|
|
||||||
|
|
||||||
Faits structurels (vérifiés en live) :
|
|
||||||
- Recherche : GET /index.php?do=search&subaction=search&story=<q> → blocs `div.search-result`.
|
|
||||||
- Nouveautés : /animes-vostfr-recement-ajoutees.html → blocs `div.movie-poster`
|
|
||||||
(lien `.play a`, alt = titre, poster `span.image img`).
|
|
||||||
- Fiche : `/444-telecharger-....html` — métadonnées dans `.slide-*`, épisodes dans un
|
|
||||||
`select.new_player_selector` (une `option value="buttons_N"` par épisode).
|
|
||||||
- Chaque `div#buttons_N` contient un ou plusieurs `div.new_player_<hoster>#player_M` ;
|
|
||||||
l'URL embed est le texte de `div#content_player_M` (URL complète ou simple ID selon
|
|
||||||
l'hébergeur — les templates de reconstruction viennent du JS `anime.js` du site).
|
|
||||||
- Pas de page par épisode : l'URL d'épisode est `<fiche>#buttons_N`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from copy import deepcopy
|
|
||||||
from urllib.parse import quote
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup, Tag
|
|
||||||
|
|
||||||
from app.scrapers.base import (
|
|
||||||
Episode,
|
|
||||||
ScrapeError,
|
|
||||||
SearchResult,
|
|
||||||
SourceScraper,
|
|
||||||
TitleDetails,
|
|
||||||
register_source,
|
|
||||||
)
|
|
||||||
from app.scrapers.config_loader import load_scraper_config
|
|
||||||
from app.scrapers.http import fetch_soup
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
DEFAULT_CONFIG: dict = {
|
|
||||||
"search": {
|
|
||||||
"result": "div.search-result",
|
|
||||||
"link": "div.title a",
|
|
||||||
"image": "span.image img",
|
|
||||||
"genres": "ul.additional li",
|
|
||||||
},
|
|
||||||
"details": {
|
|
||||||
"title": "h1",
|
|
||||||
"poster": ".slide-poster img",
|
|
||||||
"synopsis": ".slide-desc",
|
|
||||||
"genres": '.slide-top li.right a[href*="/genre/"]',
|
|
||||||
"episode_badge": ".slide-poster .year",
|
|
||||||
"season_li": "ul.slide-top li",
|
|
||||||
},
|
|
||||||
"episodes": {
|
|
||||||
"option": "select.new_player_selector option",
|
|
||||||
"button": "div.button_box",
|
|
||||||
"content_prefix": "content_",
|
|
||||||
},
|
|
||||||
"latest": {
|
|
||||||
"path": "/animes-vostfr-recement-ajoutees.html",
|
|
||||||
"item": "div.movie-poster",
|
|
||||||
"link": ".play a", # href = fiche, alt = titre
|
|
||||||
"image": "span.image img",
|
|
||||||
},
|
|
||||||
# class CSS du lecteur → template d'URL ({} = valeur de #content_player_M ;
|
|
||||||
# chaîne vide = la valeur est déjà une URL complète)
|
|
||||||
"players": {
|
|
||||||
"new_player_vip": "",
|
|
||||||
"new_player_moevideo": "",
|
|
||||||
"new_player_sibnet": "https://video.sibnet.ru/shell.php?videoid={}",
|
|
||||||
"new_player_netu": "https://video.sibnet.ru/shell.php?videoid={}",
|
|
||||||
"new_player_uqload": "https://uqload.com/embed-{}.html",
|
|
||||||
"new_player_mp4": "https://www.mp4upload.com/embed-{}.html",
|
|
||||||
"new_player_fembed": "https://www.fembed.com/v/{}",
|
|
||||||
"new_player_mytv": "https://www.myvi.top/embed/{}",
|
|
||||||
"new_player_myvi": "https://myvi.ru/player/embed/html/{}",
|
|
||||||
"new_player_moevideo_mail": "",
|
|
||||||
"new_player_rutube": "https://rutube.ru/play/embed/{}",
|
|
||||||
"new_player_ok": "https://ok.ru/video/{}",
|
|
||||||
"new_player_mail2": "https://my.mail.ru/video/embed/{}",
|
|
||||||
"new_player_rapids": "https://rapidstream.co/embed-{}.html",
|
|
||||||
"new_player_gtv": "https://iframedream.com/embed/{}.html",
|
|
||||||
"new_player_cloudvideo": "https://cloudvideo.tv/embed-{}.html",
|
|
||||||
"new_player_uptostream": "https://uptostream.com/iframe/{}",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_SLUG_RE = re.compile(r"([^/]+)\.html?$")
|
|
||||||
_NUMBER_RE = re.compile(r"(\d+(?:[.,]\d+)?)")
|
|
||||||
_SEASON_RE = re.compile(r"Saison\s*:?\s*(\d+)", re.IGNORECASE)
|
|
||||||
_SAFE_FRAGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+$")
|
|
||||||
_VERSION_RE = re.compile(r"\b(vostfr|vf)\b", re.IGNORECASE)
|
|
||||||
|
|
||||||
|
|
||||||
def _merged_config() -> dict:
|
|
||||||
merged = deepcopy(DEFAULT_CONFIG)
|
|
||||||
for key, values in load_scraper_config("vostfree").items():
|
|
||||||
if isinstance(values, dict) and isinstance(merged.get(key), dict):
|
|
||||||
merged[key].update(values)
|
|
||||||
else:
|
|
||||||
merged[key] = values
|
|
||||||
return merged
|
|
||||||
|
|
||||||
|
|
||||||
@register_source
|
|
||||||
class VostfreeScraper(SourceScraper):
|
|
||||||
name = "vostfree"
|
|
||||||
label = "Vostfree"
|
|
||||||
base_url = "https://ipv4.vostfree.ws"
|
|
||||||
media_types = ("anime",)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- helpers
|
|
||||||
|
|
||||||
def _title_url(self, source_id: str) -> str:
|
|
||||||
return f"{self.base_url}/{source_id}.html"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _slug_from_url(url: str) -> str | None:
|
|
||||||
match = _SLUG_RE.search(url)
|
|
||||||
return match.group(1) if match else None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _text(element: Tag | None) -> str:
|
|
||||||
return element.get_text(" ", strip=True) if element else ""
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- search
|
|
||||||
|
|
||||||
async def search(self, query: str) -> list[SearchResult]:
|
|
||||||
config = _merged_config()
|
|
||||||
url = f"{self.base_url}/index.php?do=search&subaction=search&story={quote(query)}"
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
results: list[SearchResult] = []
|
|
||||||
for block in soup.select(config["search"]["result"]):
|
|
||||||
link = block.select_one(config["search"]["link"])
|
|
||||||
href = link.get("href") if link else None
|
|
||||||
if not href:
|
|
||||||
logger.warning("vostfree : bloc de résultat sans lien, ignoré")
|
|
||||||
continue
|
|
||||||
source_id = self._slug_from_url(href)
|
|
||||||
if not source_id:
|
|
||||||
logger.warning("vostfree : slug introuvable dans %s", href)
|
|
||||||
continue
|
|
||||||
image = block.select_one(config["search"]["image"])
|
|
||||||
genres_text = " ".join(
|
|
||||||
li.get_text(" ", strip=True) for li in block.select(config["search"]["genres"])
|
|
||||||
)
|
|
||||||
results.append(
|
|
||||||
SearchResult(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=self._text(link),
|
|
||||||
url=href,
|
|
||||||
image_url=image.get("src") if image else None,
|
|
||||||
media_type="film"
|
|
||||||
if re.search(r"\bfilm", genres_text, re.IGNORECASE)
|
|
||||||
else "anime",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
logger.info("vostfree : %d résultats pour %r", len(results), query)
|
|
||||||
return results
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- latest
|
|
||||||
|
|
||||||
async def latest(self) -> list[SearchResult]:
|
|
||||||
"""Ajouts récents — page « Animes VOSTFR récemment ajoutés »."""
|
|
||||||
config = _merged_config()
|
|
||||||
latest_cfg = config["latest"]
|
|
||||||
soup = await fetch_soup(f"{self.base_url}{latest_cfg['path']}")
|
|
||||||
results: list[SearchResult] = []
|
|
||||||
for block in soup.select(latest_cfg["item"]):
|
|
||||||
link = block.select_one(latest_cfg["link"])
|
|
||||||
href = link.get("href") if link else None
|
|
||||||
if not href:
|
|
||||||
logger.warning("vostfree : bloc nouveauté sans lien, ignoré")
|
|
||||||
continue
|
|
||||||
source_id = self._slug_from_url(href)
|
|
||||||
if not source_id:
|
|
||||||
logger.warning("vostfree : slug introuvable dans %s", href)
|
|
||||||
continue
|
|
||||||
image = block.select_one(latest_cfg["image"])
|
|
||||||
title = link.get("alt") or self._text(link)
|
|
||||||
results.append(
|
|
||||||
SearchResult(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=title,
|
|
||||||
url=href,
|
|
||||||
image_url=image.get("src") if image else None,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
logger.info("vostfree : %d nouveautés récupérées", len(results))
|
|
||||||
return results
|
|
||||||
# ------------------------------------------------------------- details
|
|
||||||
|
|
||||||
async def get_details(self, source_id: str) -> TitleDetails:
|
|
||||||
config = _merged_config()
|
|
||||||
url = self._title_url(source_id)
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
details_cfg = config["details"]
|
|
||||||
|
|
||||||
title_el = soup.select_one(details_cfg["title"])
|
|
||||||
if title_el is None:
|
|
||||||
raise ScrapeError(f"vostfree : fiche introuvable pour {source_id} ({url})")
|
|
||||||
|
|
||||||
synopsis_el = soup.select_one(details_cfg["synopsis"])
|
|
||||||
if synopsis_el is not None:
|
|
||||||
for cast in synopsis_el.select(".cast"):
|
|
||||||
cast.decompose()
|
|
||||||
synopsis = self._text(synopsis_el) or None
|
|
||||||
|
|
||||||
badge = self._text(soup.select_one(details_cfg["episode_badge"]))
|
|
||||||
badge_match = _NUMBER_RE.search(badge)
|
|
||||||
|
|
||||||
season = 1
|
|
||||||
for li in soup.select(details_cfg["season_li"]):
|
|
||||||
season_match = _SEASON_RE.search(self._text(li))
|
|
||||||
if season_match:
|
|
||||||
season = int(season_match.group(1))
|
|
||||||
break
|
|
||||||
|
|
||||||
episodes = self._parse_episodes(soup, url, season)
|
|
||||||
|
|
||||||
return TitleDetails(
|
|
||||||
source=self.name,
|
|
||||||
source_id=source_id,
|
|
||||||
title=self._text(title_el),
|
|
||||||
url=url,
|
|
||||||
synopsis=synopsis,
|
|
||||||
image_url=(soup.select_one(details_cfg["poster"]) or Tag(name="img")).get("src"),
|
|
||||||
genres=[a.get_text(strip=True) for a in soup.select(details_cfg["genres"])],
|
|
||||||
episode_count=int(badge_match.group(1).replace(",", ".")) if badge_match else None,
|
|
||||||
episodes=episodes,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ episodes
|
|
||||||
|
|
||||||
async def list_episodes(self, source_id: str) -> list[Episode]:
|
|
||||||
url = self._title_url(source_id)
|
|
||||||
soup = await fetch_soup(url)
|
|
||||||
season = self._find_season(soup)
|
|
||||||
episodes = self._parse_episodes(soup, url, season)
|
|
||||||
if not episodes:
|
|
||||||
raise ScrapeError(f"vostfree : aucun épisode trouvé pour {source_id} ({url})")
|
|
||||||
return episodes
|
|
||||||
|
|
||||||
def _find_season(self, soup: BeautifulSoup) -> int:
|
|
||||||
config = _merged_config()
|
|
||||||
for li in soup.select(config["details"]["season_li"]):
|
|
||||||
season_match = _SEASON_RE.search(self._text(li))
|
|
||||||
if season_match:
|
|
||||||
return int(season_match.group(1))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
def _detect_version(self, soup: BeautifulSoup, page_url: str) -> str | None:
|
|
||||||
config = _merged_config()
|
|
||||||
title_el = soup.select_one(config["details"]["title"])
|
|
||||||
for text in (self._text(title_el), page_url):
|
|
||||||
match = _VERSION_RE.search(text)
|
|
||||||
if match:
|
|
||||||
return match.group(1).lower()
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _parse_episodes(self, soup: BeautifulSoup, page_url: str, season: int) -> list[Episode]:
|
|
||||||
config = _merged_config()
|
|
||||||
version = self._detect_version(soup, page_url)
|
|
||||||
options = soup.select(config["episodes"]["option"])
|
|
||||||
entries: list[tuple[str, str]] = []
|
|
||||||
seen_ids: set[str] = set()
|
|
||||||
for opt in options:
|
|
||||||
value = opt.get("value", "")
|
|
||||||
if value in seen_ids:
|
|
||||||
continue # le site duplique parfois une option (ex. buttons_268)
|
|
||||||
seen_ids.add(value)
|
|
||||||
entries.append((value, opt.get_text(strip=True)))
|
|
||||||
if not entries: # fiche sans sélecteur : on retombe sur les blocs de boutons
|
|
||||||
entries = [
|
|
||||||
(box.get("id", ""), f"Episode {index}")
|
|
||||||
for index, box in enumerate(soup.select(config["episodes"]["button"]), start=1)
|
|
||||||
]
|
|
||||||
episodes: list[Episode] = []
|
|
||||||
for index, (button_id, label) in enumerate(entries, start=1):
|
|
||||||
number_match = _NUMBER_RE.search(label)
|
|
||||||
number = (
|
|
||||||
float(number_match.group(1).replace(",", ".")) if number_match else float(index)
|
|
||||||
)
|
|
||||||
episodes.append(
|
|
||||||
Episode(
|
|
||||||
number=number,
|
|
||||||
title=label or f"Episode {index}",
|
|
||||||
url=f"{page_url}#{button_id}" if button_id else page_url,
|
|
||||||
season=season,
|
|
||||||
version=version,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return episodes
|
|
||||||
|
|
||||||
# -------------------------------------------------------------- embeds
|
|
||||||
|
|
||||||
async def extract_embed_links(self, episode_url: str) -> list[str]:
|
|
||||||
config = _merged_config()
|
|
||||||
page_url, _, fragment = episode_url.partition("#")
|
|
||||||
soup = await fetch_soup(page_url)
|
|
||||||
button_cfg = config["episodes"]
|
|
||||||
|
|
||||||
button: Tag | None = None
|
|
||||||
if fragment and _SAFE_FRAGMENT_RE.match(fragment):
|
|
||||||
button = soup.select_one(f"#{fragment}")
|
|
||||||
if button is None:
|
|
||||||
logger.warning(
|
|
||||||
"vostfree : fragment #%s sans bloc correspondant dans %s", fragment, page_url
|
|
||||||
)
|
|
||||||
if button is None:
|
|
||||||
button = soup.select_one(button_cfg["button"])
|
|
||||||
if button is None:
|
|
||||||
raise ScrapeError(f"vostfree : aucun lecteur trouvé sur {episode_url}")
|
|
||||||
|
|
||||||
prefix = button_cfg["content_prefix"]
|
|
||||||
links: list[str] = []
|
|
||||||
for player in button.find_all("div", recursive=False):
|
|
||||||
player_id = player.get("id")
|
|
||||||
content = soup.select_one(f"#{prefix}{player_id}") if player_id else None
|
|
||||||
if content is None:
|
|
||||||
logger.warning(
|
|
||||||
"vostfree : contenu absent pour le lecteur #%s (%s)", player_id, page_url
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
value = content.get_text(strip=True)
|
|
||||||
if not value:
|
|
||||||
continue
|
|
||||||
player_class = (player.get("class") or [""])[0]
|
|
||||||
template = config["players"].get(player_class)
|
|
||||||
if value.startswith("http"):
|
|
||||||
links.append(value)
|
|
||||||
elif template is not None:
|
|
||||||
links.append(template.format(value))
|
|
||||||
else:
|
|
||||||
logger.warning(
|
|
||||||
"vostfree : pas de template pour le lecteur %r (valeur %r)", player_class, value
|
|
||||||
)
|
|
||||||
if not links:
|
|
||||||
raise ScrapeError(f"vostfree : aucun lien embed extrait de {episode_url}")
|
|
||||||
logger.info("vostfree : %d liens embed pour %s", len(links), episode_url)
|
|
||||||
return links
|
|
||||||
@@ -1,513 +0,0 @@
|
|||||||
"""Découverte : nouveautés des sources, incontournables et recommandations.
|
|
||||||
|
|
||||||
Trois sections :
|
|
||||||
- **latest** — « récemment ajoutés » scrapés sur chaque source activée (cliquables
|
|
||||||
directement vers la fiche) ;
|
|
||||||
- **must_watch** — titres les plus populaires du catalogue Kitsu (tous temps) ;
|
|
||||||
- **for_you** — recommandations par genres : les genres des titres téléchargés
|
|
||||||
(serveur), des favoris (par utilisateur) et des séries téléchargées sur Sonarr
|
|
||||||
sont agrégés, puis Kitsu est interrogé sur ces catégories en excluant le
|
|
||||||
déjà-possédé.
|
|
||||||
|
|
||||||
Toutes les sources externes sont optionnelles : un échec (réseau, scraping, API)
|
|
||||||
laisse la section vide et n'est jamais remonté au caller (dégradation gracieuse).
|
|
||||||
Un cache mémoire TTL évite de re-scaper à chaque chargement de page.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import dataclasses
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import time
|
|
||||||
import unicodedata
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.db import db
|
|
||||||
from app.scrapers.base import (
|
|
||||||
ScrapeError,
|
|
||||||
SourceScraper,
|
|
||||||
all_sources,
|
|
||||||
get_source,
|
|
||||||
import_all_scrapers,
|
|
||||||
)
|
|
||||||
from app.services.kitsu import KitsuService, normalize_title
|
|
||||||
from app.services.settings import is_source_enabled
|
|
||||||
from app.services.sonarr import sonarr
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
import_all_scrapers()
|
|
||||||
|
|
||||||
# Bornes de l'algorithme
|
|
||||||
_MAX_HISTORY_TITLES = 12 # titres récents analysés (téléchargements + favoris)
|
|
||||||
_MAX_GENRES = 4 # genres retenus pour la requête Kitsu
|
|
||||||
_KITSU_PAGE_MAX = 20 # limite dure de l'API Kitsu (page[limit] > 20 → 400)
|
|
||||||
# (les TTL des sections suivent, plus bas)
|
|
||||||
|
|
||||||
# Genres animés proposés à l'exploration — slugs des catégories Kitsu officielles.
|
|
||||||
ANIME_GENRES: dict[str, str] = {
|
|
||||||
"action": "Action",
|
|
||||||
"adventure": "Aventure",
|
|
||||||
"comedy": "Comédie",
|
|
||||||
"drama": "Drame",
|
|
||||||
"fantasy": "Fantasy",
|
|
||||||
"science-fiction": "Science-Fiction",
|
|
||||||
"romance": "Romance",
|
|
||||||
"slice-of-life": "Tranche de vie",
|
|
||||||
"sports": "Sport",
|
|
||||||
"supernatural": "Surnaturel",
|
|
||||||
"mystery": "Mystère",
|
|
||||||
"psychological": "Psychologique",
|
|
||||||
"horror": "Horreur",
|
|
||||||
"mecha": "Mecha",
|
|
||||||
"isekai": "Isekai",
|
|
||||||
"music": "Musique",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Genres des fiches séries/films (French-Stream, libellés FR, slugifiés) →
|
|
||||||
# catégorie Kitsu équivalente. Les genres sans équivalent (Médical, Western…)
|
|
||||||
# sont ignorés : mieux vaut un signal incomplet qu'une catégorie inexistante.
|
|
||||||
_GENRE_FR_TO_KITSU: dict[str, str] = {
|
|
||||||
"action": "Action",
|
|
||||||
"aventure": "Adventure",
|
|
||||||
"aventures": "Adventure",
|
|
||||||
"comedie": "Comedy",
|
|
||||||
"comedies": "Comedy",
|
|
||||||
"drame": "Drama",
|
|
||||||
"drames": "Drama",
|
|
||||||
"epouvante-horreur": "Horror",
|
|
||||||
"horreur": "Horror",
|
|
||||||
"fantastique": "Fantasy",
|
|
||||||
"fantastiques": "Fantasy",
|
|
||||||
"romance": "Romance",
|
|
||||||
"romances": "Romance",
|
|
||||||
"science-fiction": "Science Fiction",
|
|
||||||
"science-fictions": "Science Fiction",
|
|
||||||
"surnaturel": "Supernatural",
|
|
||||||
"thriller": "Thriller",
|
|
||||||
"thrillers": "Thriller",
|
|
||||||
}
|
|
||||||
|
|
||||||
_LATEST_TTL_SECONDS = 600 # nouveautés : re-scrape au bout de 10 min
|
|
||||||
_MUST_WATCH_TTL_SECONDS = 21600 # incontournables : quasi statique, 6 h
|
|
||||||
_FOR_YOU_TTL_SECONDS = 3600 # recommandations : 1 h (l'historique évolue lentement)
|
|
||||||
_ENRICH_CONCURRENCY = 6 # enrichissements Kitsu parallèles max (nouveautés)
|
|
||||||
|
|
||||||
|
|
||||||
class _TTLCache:
|
|
||||||
"""Cache mémoire minimal avec expiration (mono-processus, suffisant ici)."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self._data: dict[str, tuple[float, object]] = {}
|
|
||||||
|
|
||||||
def get(self, key: str) -> object | None:
|
|
||||||
entry = self._data.get(key)
|
|
||||||
if entry is None:
|
|
||||||
return None
|
|
||||||
expires_at, value = entry
|
|
||||||
if expires_at <= time.monotonic():
|
|
||||||
del self._data[key]
|
|
||||||
return None
|
|
||||||
return value
|
|
||||||
|
|
||||||
def set(self, key: str, value: object, ttl_seconds: float) -> None:
|
|
||||||
self._data[key] = (time.monotonic() + ttl_seconds, value)
|
|
||||||
|
|
||||||
def clear(self, prefix: str = "") -> None:
|
|
||||||
"""Invalide les clés commençant par prefix (vide = tout le cache)."""
|
|
||||||
for key in [k for k in self._data if k.startswith(prefix)]:
|
|
||||||
del self._data[key]
|
|
||||||
|
|
||||||
|
|
||||||
def category_slug(name: str) -> str:
|
|
||||||
"""Nom de genre → slug de catégorie Kitsu (« Slice of Life » → « slice-of-life »)."""
|
|
||||||
decomposed = unicodedata.normalize("NFKD", name)
|
|
||||||
ascii_only = "".join(char for char in decomposed if not unicodedata.combining(char))
|
|
||||||
return re.sub(r"[^a-z0-9]+", "-", ascii_only.casefold()).strip("-")
|
|
||||||
|
|
||||||
|
|
||||||
class DiscoverService:
|
|
||||||
"""Agrégation des trois sections de découverte, avec cache mémoire."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self._cache = _TTLCache()
|
|
||||||
self._kitsu = KitsuService()
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ nouveautés
|
|
||||||
|
|
||||||
async def latest_by_type(self, limit: int = 24) -> dict[str, list[dict]]:
|
|
||||||
"""Nouveautés par type de média : ``{"anime": [...], "serie": [...], "film": [...]}``.
|
|
||||||
|
|
||||||
Les « récemment ajoutés » de chaque source activée sont fusionnés (doublons
|
|
||||||
retirés) puis répartis en rails indépendants — chaque type garde sa place,
|
|
||||||
aucun n'évince les autres. Le rail animés est enrichi via Kitsu (date de
|
|
||||||
sortie, statut) et trié du plus récent au plus ancien ; les séries et films
|
|
||||||
réels, absents de Kitsu, gardent l'ordre du site (déjà « récents d'abord »).
|
|
||||||
Chaque rail est tronqué à ``limit``.
|
|
||||||
"""
|
|
||||||
cache_key = f"latest_by_type:{limit}"
|
|
||||||
cached = self._cache.get(cache_key)
|
|
||||||
if cached is not None:
|
|
||||||
return cached # type: ignore[return-value]
|
|
||||||
|
|
||||||
sources = [s for s in all_sources() if await is_source_enabled(s.name)]
|
|
||||||
outcomes = await asyncio.gather(*(self._latest_of(source) for source in sources))
|
|
||||||
items = [item for outcome in outcomes for item in (outcome or [])]
|
|
||||||
|
|
||||||
merged: dict[str, dict] = {}
|
|
||||||
for item in items:
|
|
||||||
key = normalize_title(item["title"]).casefold()
|
|
||||||
existing = merged.get(key)
|
|
||||||
if existing is None or (not existing.get("image_url") and item.get("image_url")):
|
|
||||||
merged[key] = item
|
|
||||||
|
|
||||||
by_type: dict[str, list[dict]] = {"anime": [], "serie": [], "film": []}
|
|
||||||
for item in merged.values():
|
|
||||||
by_type.setdefault(item.get("media_type", "anime"), []).append(item)
|
|
||||||
|
|
||||||
# Enrichissement Kitsu limité aux animés (seul catalogue couvert) —
|
|
||||||
# inutile de bombarder l'API pour des titres qui n'y figurent pas.
|
|
||||||
semaphore = asyncio.Semaphore(_ENRICH_CONCURRENCY)
|
|
||||||
|
|
||||||
async def bounded(item: dict) -> dict:
|
|
||||||
async with semaphore:
|
|
||||||
return await self._with_release_info(item)
|
|
||||||
|
|
||||||
enriched = await asyncio.gather(*(bounded(item) for item in by_type["anime"]))
|
|
||||||
by_type["anime"] = sorted(
|
|
||||||
enriched, key=lambda it: it.get("start_date") or "", reverse=True
|
|
||||||
)[:limit]
|
|
||||||
for kind in ("serie", "film"):
|
|
||||||
by_type[kind] = by_type[kind][:limit]
|
|
||||||
|
|
||||||
self._cache.set(cache_key, by_type, _LATEST_TTL_SECONDS)
|
|
||||||
return by_type
|
|
||||||
|
|
||||||
async def _latest_of(self, source: SourceScraper) -> list[dict] | None:
|
|
||||||
"""Items latest() d'une source, aplatis avec les infos de source ([] si KO)."""
|
|
||||||
try:
|
|
||||||
results = await source.latest()
|
|
||||||
except ScrapeError as exc:
|
|
||||||
logger.warning("Nouveautés indisponibles pour %s : %s", source.name, exc)
|
|
||||||
return None
|
|
||||||
return [
|
|
||||||
{**dataclasses.asdict(r), "source": source.name, "label": source.label}
|
|
||||||
for r in results
|
|
||||||
]
|
|
||||||
|
|
||||||
async def _with_release_info(self, item: dict) -> dict:
|
|
||||||
"""Complète un item de nouveauté avec sa date de sortie Kitsu (None si absent)."""
|
|
||||||
item.setdefault("start_date", None)
|
|
||||||
item.setdefault("status", None)
|
|
||||||
item.setdefault("rating", None)
|
|
||||||
match = await self._kitsu_match_for_title(item["title"])
|
|
||||||
if match is None:
|
|
||||||
return item
|
|
||||||
attrs = match.get("attributes", {})
|
|
||||||
item["start_date"] = attrs.get("startDate")
|
|
||||||
item["status"] = attrs.get("status")
|
|
||||||
item["rating"] = KitsuService._to_rating_10(attrs.get("averageRating"))
|
|
||||||
return item
|
|
||||||
|
|
||||||
# --------------------------------------------------------- incontournables
|
|
||||||
|
|
||||||
async def must_watch(self, limit: int = _KITSU_PAGE_MAX) -> list[dict]:
|
|
||||||
"""Titres les plus populaires du catalogue Kitsu (tous temps)."""
|
|
||||||
limit = min(limit, _KITSU_PAGE_MAX)
|
|
||||||
key = f"must_watch:{limit}"
|
|
||||||
cached = self._cache.get(key)
|
|
||||||
if cached is not None:
|
|
||||||
return cached # type: ignore[return-value]
|
|
||||||
items = await self._kitsu_anime({"sort": "-userCount", "page[limit]": limit})
|
|
||||||
self._cache.set(key, items, _MUST_WATCH_TTL_SECONDS)
|
|
||||||
return items
|
|
||||||
|
|
||||||
|
|
||||||
async def browse_serie_film(self, media_type: str, genre: str, limit: int = 24) -> list[dict]:
|
|
||||||
"""Séries/films d'un genre French-Stream, aplatis comme les nouveautés.
|
|
||||||
|
|
||||||
Liste vide si la catégorie est inconnue ou la source en échec.
|
|
||||||
"""
|
|
||||||
key = f"browse_fs:{media_type}:{genre}:{limit}"
|
|
||||||
cached = self._cache.get(key)
|
|
||||||
if cached is not None:
|
|
||||||
return cached # type: ignore[return-value]
|
|
||||||
try:
|
|
||||||
scraper = get_source("french_stream")
|
|
||||||
results = await scraper.browse(media_type, genre)
|
|
||||||
except ScrapeError as exc:
|
|
||||||
logger.warning("Parcours %s/%s indisponible : %s", media_type, genre, exc)
|
|
||||||
return []
|
|
||||||
items = [
|
|
||||||
{**dataclasses.asdict(r), "source": scraper.name, "label": scraper.label}
|
|
||||||
for r in results
|
|
||||||
][:limit]
|
|
||||||
self._cache.set(key, items, _LATEST_TTL_SECONDS)
|
|
||||||
return items
|
|
||||||
async def browse_anime(self, genre: str, limit: int = _KITSU_PAGE_MAX) -> list[dict]:
|
|
||||||
"""Animés populaires d'une catégorie Kitsu (genre = slug du catalogue).
|
|
||||||
|
|
||||||
Liste vide si le genre est inconnu ou si Kitsu échoue (dégradation gracieuse).
|
|
||||||
"""
|
|
||||||
limit = min(limit, _KITSU_PAGE_MAX)
|
|
||||||
if genre not in ANIME_GENRES:
|
|
||||||
return []
|
|
||||||
key = f"browse_anime:{genre}:{limit}"
|
|
||||||
cached = self._cache.get(key)
|
|
||||||
if cached is not None:
|
|
||||||
return cached # type: ignore[return-value]
|
|
||||||
items = await self._kitsu_anime(
|
|
||||||
{"filter[categories]": genre, "sort": "-userCount", "page[limit]": limit}
|
|
||||||
)
|
|
||||||
self._cache.set(key, items, _FOR_YOU_TTL_SECONDS)
|
|
||||||
return items
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ pour toi
|
|
||||||
|
|
||||||
async def for_you(self, user_id: int, limit: int = _KITSU_PAGE_MAX) -> dict:
|
|
||||||
"""Recommandations par genres, à partir de l'historique de l'utilisateur.
|
|
||||||
|
|
||||||
Genres = téléchargements du serveur (titres → Kitsu) + favoris de l'utilisateur
|
|
||||||
(genres du payload) + séries téléchargées sur Sonarr (genres fournis par
|
|
||||||
Sonarr). On exclut les titres déjà possédés (local et Sonarr).
|
|
||||||
"""
|
|
||||||
limit = min(limit, _KITSU_PAGE_MAX)
|
|
||||||
cache_key = f"for_you:{user_id}:{limit}"
|
|
||||||
cached = self._cache.get(cache_key)
|
|
||||||
if cached is not None:
|
|
||||||
return cached # type: ignore[return-value]
|
|
||||||
|
|
||||||
owned, favorite_genres, series_pages = await self._owned(user_id)
|
|
||||||
genre_counts = await self._genres_from_downloads(owned)
|
|
||||||
for genre, count in favorite_genres.items():
|
|
||||||
genre_counts[genre] = genre_counts.get(genre, 0) + count
|
|
||||||
if series_pages:
|
|
||||||
# Séries/films téléchargés : genres lus sur leur fiche source (sans Sonarr)
|
|
||||||
for genre, count in (await self._genres_from_series_pages(series_pages)).items():
|
|
||||||
genre_counts[genre] = genre_counts.get(genre, 0) + count
|
|
||||||
sonarr_owned, sonarr_genres = await sonarr.profile()
|
|
||||||
for genre, count in sonarr_genres.items():
|
|
||||||
genre_counts[genre] = genre_counts.get(genre, 0) + count
|
|
||||||
|
|
||||||
if not (owned or favorite_genres or sonarr_owned):
|
|
||||||
# Aucun historique : proposition d'amorçage plutôt qu'une section muette
|
|
||||||
return {"based_on": [], "items": [], "cold_start": True}
|
|
||||||
owned |= sonarr_owned
|
|
||||||
|
|
||||||
if not genre_counts:
|
|
||||||
result: dict = {"based_on": [], "items": []}
|
|
||||||
self._cache.set(cache_key, result, _FOR_YOU_TTL_SECONDS)
|
|
||||||
return result
|
|
||||||
|
|
||||||
top_genres = sorted(genre_counts, key=genre_counts.get, reverse=True)[:_MAX_GENRES]
|
|
||||||
# Une requête par genre (Kitsu cumule les catégories en ET : quatre genres
|
|
||||||
# ensemble ne laissent que quelques titres — parfois le déjà-possédé !).
|
|
||||||
# Fusion entrelacée : chaque genre contribue, doublons retirés.
|
|
||||||
pools = [
|
|
||||||
list(pool)
|
|
||||||
for pool in await asyncio.gather(*(
|
|
||||||
self._kitsu_anime(
|
|
||||||
{"filter[categories]": category_slug(g), "sort": "-userCount", "page[limit]": limit}
|
|
||||||
)
|
|
||||||
for g in top_genres
|
|
||||||
))
|
|
||||||
if pool
|
|
||||||
]
|
|
||||||
candidates: list[dict] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
while pools:
|
|
||||||
for pool in pools[:]:
|
|
||||||
item = pool.pop(0)
|
|
||||||
key = str(item.get("kitsu_id") or item.get("title", "").casefold())
|
|
||||||
if key not in seen:
|
|
||||||
seen.add(key)
|
|
||||||
candidates.append(item)
|
|
||||||
if not pool:
|
|
||||||
pools.remove(pool)
|
|
||||||
kept = [
|
|
||||||
item for item in candidates
|
|
||||||
if item["title"] and normalize_title(item["title"]).casefold() not in owned
|
|
||||||
][:limit]
|
|
||||||
result = {"based_on": top_genres, "items": kept}
|
|
||||||
self._cache.set(cache_key, result, _FOR_YOU_TTL_SECONDS)
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def _genres_from_series_pages(self, pages: dict[str, str]) -> dict[str, int]:
|
|
||||||
"""Genres Kitsu des séries/films téléchargés, lus sur leur fiche source.
|
|
||||||
|
|
||||||
Les libellés français des fiches sont convertis vers les catégories Kitsu
|
|
||||||
(anglais) ; les genres sans équivalent sont ignorés.
|
|
||||||
"""
|
|
||||||
if not pages:
|
|
||||||
return {}
|
|
||||||
try:
|
|
||||||
scraper = get_source("french_stream")
|
|
||||||
except ScrapeError:
|
|
||||||
return {}
|
|
||||||
counts: dict[str, int] = {}
|
|
||||||
for page_url in list(pages.values())[:_MAX_HISTORY_TITLES]:
|
|
||||||
for genre_fr in await scraper.genres_of_page(page_url):
|
|
||||||
canonical = _GENRE_FR_TO_KITSU.get(category_slug(genre_fr))
|
|
||||||
if canonical:
|
|
||||||
counts[canonical] = counts.get(canonical, 0) + 1
|
|
||||||
return counts
|
|
||||||
|
|
||||||
def invalidate_for_you(self) -> None:
|
|
||||||
"""Recommandations recalculées au prochain appel (réglages Sonarr modifiés)."""
|
|
||||||
self._cache.clear("for_you:")
|
|
||||||
|
|
||||||
async def _owned(self, user_id: int) -> tuple[set[str], dict[str, int], dict[str, str]]:
|
|
||||||
"""Titres possédés (normalisés), genres des favoris, fiches des téléchargements.
|
|
||||||
|
|
||||||
``series_pages`` relie un titre possédé à l'URL de sa fiche chez la source
|
|
||||||
(séries/films French-Stream) : sans Sonarr, c'est la seule source de genres.
|
|
||||||
"""
|
|
||||||
rows = await db.fetchall(
|
|
||||||
"SELECT DISTINCT title, page_url FROM downloads ORDER BY created_at DESC LIMIT ?",
|
|
||||||
(_MAX_HISTORY_TITLES,),
|
|
||||||
)
|
|
||||||
fav_rows = await db.fetchall(
|
|
||||||
"SELECT payload FROM favorites WHERE user_id = ? ORDER BY created_at DESC LIMIT ?",
|
|
||||||
(user_id, _MAX_HISTORY_TITLES),
|
|
||||||
)
|
|
||||||
owned: set[str] = set()
|
|
||||||
series_pages: dict[str, str] = {}
|
|
||||||
for row in rows:
|
|
||||||
key = normalize_title(row["title"]).casefold()
|
|
||||||
if not key:
|
|
||||||
continue
|
|
||||||
owned.add(key)
|
|
||||||
page_url = row["page_url"] or ""
|
|
||||||
if page_url and key not in series_pages:
|
|
||||||
series_pages[key] = page_url
|
|
||||||
genre_counts: dict[str, int] = {}
|
|
||||||
for row in fav_rows:
|
|
||||||
try:
|
|
||||||
payload = json.loads(row["payload"]) if row["payload"] else {}
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
continue
|
|
||||||
for genre in payload.get("genres") or []:
|
|
||||||
if isinstance(genre, str) and genre.strip():
|
|
||||||
genre_counts[genre.strip()] = genre_counts.get(genre.strip(), 0) + 1
|
|
||||||
return owned, genre_counts, series_pages
|
|
||||||
|
|
||||||
async def _genres_from_downloads(self, titles: set[str]) -> dict[str, int]:
|
|
||||||
"""Genres Kitsu des titres téléchargés (cache DB puis recherche)."""
|
|
||||||
semaphore = asyncio.Semaphore(_MAX_HISTORY_TITLES)
|
|
||||||
|
|
||||||
async def genres_of(title: str) -> list[str]:
|
|
||||||
async with semaphore:
|
|
||||||
return await self._kitsu_genres_for_title(title)
|
|
||||||
|
|
||||||
outcomes = await asyncio.gather(*(genres_of(t) for t in list(titles)[:_MAX_HISTORY_TITLES]))
|
|
||||||
counts: dict[str, int] = {}
|
|
||||||
for genres in outcomes:
|
|
||||||
for genre in genres:
|
|
||||||
counts[genre] = counts.get(genre, 0) + 1
|
|
||||||
return counts
|
|
||||||
|
|
||||||
async def _kitsu_match_for_title(self, title: str) -> dict | None:
|
|
||||||
"""Match Kitsu d'un titre scrapé (cache DB 72 h via metadata_cache)."""
|
|
||||||
query = normalize_title(title)
|
|
||||||
if not query:
|
|
||||||
return None
|
|
||||||
cache_key = f"kitsu:anime:{query.casefold()}"
|
|
||||||
match = await self._kitsu.get_cached(cache_key)
|
|
||||||
if match is None:
|
|
||||||
match = await self._kitsu.search_anime(title)
|
|
||||||
if match is not None:
|
|
||||||
await self._kitsu.set_cached(cache_key, match)
|
|
||||||
return match
|
|
||||||
|
|
||||||
async def _kitsu_genres_for_title(self, title: str) -> list[str]:
|
|
||||||
"""Genres Kitsu d'un titre scrapé (cache DB 72 h via metadata_cache).
|
|
||||||
|
|
||||||
La recherche Kitsu ne renvoie plus les genres (`include=genres` vide) :
|
|
||||||
on complète avec l'endpoint /anime/<id>/categories.
|
|
||||||
"""
|
|
||||||
match = await self._kitsu_match_for_title(title)
|
|
||||||
if match is None:
|
|
||||||
return []
|
|
||||||
genres = [g for g in match.get("genres", []) if isinstance(g, str)]
|
|
||||||
if not genres:
|
|
||||||
genres = await self._kitsu_categories(match.get("id"))
|
|
||||||
match["genres"] = genres
|
|
||||||
query = normalize_title(title)
|
|
||||||
await self._kitsu.set_cached( # refresh avec les genres
|
|
||||||
f"kitsu:anime:{query.casefold()}", match
|
|
||||||
)
|
|
||||||
return genres
|
|
||||||
|
|
||||||
# -------------------------------------------------------------- Kitsu
|
|
||||||
|
|
||||||
async def _kitsu_anime(self, params: dict) -> list[dict]:
|
|
||||||
"""Requête générique liste Kitsu → items normalisés ([] si échec)."""
|
|
||||||
settings = get_settings()
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(
|
|
||||||
timeout=settings.http_timeout,
|
|
||||||
headers={
|
|
||||||
"User-Agent": settings.user_agent,
|
|
||||||
"Accept": "application/vnd.api+json",
|
|
||||||
},
|
|
||||||
) as client:
|
|
||||||
response = await client.get(f"{settings.kitsu_base_url}/anime", params=params)
|
|
||||||
response.raise_for_status()
|
|
||||||
payload = response.json()
|
|
||||||
except (httpx.HTTPError, ValueError) as exc:
|
|
||||||
logger.warning("Liste Kitsu échouée (%s) : %s", params, exc)
|
|
||||||
return []
|
|
||||||
return [
|
|
||||||
self._normalize_anime(item)
|
|
||||||
for item in payload.get("data", [])
|
|
||||||
if item.get("type") == "anime"
|
|
||||||
]
|
|
||||||
|
|
||||||
async def _kitsu_categories(self, anime_id: object) -> list[str]:
|
|
||||||
"""Titres des catégories Kitsu d'un anime ([] si échec)."""
|
|
||||||
if not anime_id:
|
|
||||||
return []
|
|
||||||
settings = get_settings()
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(
|
|
||||||
timeout=settings.http_timeout,
|
|
||||||
headers={
|
|
||||||
"User-Agent": settings.user_agent,
|
|
||||||
"Accept": "application/vnd.api+json",
|
|
||||||
},
|
|
||||||
) as client:
|
|
||||||
response = await client.get(
|
|
||||||
f"{settings.kitsu_base_url}/anime/{anime_id}/categories",
|
|
||||||
params={"page[limit]": _KITSU_PAGE_MAX},
|
|
||||||
)
|
|
||||||
response.raise_for_status()
|
|
||||||
payload = response.json()
|
|
||||||
except (httpx.HTTPError, ValueError) as exc:
|
|
||||||
logger.warning("Catégories Kitsu échouées (anime %s) : %s", anime_id, exc)
|
|
||||||
return []
|
|
||||||
return [
|
|
||||||
attrs["title"]
|
|
||||||
for item in payload.get("data", [])
|
|
||||||
if isinstance(attrs := item.get("attributes", {}), dict) and attrs.get("title")
|
|
||||||
]
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _normalize_anime(item: dict) -> dict:
|
|
||||||
attrs = item.get("attributes", {})
|
|
||||||
titles = attrs.get("titles") or {}
|
|
||||||
poster = attrs.get("posterImage") or {}
|
|
||||||
return {
|
|
||||||
"kitsu_id": item.get("id"),
|
|
||||||
"title": attrs.get("canonicalTitle") or titles.get("en_jp"),
|
|
||||||
"image_url": poster.get("large") or poster.get("medium") or poster.get("tiny"),
|
|
||||||
"rating": KitsuService._to_rating_10(attrs.get("averageRating")),
|
|
||||||
"year": KitsuService._extract_year(attrs.get("startDate")),
|
|
||||||
"subtype": attrs.get("subtype"),
|
|
||||||
"user_count": attrs.get("userCount"),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
discover = DiscoverService()
|
|
||||||
@@ -1,586 +0,0 @@
|
|||||||
"""Gestionnaire de téléchargements : file asyncio, parallélisme limité,
|
|
||||||
pause/reprise (Range HTTP), anti-doublons, persistance, progression temps réel.
|
|
||||||
|
|
||||||
Les statuts : pending → downloading → done | failed | cancelled
|
|
||||||
↕ paused
|
|
||||||
"""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import contextlib
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import signal
|
|
||||||
import time
|
|
||||||
import unicodedata
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
from urllib.parse import urljoin
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.db import db
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
ACTIVE_STATUSES = ("pending", "downloading", "paused")
|
|
||||||
|
|
||||||
_FFMPEG_TIME_RE = re.compile(r"time=(\d+:\d+:\d+(?:\.\d+)?)")
|
|
||||||
_EXTINF_RE = re.compile(r"#EXTINF:([\d.]+)")
|
|
||||||
_BANDWIDTH_RE = re.compile(r"#EXT-X-STREAM-INF:[^\n]*BANDWIDTH=(\d+)[^\n]*\n(\S+)")
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_ffmpeg_time(value: str) -> float:
|
|
||||||
parts = value.split(":")
|
|
||||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
|
||||||
|
|
||||||
|
|
||||||
def _best_variant(master_body: str, base_url: str) -> str | None:
|
|
||||||
"""URL de la variante au plus haut débit d'une playlist maître HLS."""
|
|
||||||
variants = [
|
|
||||||
(int(bw), urljoin(base_url, uri)) for bw, uri in _BANDWIDTH_RE.findall(master_body)
|
|
||||||
]
|
|
||||||
return max(variants)[1] if variants else None
|
|
||||||
|
|
||||||
_STATUS_LABELS = {
|
|
||||||
"pending": "en attente",
|
|
||||||
"downloading": "en cours",
|
|
||||||
"paused": "en pause",
|
|
||||||
"done": "terminé",
|
|
||||||
"failed": "échec",
|
|
||||||
"cancelled": "annulé",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_filename(name: str) -> str:
|
|
||||||
"""Nettoie un nom de fichier : caractères interdits retirés, anti-traversée."""
|
|
||||||
name = unicodedata.normalize("NFKC", name)
|
|
||||||
name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', " ", name)
|
|
||||||
name = re.sub(r"\s+", " ", name).strip(" .")
|
|
||||||
if not name or name in (".", ".."):
|
|
||||||
name = "video"
|
|
||||||
return name[:150]
|
|
||||||
|
|
||||||
|
|
||||||
_SERIES_RE = re.compile(r"^(.*?)[\s\-–—]*(?:épisode|episode|ep|e)\s*\d", re.IGNORECASE)
|
|
||||||
|
|
||||||
|
|
||||||
def series_dirname(title: str) -> str | None:
|
|
||||||
"""Nom du sous-dossier d'un titre (« One Piece - E12 » → « One Piece »).
|
|
||||||
|
|
||||||
Range les épisodes par animé sur le disque (bien meilleur parsing Plex).
|
|
||||||
None si aucun marqueur d'épisode détecté → fichier à plat.
|
|
||||||
"""
|
|
||||||
match = _SERIES_RE.match(title)
|
|
||||||
base = sanitize_filename(match.group(1).strip()) if match else ""
|
|
||||||
return base or None
|
|
||||||
|
|
||||||
|
|
||||||
_EPISODE_RE = re.compile(
|
|
||||||
r"^(?P<series>.+?)\s*-\s*(?:Saison\s+(?P<season>\d+)\s*-\s*)?"
|
|
||||||
r"E(?P<ep>\d+(?:[.,]\d+)?)(?P<tail>\s*\([^)]*\))?\s*$",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def plex_filename(title: str, extension: str) -> str | None:
|
|
||||||
"""Nom de fichier compatible Plex (« S01E09 ») pour un titre d'épisode.
|
|
||||||
|
|
||||||
Les scrapers produisent « Série - Saison 1 - E9 (VF) » ; le scanner Plex
|
|
||||||
exige « Série - S01E09 (VF) » (les épisodes 1-9 sont sinon ignorés).
|
|
||||||
None si le titre ne suit pas le format épisode → nommage inchangé.
|
|
||||||
"""
|
|
||||||
match = _EPISODE_RE.match(title)
|
|
||||||
if not match:
|
|
||||||
return None
|
|
||||||
series = match.group("series")
|
|
||||||
season = int(match.group("season") or 1)
|
|
||||||
ep_raw = match.group("ep").replace(",", ".")
|
|
||||||
ep = float(ep_raw)
|
|
||||||
ep_label = f"{int(ep):02d}" if ep.is_integer() else ep_raw
|
|
||||||
tail = match.group("tail") or ""
|
|
||||||
return sanitize_filename(f"{series} - S{season:02d}E{ep_label}{tail}") + extension
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadManager:
|
|
||||||
"""File d'attente de téléchargements, injectée dans les routes via app.state."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self._queue: asyncio.Queue[int] # créée dans start() (affinité avec la boucle)
|
|
||||||
self._tasks: dict[int, asyncio.Task] = {} # download_id → tâche asyncio
|
|
||||||
self._pause_events: dict[int, asyncio.Event] = {} # set = peut tourner
|
|
||||||
self._progress: dict[int, dict[str, Any]] = {} # progression temps réel en mémoire
|
|
||||||
self._listeners: list[asyncio.Queue] = []
|
|
||||||
self._workers: list[asyncio.Task] = []
|
|
||||||
self._client: httpx.AsyncClient | None = None
|
|
||||||
self._hls_processes: dict[int, asyncio.subprocess.Process] = {}
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ cycle de vie
|
|
||||||
|
|
||||||
async def start(self) -> None:
|
|
||||||
self._queue = asyncio.Queue()
|
|
||||||
settings = get_settings()
|
|
||||||
self._client = httpx.AsyncClient(
|
|
||||||
timeout=httpx.Timeout(30.0, read=300.0),
|
|
||||||
follow_redirects=True,
|
|
||||||
headers={
|
|
||||||
"User-Agent": settings.user_agent,
|
|
||||||
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
# Restaure les téléchargements interrompus (crash/arrêt) en 'pending'
|
|
||||||
await db.execute(
|
|
||||||
"UPDATE downloads SET status = 'pending', updated_at = datetime('now') "
|
|
||||||
"WHERE status = 'downloading'"
|
|
||||||
)
|
|
||||||
await self._scan_download_dir()
|
|
||||||
for _ in range(settings.max_parallel_downloads):
|
|
||||||
self._workers.append(asyncio.create_task(self._worker()))
|
|
||||||
logger.info("DownloadManager démarré (%d workers)", settings.max_parallel_downloads)
|
|
||||||
|
|
||||||
async def stop(self) -> None:
|
|
||||||
for worker in self._workers:
|
|
||||||
worker.cancel()
|
|
||||||
for task in self._tasks.values():
|
|
||||||
task.cancel()
|
|
||||||
for proc in self._hls_processes.values():
|
|
||||||
if proc.returncode is None:
|
|
||||||
proc.kill()
|
|
||||||
if self._client:
|
|
||||||
await self._client.aclose()
|
|
||||||
self._workers.clear()
|
|
||||||
self._tasks.clear()
|
|
||||||
|
|
||||||
async def _scan_download_dir(self) -> None:
|
|
||||||
"""Restaure en 'done' les fichiers présents sur disque mais inconnus de la DB."""
|
|
||||||
download_dir = get_settings().download_dir
|
|
||||||
rows = await db.fetchall("SELECT file_path FROM downloads WHERE file_path IS NOT NULL")
|
|
||||||
known = {row["file_path"] for row in rows}
|
|
||||||
for path in download_dir.rglob("*"):
|
|
||||||
if not path.is_file() or path.suffix == ".part":
|
|
||||||
continue
|
|
||||||
rel_path = path.relative_to(download_dir).as_posix()
|
|
||||||
if rel_path in known:
|
|
||||||
continue
|
|
||||||
size = path.stat().st_size
|
|
||||||
await db.execute(
|
|
||||||
"INSERT INTO downloads "
|
|
||||||
"(source_key, video_url, page_url, title, file_path, status, "
|
|
||||||
" total_bytes, downloaded_bytes) "
|
|
||||||
"VALUES (?, ?, ?, ?, ?, 'done', ?, ?)",
|
|
||||||
(
|
|
||||||
f"file:{rel_path}",
|
|
||||||
"",
|
|
||||||
"",
|
|
||||||
path.stem,
|
|
||||||
rel_path,
|
|
||||||
size,
|
|
||||||
size,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
logger.info("Fichier restauré depuis le disque : %s", rel_path)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ API publique
|
|
||||||
|
|
||||||
async def enqueue(
|
|
||||||
self, video_url: str, page_url: str, title: str, source_key: str | None = None
|
|
||||||
) -> dict:
|
|
||||||
"""Ajoute un téléchargement ; retourne la tâche existante si doublon actif.
|
|
||||||
|
|
||||||
source_key : clé de déduplication (par défaut l'URL vidéo). Les grabs
|
|
||||||
Sonarr utilisent « sonarr:<infohash>|<url> » pour rester suivis.
|
|
||||||
"""
|
|
||||||
source_key = source_key or video_url
|
|
||||||
existing = await db.fetchone(
|
|
||||||
f"SELECT * FROM downloads WHERE source_key = ? AND status IN "
|
|
||||||
f"({','.join('?' * len(ACTIVE_STATUSES))})",
|
|
||||||
(source_key, *ACTIVE_STATUSES),
|
|
||||||
)
|
|
||||||
if existing:
|
|
||||||
logger.info("Anti-doublon : %s déjà en file (id=%s)", title, existing["id"])
|
|
||||||
return self._to_dict(existing, duplicate=True)
|
|
||||||
|
|
||||||
extension = self._guess_extension(video_url)
|
|
||||||
filename = plex_filename(title, extension) or sanitize_filename(title) + extension
|
|
||||||
series = series_dirname(title)
|
|
||||||
file_path = f"{series}/{filename}" if series else filename
|
|
||||||
cursor = await db.execute(
|
|
||||||
"INSERT INTO downloads (source_key, video_url, page_url, title, file_path) "
|
|
||||||
"VALUES (?, ?, ?, ?, ?)",
|
|
||||||
(source_key, video_url, page_url, title, file_path),
|
|
||||||
)
|
|
||||||
download_id = cursor.lastrowid
|
|
||||||
await self._queue.put(download_id)
|
|
||||||
await self._emit(download_id)
|
|
||||||
logger.info("Téléchargement ajouté : %s (id=%s)", title, download_id)
|
|
||||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
|
||||||
return self._to_dict(row)
|
|
||||||
|
|
||||||
async def pause(self, download_id: int) -> bool:
|
|
||||||
event = self._pause_events.get(download_id)
|
|
||||||
if event:
|
|
||||||
event.clear()
|
|
||||||
await self._set_status(download_id, "paused")
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def resume(self, download_id: int) -> bool:
|
|
||||||
row = await self._get_row(download_id)
|
|
||||||
if row["status"] != "paused":
|
|
||||||
return False
|
|
||||||
await self._set_status(download_id, "pending")
|
|
||||||
await self._queue.put(download_id)
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def retry(self, download_id: int) -> bool:
|
|
||||||
row = await self._get_row(download_id)
|
|
||||||
if row["status"] not in ("failed", "cancelled"):
|
|
||||||
return False
|
|
||||||
await db.execute(
|
|
||||||
"UPDATE downloads SET status = 'pending', error = NULL, downloaded_bytes = 0, "
|
|
||||||
"updated_at = datetime('now') WHERE id = ?",
|
|
||||||
(download_id,),
|
|
||||||
)
|
|
||||||
part = self._part_path(row["file_path"])
|
|
||||||
part.unlink(missing_ok=True)
|
|
||||||
await self._queue.put(download_id)
|
|
||||||
await self._emit(download_id)
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def cancel(self, download_id: int) -> bool:
|
|
||||||
task = self._tasks.get(download_id)
|
|
||||||
if task:
|
|
||||||
task.cancel()
|
|
||||||
proc = self._hls_processes.get(download_id)
|
|
||||||
if proc and proc.returncode is None:
|
|
||||||
proc.kill()
|
|
||||||
await self._set_status(download_id, "cancelled")
|
|
||||||
row = await self._get_row(download_id)
|
|
||||||
self._part_path(row["file_path"]).unlink(missing_ok=True)
|
|
||||||
await self._emit(download_id)
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def cancel_all(self) -> int:
|
|
||||||
rows = await db.fetchall(
|
|
||||||
f"SELECT id FROM downloads WHERE status IN ({','.join('?' * len(ACTIVE_STATUSES))})",
|
|
||||||
ACTIVE_STATUSES,
|
|
||||||
)
|
|
||||||
for row in rows:
|
|
||||||
await self.cancel(row["id"])
|
|
||||||
return len(rows)
|
|
||||||
|
|
||||||
async def clear_finished(self) -> int:
|
|
||||||
"""Supprime de la file les tâches terminées/échouées/annulées (fichiers gardés)."""
|
|
||||||
cursor = await db.execute(
|
|
||||||
"DELETE FROM downloads WHERE status IN ('done', 'failed', 'cancelled')"
|
|
||||||
)
|
|
||||||
return cursor.rowcount or 0
|
|
||||||
|
|
||||||
async def delete(self, download_id: int, delete_file: bool = False) -> bool:
|
|
||||||
"""Supprime une tâche de la file (annulée avant si active).
|
|
||||||
|
|
||||||
delete_file=True efface aussi le fichier téléchargé du disque.
|
|
||||||
"""
|
|
||||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
|
||||||
if row is None:
|
|
||||||
return False
|
|
||||||
if row["status"] in ACTIVE_STATUSES:
|
|
||||||
await self.cancel(download_id)
|
|
||||||
if delete_file and row["file_path"]:
|
|
||||||
target = get_settings().download_dir / row["file_path"]
|
|
||||||
target.unlink(missing_ok=True)
|
|
||||||
if target.parent != get_settings().download_dir:
|
|
||||||
with contextlib.suppress(OSError): # dossier de l'animé vide → retiré
|
|
||||||
target.parent.rmdir()
|
|
||||||
self._part_path(row["file_path"]).unlink(missing_ok=True)
|
|
||||||
await db.execute("DELETE FROM downloads WHERE id = ?", (download_id,))
|
|
||||||
await self._emit_removed(download_id)
|
|
||||||
logger.info("Téléchargement supprimé (id=%s, fichier=%s)", download_id, delete_file)
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def list_all(self, limit: int = 2000) -> list[dict]:
|
|
||||||
rows = await db.fetchall(
|
|
||||||
"SELECT * FROM downloads ORDER BY "
|
|
||||||
"CASE status WHEN 'downloading' THEN 0 WHEN 'pending' THEN 1 WHEN 'paused' THEN 2 "
|
|
||||||
"ELSE 3 END, updated_at DESC LIMIT ?",
|
|
||||||
(limit,),
|
|
||||||
)
|
|
||||||
return [self._to_dict(row) for row in rows]
|
|
||||||
|
|
||||||
async def get(self, download_id: int) -> dict | None:
|
|
||||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
|
||||||
return self._to_dict(row) if row else None
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ événements SSE
|
|
||||||
|
|
||||||
async def subscribe(self) -> AsyncIterator[dict]:
|
|
||||||
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
|
|
||||||
self._listeners.append(queue)
|
|
||||||
try:
|
|
||||||
while True:
|
|
||||||
yield await queue.get()
|
|
||||||
finally:
|
|
||||||
self._listeners.remove(queue)
|
|
||||||
|
|
||||||
async def _emit(self, download_id: int) -> None:
|
|
||||||
data = await self.get(download_id)
|
|
||||||
if data is None:
|
|
||||||
return
|
|
||||||
self._publish({"type": "update", "item": data})
|
|
||||||
|
|
||||||
async def _emit_removed(self, download_id: int) -> None:
|
|
||||||
self._publish({"type": "removed", "id": download_id})
|
|
||||||
|
|
||||||
def _publish(self, message: dict) -> None:
|
|
||||||
for queue in self._listeners:
|
|
||||||
with contextlib.suppress(asyncio.QueueFull):
|
|
||||||
queue.put_nowait(message)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ worker interne
|
|
||||||
|
|
||||||
async def _worker(self) -> None:
|
|
||||||
while True:
|
|
||||||
download_id = await self._queue.get()
|
|
||||||
row = await db.fetchone("SELECT status FROM downloads WHERE id = ?", (download_id,))
|
|
||||||
if row is None or row["status"] != "pending":
|
|
||||||
continue # annulé/pausé entre-temps
|
|
||||||
# Tâche dédiée : annuler un téléchargement ne doit pas tuer le worker
|
|
||||||
task = asyncio.create_task(self._download(download_id))
|
|
||||||
self._tasks[download_id] = task
|
|
||||||
self._pause_events[download_id] = asyncio.Event()
|
|
||||||
self._pause_events[download_id].set()
|
|
||||||
try:
|
|
||||||
await task
|
|
||||||
except asyncio.CancelledError:
|
|
||||||
if asyncio.current_task().cancelling() > 0:
|
|
||||||
raise # le worker lui-même s'arrête (stop())
|
|
||||||
finally:
|
|
||||||
self._tasks.pop(download_id, None)
|
|
||||||
self._pause_events.pop(download_id, None)
|
|
||||||
self._progress.pop(download_id, None)
|
|
||||||
|
|
||||||
async def _download(self, download_id: int) -> None:
|
|
||||||
"""Dispatche HTTP/HLS ; gestion d'erreurs centralisée ici."""
|
|
||||||
row = await self._get_row(download_id)
|
|
||||||
try:
|
|
||||||
if ".m3u8" in row["video_url"]:
|
|
||||||
await self._download_hls(download_id, row)
|
|
||||||
else:
|
|
||||||
await self._download_http(download_id, row)
|
|
||||||
except asyncio.CancelledError:
|
|
||||||
logger.info("Téléchargement annulé : %s", row["file_path"])
|
|
||||||
raise
|
|
||||||
except (httpx.HTTPError, OSError) as exc:
|
|
||||||
# Échec réseau/disque : journalisé, statut 'failed' visible dans l'UI
|
|
||||||
logger.error("Échec du téléchargement de %s : %s", row["file_path"], exc)
|
|
||||||
part = self._part_path(row["file_path"])
|
|
||||||
downloaded = part.stat().st_size if part.exists() else 0
|
|
||||||
await self._fail(download_id, exc, downloaded)
|
|
||||||
|
|
||||||
async def _download_http(self, download_id: int, row: Any) -> None:
|
|
||||||
"""Téléchargement HTTP direct avec reprise via Range."""
|
|
||||||
video_url, file_path = row["video_url"], row["file_path"]
|
|
||||||
target = get_settings().download_dir / file_path
|
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
part = self._part_path(file_path)
|
|
||||||
downloaded = part.stat().st_size if part.exists() else 0
|
|
||||||
|
|
||||||
headers: dict[str, str] = {}
|
|
||||||
if row["page_url"]:
|
|
||||||
headers["Referer"] = row["page_url"]
|
|
||||||
if downloaded:
|
|
||||||
headers["Range"] = f"bytes={downloaded}-"
|
|
||||||
logger.info("Reprise de %s à %d octets", file_path, downloaded)
|
|
||||||
|
|
||||||
await self._set_status(download_id, "downloading")
|
|
||||||
await self._emit(download_id)
|
|
||||||
started = time.monotonic()
|
|
||||||
last_emit = 0.0
|
|
||||||
|
|
||||||
async with self._client.stream("GET", video_url, headers=headers) as response:
|
|
||||||
if response.status_code == 416: # plage invalide → déjà complet
|
|
||||||
part.rename(target)
|
|
||||||
await self._finish(download_id, downloaded)
|
|
||||||
return
|
|
||||||
response.raise_for_status()
|
|
||||||
if downloaded and response.status_code != 206:
|
|
||||||
downloaded = 0 # serveur sans support Range → on repart de zéro
|
|
||||||
logger.warning("Pas de reprise possible pour %s", file_path)
|
|
||||||
total = int(response.headers.get("content-length") or 0) + downloaded or None
|
|
||||||
await db.execute(
|
|
||||||
"UPDATE downloads SET total_bytes = ? WHERE id = ?", (total, download_id)
|
|
||||||
)
|
|
||||||
|
|
||||||
mode = "ab" if downloaded else "wb"
|
|
||||||
with part.open(mode) as fh:
|
|
||||||
async for chunk in response.aiter_bytes(1 << 16):
|
|
||||||
event = self._pause_events.get(download_id)
|
|
||||||
if event is not None:
|
|
||||||
await event.wait() # pause coopérative
|
|
||||||
fh.write(chunk)
|
|
||||||
downloaded += len(chunk)
|
|
||||||
now = time.monotonic()
|
|
||||||
if now - last_emit >= 1.0:
|
|
||||||
last_emit = now
|
|
||||||
await self._report_progress(download_id, downloaded, total, started)
|
|
||||||
|
|
||||||
part.rename(target)
|
|
||||||
await self._finish(download_id, downloaded)
|
|
||||||
|
|
||||||
async def _download_hls(self, download_id: int, row: Any) -> None:
|
|
||||||
"""Télécharge un flux HLS (.m3u8) via ffmpeg (remux en mp4).
|
|
||||||
|
|
||||||
Pause via SIGSTOP/SIGCONT du processus, annulation via kill.
|
|
||||||
La progression est estimée depuis la durée totale de la playlist.
|
|
||||||
"""
|
|
||||||
video_url, file_path = row["video_url"], row["file_path"]
|
|
||||||
target = get_settings().download_dir / file_path
|
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
part = self._part_path(file_path)
|
|
||||||
part.unlink(missing_ok=True) # pas de reprise partielle en HLS
|
|
||||||
|
|
||||||
ffmpeg_headers = f"User-Agent: {get_settings().user_agent}\r\n"
|
|
||||||
if row["page_url"]:
|
|
||||||
ffmpeg_headers += f"Referer: {row['page_url']}\r\n"
|
|
||||||
ffmpeg_headers += "Accept-Language: fr-FR,fr;q=0.9,en;q=0.8\r\n"
|
|
||||||
|
|
||||||
total_seconds = await self._hls_duration(video_url, row["page_url"])
|
|
||||||
|
|
||||||
await self._set_status(download_id, "downloading")
|
|
||||||
await self._emit(download_id)
|
|
||||||
started = time.monotonic()
|
|
||||||
|
|
||||||
process = await asyncio.create_subprocess_exec(
|
|
||||||
"ffmpeg", "-y", "-nostdin", "-v", "error", "-nostats", "-progress", "pipe:2",
|
|
||||||
"-headers", ffmpeg_headers,
|
|
||||||
"-i", video_url,
|
|
||||||
"-c", "copy", "-bsf:a", "aac_adtstoasc", "-f", "mp4",
|
|
||||||
str(part),
|
|
||||||
stdout=asyncio.subprocess.DEVNULL,
|
|
||||||
stderr=asyncio.subprocess.PIPE,
|
|
||||||
)
|
|
||||||
self._hls_processes[download_id] = process
|
|
||||||
try:
|
|
||||||
assert process.stderr is not None
|
|
||||||
async for raw_line in process.stderr:
|
|
||||||
event = self._pause_events.get(download_id)
|
|
||||||
if event is not None and not event.is_set():
|
|
||||||
process.send_signal(signal.SIGSTOP)
|
|
||||||
await event.wait()
|
|
||||||
process.send_signal(signal.SIGCONT)
|
|
||||||
line = raw_line.decode(errors="replace")
|
|
||||||
if match := _FFMPEG_TIME_RE.search(line):
|
|
||||||
elapsed_video = _parse_ffmpeg_time(match.group(1))
|
|
||||||
size = part.stat().st_size if part.exists() else 0
|
|
||||||
total_est = None
|
|
||||||
if total_seconds and elapsed_video > 0:
|
|
||||||
total_est = int(size / elapsed_video * total_seconds)
|
|
||||||
await self._report_progress(download_id, size, total_est, started)
|
|
||||||
return_code = await process.wait()
|
|
||||||
finally:
|
|
||||||
self._hls_processes.pop(download_id, None)
|
|
||||||
|
|
||||||
if return_code != 0:
|
|
||||||
part.unlink(missing_ok=True)
|
|
||||||
raise OSError(f"ffmpeg a échoué (code {return_code}) sur le flux HLS")
|
|
||||||
size = part.stat().st_size
|
|
||||||
part.rename(target)
|
|
||||||
await self._finish(download_id, size)
|
|
||||||
|
|
||||||
async def _hls_duration(self, playlist_url: str, referer: str | None) -> float | None:
|
|
||||||
"""Durée totale d'une playlist HLS (somme des EXTINF de la variante max)."""
|
|
||||||
headers = {"Referer": referer} if referer else {}
|
|
||||||
try:
|
|
||||||
response = await self._client.get(playlist_url, headers=headers)
|
|
||||||
response.raise_for_status()
|
|
||||||
body = response.text
|
|
||||||
# Playlist maître → on suit la variante de plus haut débit
|
|
||||||
variant = _best_variant(body, playlist_url)
|
|
||||||
if variant and variant != playlist_url:
|
|
||||||
response = await self._client.get(variant, headers=headers)
|
|
||||||
response.raise_for_status()
|
|
||||||
body = response.text
|
|
||||||
durations = [float(m) for m in _EXTINF_RE.findall(body)]
|
|
||||||
return sum(durations) if durations else None
|
|
||||||
except (httpx.HTTPError, ValueError) as exc:
|
|
||||||
logger.warning("Durée HLS indéterminée pour %s : %s", playlist_url, exc)
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def _report_progress(
|
|
||||||
self, download_id: int, downloaded: int, total: int | None, started: float
|
|
||||||
) -> None:
|
|
||||||
elapsed = time.monotonic() - started
|
|
||||||
self._progress[download_id] = {
|
|
||||||
"downloaded_bytes": downloaded,
|
|
||||||
"total_bytes": total,
|
|
||||||
"speed_bps": int(downloaded / elapsed) if elapsed > 0 else 0,
|
|
||||||
}
|
|
||||||
await self._emit(download_id)
|
|
||||||
|
|
||||||
async def _fail(self, download_id: int, exc: Exception, downloaded: int = 0) -> None:
|
|
||||||
await db.execute(
|
|
||||||
"UPDATE downloads SET status = 'failed', error = ?, "
|
|
||||||
"downloaded_bytes = ?, updated_at = datetime('now') WHERE id = ?",
|
|
||||||
(str(exc)[:500], downloaded, download_id),
|
|
||||||
)
|
|
||||||
await self._emit(download_id)
|
|
||||||
|
|
||||||
async def _finish(self, download_id: int, size: int) -> None:
|
|
||||||
await db.execute(
|
|
||||||
"UPDATE downloads SET status = 'done', total_bytes = ?, downloaded_bytes = ?, "
|
|
||||||
"updated_at = datetime('now') WHERE id = ?",
|
|
||||||
(size, size, download_id),
|
|
||||||
)
|
|
||||||
await self._emit(download_id)
|
|
||||||
logger.info("Téléchargement terminé (id=%s, %d octets)", download_id, size)
|
|
||||||
# L'historique a changé : les recommandations « Pour toi » se recalculent
|
|
||||||
from app.services.discover import discover
|
|
||||||
|
|
||||||
discover.invalidate_for_you()
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ helpers
|
|
||||||
|
|
||||||
async def _get_row(self, download_id: int) -> Any:
|
|
||||||
row = await db.fetchone("SELECT * FROM downloads WHERE id = ?", (download_id,))
|
|
||||||
if row is None:
|
|
||||||
raise KeyError(f"Téléchargement introuvable : {download_id}")
|
|
||||||
return row
|
|
||||||
|
|
||||||
async def _set_status(self, download_id: int, status: str) -> None:
|
|
||||||
await db.execute(
|
|
||||||
"UPDATE downloads SET status = ?, updated_at = datetime('now') WHERE id = ?",
|
|
||||||
(status, download_id),
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _part_path(file_path: str | None) -> Path:
|
|
||||||
name = file_path or "video"
|
|
||||||
return get_settings().download_dir / (name + ".part")
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _guess_extension(url: str) -> str:
|
|
||||||
match = re.search(r"\.(mp4|mkv|webm|avi|m3u8)(?:\?|$)", url)
|
|
||||||
ext = match.group(1) if match else "mp4"
|
|
||||||
return ".mp4" if ext == "m3u8" else f".{ext}"
|
|
||||||
|
|
||||||
def _to_dict(self, row: Any, duplicate: bool = False) -> dict:
|
|
||||||
data = dict(row)
|
|
||||||
live = self._progress.get(data["id"], {})
|
|
||||||
downloaded = live.get("downloaded_bytes", data["downloaded_bytes"])
|
|
||||||
total = live.get("total_bytes", data["total_bytes"])
|
|
||||||
speed = live.get("speed_bps", 0)
|
|
||||||
percent = round(downloaded / total * 100, 1) if total else None
|
|
||||||
eta = int((total - downloaded) / speed) if total and speed else None
|
|
||||||
data.update(
|
|
||||||
downloaded_bytes=downloaded,
|
|
||||||
total_bytes=total,
|
|
||||||
percent=percent,
|
|
||||||
speed_bps=speed,
|
|
||||||
eta_seconds=eta,
|
|
||||||
status_label=_STATUS_LABELS.get(data["status"], data["status"]),
|
|
||||||
duplicate=duplicate,
|
|
||||||
)
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
download_manager = DownloadManager()
|
|
||||||
@@ -1,209 +0,0 @@
|
|||||||
"""Enrichissement de métadonnées via l'API Kitsu (https://kitsu.io/api/edge).
|
|
||||||
|
|
||||||
Fusion intelligente : seuls les champs manquants d'un TitleDetails sont complétés.
|
|
||||||
Les échecs (réseau, cache indisponible) sont loggés en warning et jamais remontés
|
|
||||||
au caller — la fiche d'origine est retournée inchangée (dégradation gracieuse).
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import sqlite3
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.db import db
|
|
||||||
from app.scrapers.base import TitleDetails
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_NOISE_WORDS_RE = re.compile(
|
|
||||||
r"\b(?:VOSTFR\d*|VOST|VF[IV]?|TRUEFRENCH|FRENCH|MULTI|SUBFR?)\b", re.IGNORECASE
|
|
||||||
)
|
|
||||||
_TRAILING_SEASON_RE = re.compile(r"[\s\-–—:.]*\b(?:saison|season)\s*\d+\s*$", re.IGNORECASE)
|
|
||||||
_TRAILING_CODE_RE = re.compile(r"[\s\-–—:.]*\bS\d+(?:\s*E\d+)?\s*$", re.IGNORECASE)
|
|
||||||
_TRAILING_EPISODE_RE = re.compile(r"[\s\-–—:.]*\b(?:e|ep|episode)\s*\d+\s*$", re.IGNORECASE)
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_title(title: str) -> str:
|
|
||||||
"""Nettoie un titre de scraping avant comparaison/recherche Kitsu.
|
|
||||||
|
|
||||||
Retire le bruit (VF/VOSTFR…), les marqueurs de saison/épisode en fin de titre
|
|
||||||
(« - Saison 1 - E3 », « S1 E1 ») — appliqués en boucle : un titre canonique
|
|
||||||
émergera identique d'un téléchargement (« Titre - Saison 1 - E3 ») ou d'une
|
|
||||||
fiche Kitsu (« Titre »), condition de l'exclusion du déjà-possédé.
|
|
||||||
"""
|
|
||||||
cleaned = _NOISE_WORDS_RE.sub(" ", title)
|
|
||||||
previous = None
|
|
||||||
while previous != cleaned:
|
|
||||||
previous = cleaned
|
|
||||||
cleaned = _TRAILING_SEASON_RE.sub("", cleaned)
|
|
||||||
cleaned = _TRAILING_CODE_RE.sub("", cleaned)
|
|
||||||
cleaned = _TRAILING_EPISODE_RE.sub("", cleaned)
|
|
||||||
cleaned = re.sub(r"\s*[-–—_]+\s*", " ", cleaned)
|
|
||||||
cleaned = re.sub(r"\s+", " ", cleaned)
|
|
||||||
return cleaned.strip(" -–—:.")
|
|
||||||
|
|
||||||
|
|
||||||
class KitsuService:
|
|
||||||
"""Recherche et cache des métadonnées anime depuis l'API Kitsu."""
|
|
||||||
|
|
||||||
async def search_anime(self, title: str) -> dict | None:
|
|
||||||
"""Recherche le meilleur match Kitsu pour un titre (None si échec/rien trouvé)."""
|
|
||||||
settings = get_settings()
|
|
||||||
query = normalize_title(title)
|
|
||||||
if not query:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(
|
|
||||||
timeout=settings.http_timeout,
|
|
||||||
headers={
|
|
||||||
"User-Agent": settings.user_agent,
|
|
||||||
"Accept": "application/vnd.api+json",
|
|
||||||
},
|
|
||||||
) as client:
|
|
||||||
response = await client.get(
|
|
||||||
f"{settings.kitsu_base_url}/anime",
|
|
||||||
params={"filter[text]": query, "page[limit]": 5, "include": "genres"},
|
|
||||||
)
|
|
||||||
response.raise_for_status()
|
|
||||||
payload = response.json()
|
|
||||||
except (httpx.HTTPError, ValueError) as exc:
|
|
||||||
logger.warning("Recherche Kitsu échouée pour %r : %s", title, exc)
|
|
||||||
return None
|
|
||||||
return self._pick_best_match(payload, query)
|
|
||||||
|
|
||||||
async def enrich(self, details: TitleDetails) -> TitleDetails:
|
|
||||||
"""Complète les champs manquants de details via Kitsu (jamais episodes/url/source)."""
|
|
||||||
query = normalize_title(details.title)
|
|
||||||
cache_key = f"kitsu:anime:{query.casefold()}"
|
|
||||||
|
|
||||||
result = await self.get_cached(cache_key)
|
|
||||||
from_cache = result is not None
|
|
||||||
if result is None:
|
|
||||||
result = await self.search_anime(details.title)
|
|
||||||
if result is not None:
|
|
||||||
await self.set_cached(cache_key, result)
|
|
||||||
|
|
||||||
if result is None:
|
|
||||||
logger.info("Aucune métadonnée Kitsu pour %r — fiche inchangée", details.title)
|
|
||||||
return details
|
|
||||||
|
|
||||||
attrs = result.get("attributes", {})
|
|
||||||
if not details.synopsis:
|
|
||||||
details.synopsis = attrs.get("synopsis")
|
|
||||||
if not details.image_url:
|
|
||||||
poster = attrs.get("posterImage") or {}
|
|
||||||
details.image_url = poster.get("large") or poster.get("medium")
|
|
||||||
if not details.banner_url:
|
|
||||||
cover = attrs.get("coverImage") or {}
|
|
||||||
details.banner_url = cover.get("large") or cover.get("original")
|
|
||||||
if not details.genres:
|
|
||||||
details.genres = list(result.get("genres", []))
|
|
||||||
if details.rating is None:
|
|
||||||
details.rating = self._to_rating_10(attrs.get("averageRating"))
|
|
||||||
if details.year is None:
|
|
||||||
details.year = self._extract_year(attrs.get("startDate"))
|
|
||||||
if details.episode_count is None:
|
|
||||||
details.episode_count = attrs.get("episodeCount")
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
"Fiche %r enrichie via Kitsu (cache=%s, id=%s)",
|
|
||||||
details.title,
|
|
||||||
from_cache,
|
|
||||||
result.get("id"),
|
|
||||||
)
|
|
||||||
return details
|
|
||||||
|
|
||||||
async def get_cached(self, cache_key: str) -> dict | None:
|
|
||||||
"""Retourne le payload en cache s'il existe et est frais (TTL), sinon None."""
|
|
||||||
ttl_hours = get_settings().metadata_cache_ttl_hours
|
|
||||||
try:
|
|
||||||
row = await db.fetchone(
|
|
||||||
"SELECT payload FROM metadata_cache "
|
|
||||||
f"WHERE cache_key = ? AND fetched_at > datetime('now', '-{ttl_hours} hours')",
|
|
||||||
(cache_key,),
|
|
||||||
)
|
|
||||||
except (RuntimeError, sqlite3.Error) as exc:
|
|
||||||
logger.warning(
|
|
||||||
"Cache métadonnées illisible (%s), continuation sans cache : %s", cache_key, exc
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
if row is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return json.loads(row["payload"])
|
|
||||||
except (TypeError, ValueError) as exc:
|
|
||||||
logger.warning("Cache métadonnées corrompu pour %s, re-fetch : %s", cache_key, exc)
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def set_cached(self, cache_key: str, payload: dict) -> None:
|
|
||||||
"""Écrit/upsert une entrée de cache (échec non bloquant, warning seulement)."""
|
|
||||||
try:
|
|
||||||
await db.execute(
|
|
||||||
"INSERT INTO metadata_cache (cache_key, payload, fetched_at) "
|
|
||||||
"VALUES (?, ?, datetime('now')) "
|
|
||||||
"ON CONFLICT(cache_key) DO UPDATE SET "
|
|
||||||
"payload = excluded.payload, fetched_at = excluded.fetched_at",
|
|
||||||
(cache_key, json.dumps(payload, ensure_ascii=False)),
|
|
||||||
)
|
|
||||||
except (RuntimeError, sqlite3.Error) as exc:
|
|
||||||
logger.warning(
|
|
||||||
"Cache métadonnées non écrit (%s), continuation sans cache : %s", cache_key, exc
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _pick_best_match(payload: dict, query: str) -> dict | None:
|
|
||||||
items = [item for item in payload.get("data", []) if item.get("type") == "anime"]
|
|
||||||
if not items:
|
|
||||||
return None
|
|
||||||
|
|
||||||
genre_names = {
|
|
||||||
obj.get("id"): obj.get("attributes", {}).get("name")
|
|
||||||
for obj in payload.get("included", [])
|
|
||||||
if obj.get("type") == "genres"
|
|
||||||
}
|
|
||||||
|
|
||||||
def titles_of(item: dict) -> set[str]:
|
|
||||||
attrs = item.get("attributes", {})
|
|
||||||
titles = attrs.get("titles") or {}
|
|
||||||
return {
|
|
||||||
t.strip().casefold()
|
|
||||||
for t in (attrs.get("canonicalTitle"), titles.get("en"), titles.get("en_jp"))
|
|
||||||
if t
|
|
||||||
}
|
|
||||||
|
|
||||||
def popularity(item: dict) -> tuple[int, float]:
|
|
||||||
attrs = item.get("attributes", {})
|
|
||||||
return (
|
|
||||||
attrs.get("userCount") or 0,
|
|
||||||
float(attrs.get("averageRating") or 0),
|
|
||||||
)
|
|
||||||
|
|
||||||
best = next((item for item in items if query.casefold() in titles_of(item)), None)
|
|
||||||
if best is None:
|
|
||||||
best = max(items, key=popularity)
|
|
||||||
|
|
||||||
genre_ids = best.get("relationships", {}).get("genres", {}).get("data") or []
|
|
||||||
genres = [genre_names[g["id"]] for g in genre_ids if g.get("id") in genre_names]
|
|
||||||
return {"id": best.get("id"), "attributes": best.get("attributes", {}), "genres": genres}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _to_rating_10(average_rating: str | None) -> float | None:
|
|
||||||
"""Kitsu note sur 100 (chaîne) → note sur 10 arrondie à 1 décimale."""
|
|
||||||
if average_rating is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return round(float(average_rating) / 10, 1)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _extract_year(start_date: str | None) -> int | None:
|
|
||||||
if not start_date:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return int(str(start_date)[:4])
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
@@ -1,116 +0,0 @@
|
|||||||
"""Paramètres persistés en DB (activation des sources, santé, réglages UI…)."""
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import secrets
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
|
|
||||||
from app.db import db
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
async def get_setting(key: str, default: object = None) -> object:
|
|
||||||
row = await db.fetchone("SELECT value FROM settings WHERE key = ?", (key,))
|
|
||||||
if row is None:
|
|
||||||
return default
|
|
||||||
try:
|
|
||||||
return json.loads(row["value"])
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
logger.warning("Paramètre %r corrompu, valeur par défaut utilisée", key)
|
|
||||||
return default
|
|
||||||
|
|
||||||
|
|
||||||
async def set_setting(key: str, value: object) -> None:
|
|
||||||
await db.execute(
|
|
||||||
"INSERT INTO settings (key, value) VALUES (?, ?) "
|
|
||||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
|
||||||
(key, json.dumps(value, ensure_ascii=False)),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def is_source_enabled(name: str) -> bool:
|
|
||||||
return bool(await get_setting(f"source:{name}:enabled", True))
|
|
||||||
|
|
||||||
|
|
||||||
async def set_source_enabled(name: str, enabled: bool) -> None:
|
|
||||||
await set_setting(f"source:{name}:enabled", enabled)
|
|
||||||
logger.info("Source %s %s", name, "activée" if enabled else "désactivée")
|
|
||||||
|
|
||||||
|
|
||||||
async def get_source_health(name: str) -> dict | None:
|
|
||||||
"""Dernier état de santé connu d'une source (None si jamais testée)."""
|
|
||||||
value = await get_setting(f"source:{name}:health")
|
|
||||||
return value if isinstance(value, dict) else None
|
|
||||||
|
|
||||||
|
|
||||||
async def set_source_health(name: str, healthy: bool, detail: str) -> dict:
|
|
||||||
state = {
|
|
||||||
"healthy": healthy,
|
|
||||||
"detail": detail,
|
|
||||||
"checked_at": datetime.now(UTC).isoformat(),
|
|
||||||
}
|
|
||||||
await set_setting(f"source:{name}:health", state)
|
|
||||||
return state
|
|
||||||
|
|
||||||
|
|
||||||
async def get_source_base_url(name: str) -> str | None:
|
|
||||||
"""URL personnalisée d'une source (None = valeur par défaut du code)."""
|
|
||||||
value = await get_setting(f"source:{name}:base_url")
|
|
||||||
return value if isinstance(value, str) and value else None
|
|
||||||
|
|
||||||
|
|
||||||
async def set_source_base_url(name: str, url: str | None) -> None:
|
|
||||||
"""Persiste l'URL personnalisée (None/'' → retour à la valeur par défaut)."""
|
|
||||||
key = f"source:{name}:base_url"
|
|
||||||
if url:
|
|
||||||
await set_setting(key, url)
|
|
||||||
else:
|
|
||||||
await db.execute("DELETE FROM settings WHERE key = ?", (key,))
|
|
||||||
|
|
||||||
|
|
||||||
async def apply_source_base_urls() -> None:
|
|
||||||
"""Applique les URL personnalisées aux instances de sources (au démarrage)."""
|
|
||||||
from app.scrapers.base import all_sources
|
|
||||||
|
|
||||||
for source in all_sources():
|
|
||||||
override = await get_source_base_url(source.name)
|
|
||||||
if override and override != type(source).base_url:
|
|
||||||
source.base_url = override
|
|
||||||
logger.info("URL personnalisée pour %s : %s", source.name, override)
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- intégrations *arr
|
|
||||||
|
|
||||||
TORZNAB_APIKEY_KEY = "torznab:apikey"
|
|
||||||
SONARR_URL_KEY = "sonarr:url"
|
|
||||||
SONARR_APIKEY_KEY = "sonarr:apikey"
|
|
||||||
|
|
||||||
|
|
||||||
async def get_torznab_apikey() -> str:
|
|
||||||
"""Clé API Torznab (générée au premier appel, persistée en DB)."""
|
|
||||||
key = await get_setting(TORZNAB_APIKEY_KEY)
|
|
||||||
if not isinstance(key, str) or not key:
|
|
||||||
key = secrets.token_hex(16)
|
|
||||||
await set_setting(TORZNAB_APIKEY_KEY, key)
|
|
||||||
logger.info("Clé API Torznab générée")
|
|
||||||
return key
|
|
||||||
|
|
||||||
|
|
||||||
async def reset_torznab_apikey() -> str:
|
|
||||||
key = secrets.token_hex(16)
|
|
||||||
await set_setting(TORZNAB_APIKEY_KEY, key)
|
|
||||||
logger.info("Clé API Torznab régénérée")
|
|
||||||
return key
|
|
||||||
|
|
||||||
|
|
||||||
async def get_sonarr_config() -> dict[str, str]:
|
|
||||||
return {
|
|
||||||
"url": await get_setting(SONARR_URL_KEY, ""),
|
|
||||||
"apikey": await get_setting(SONARR_APIKEY_KEY, ""),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def set_sonarr_config(url: str, apikey: str) -> None:
|
|
||||||
await set_setting(SONARR_URL_KEY, url.rstrip("/"))
|
|
||||||
await set_setting(SONARR_APIKEY_KEY, apikey.strip())
|
|
||||||
logger.info("Configuration Sonarr enregistrée (%s)", url)
|
|
||||||
|
|
||||||
@@ -1,153 +0,0 @@
|
|||||||
"""Client Sonarr (API v3) — personnalisation de « Pour toi ».
|
|
||||||
|
|
||||||
OhmStreaming lit ce qui est téléchargé/grabé sur Sonarr :
|
|
||||||
- `/api/v3/history` (événements grab/import) → titres récemment obtenus ;
|
|
||||||
- `/api/v3/series` → genres de chaque série (fournis par Sonarr lui-même).
|
|
||||||
|
|
||||||
Ces données alimentent le profil de genres des recommandations et la liste
|
|
||||||
d'exclusion (le déjà-possédé sur Sonarr n'est pas re-recommandé).
|
|
||||||
|
|
||||||
Toujours tolérant aux pannes : Sonarr absent/non configuré → profil vide,
|
|
||||||
la découverte continue de fonctionner avec l'historique local.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import time
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.services.kitsu import normalize_title
|
|
||||||
from app.services.settings import get_sonarr_config
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_HISTORY_PAGE_SIZE = 100
|
|
||||||
_MAX_TITLES = 30 # titres Sonarr récents analysés
|
|
||||||
_PROFILE_TTL = 900.0 # profil Sonarr re-quantifié au bout de 15 min
|
|
||||||
_REQUEST_TIMEOUT = 15.0
|
|
||||||
|
|
||||||
|
|
||||||
class SonarrService:
|
|
||||||
"""Profil de consommation Sonarr : titres téléchargés + genres associés."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self._cache: tuple[float, tuple[set[str], dict[str, int]]] | None = None
|
|
||||||
|
|
||||||
async def _config(self) -> tuple[str, str]:
|
|
||||||
"""(url, apikey) — lu en DB à chaque usage (lecture SQLite négligeable)."""
|
|
||||||
config = await get_sonarr_config()
|
|
||||||
return config["url"], config["apikey"]
|
|
||||||
|
|
||||||
def invalidate(self) -> None:
|
|
||||||
"""Force le recalcul du profil au prochain appel (réglages modifiés)."""
|
|
||||||
self._cache = None
|
|
||||||
|
|
||||||
async def _client(self) -> httpx.AsyncClient | None:
|
|
||||||
url, apikey = await self._config()
|
|
||||||
if not url or not apikey:
|
|
||||||
return None
|
|
||||||
return httpx.AsyncClient(
|
|
||||||
base_url=url,
|
|
||||||
timeout=_REQUEST_TIMEOUT,
|
|
||||||
headers={"X-Api-Key": apikey},
|
|
||||||
verify=False, # reverse proxy swizzin : certificat auto-signé (LAN uniquement)
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _get(self, path: str, params: dict | None = None) -> dict | list | None:
|
|
||||||
client = await self._client()
|
|
||||||
if client is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
async with client:
|
|
||||||
response = await client.get(path, params=params)
|
|
||||||
response.raise_for_status()
|
|
||||||
return response.json()
|
|
||||||
except (httpx.HTTPError, ValueError) as exc:
|
|
||||||
logger.warning("Sonarr %s KO : %s", path, exc)
|
|
||||||
return None
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ profil
|
|
||||||
|
|
||||||
async def profile(self) -> tuple[set[str], dict[str, int]]:
|
|
||||||
"""(titres possédés normalisés, comptage de genres) — vide si non configuré."""
|
|
||||||
if self._cache is not None and self._cache[0] > time.monotonic():
|
|
||||||
return self._cache[1]
|
|
||||||
|
|
||||||
titles = await self.downloaded_titles()
|
|
||||||
genres_by_title = await self.series_genres()
|
|
||||||
|
|
||||||
owned: set[str] = set()
|
|
||||||
genre_counts: dict[str, int] = {}
|
|
||||||
for title in titles:
|
|
||||||
normalized = normalize_title(title).casefold()
|
|
||||||
if normalized:
|
|
||||||
owned.add(normalized)
|
|
||||||
for genre in genres_by_title.get(normalized, []):
|
|
||||||
genre_counts[genre] = genre_counts.get(genre, 0) + 1
|
|
||||||
|
|
||||||
result = (owned, genre_counts)
|
|
||||||
self._cache = (time.monotonic() + _PROFILE_TTL, result)
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def downloaded_titles(self) -> list[str]:
|
|
||||||
"""Titres récemment grabés/importés sur Sonarr (les plus récents d'abord)."""
|
|
||||||
payload = await self._get(
|
|
||||||
"/api/v3/history",
|
|
||||||
{
|
|
||||||
"page": 1,
|
|
||||||
"pageSize": _HISTORY_PAGE_SIZE,
|
|
||||||
"eventType": 1, # grab ; les imports (3) suivent le même titre
|
|
||||||
"sortKey": "date",
|
|
||||||
"sortDir": "desc",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if not isinstance(payload, dict):
|
|
||||||
return []
|
|
||||||
titles: list[str] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
for record in payload.get("records", []):
|
|
||||||
title = record.get("series", {}).get("title")
|
|
||||||
if not title or title in seen:
|
|
||||||
continue
|
|
||||||
seen.add(title)
|
|
||||||
titles.append(title)
|
|
||||||
if len(titles) >= _MAX_TITLES:
|
|
||||||
break
|
|
||||||
return titles
|
|
||||||
|
|
||||||
async def series_genres(self) -> dict[str, list[str]]:
|
|
||||||
"""Genres par série, clé = titre normalisé (minuscule)."""
|
|
||||||
payload = await self._get("/api/v3/series")
|
|
||||||
if not isinstance(payload, list):
|
|
||||||
return {}
|
|
||||||
return {
|
|
||||||
normalize_title(s.get("title", "")).casefold(): [
|
|
||||||
g for g in s.get("genres", []) if isinstance(g, str)
|
|
||||||
]
|
|
||||||
for s in payload
|
|
||||||
if s.get("title")
|
|
||||||
}
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ admin
|
|
||||||
|
|
||||||
async def test_connection(self) -> dict:
|
|
||||||
"""Diagnostic admin : version Sonarr, séries, grabs récents."""
|
|
||||||
url, apikey = await self._config()
|
|
||||||
if not url or not apikey:
|
|
||||||
return {"ok": False, "detail": "URL ou clé API manquante"}
|
|
||||||
payload = await self._get("/api/v3/system/status")
|
|
||||||
if payload is None:
|
|
||||||
return {"ok": False, "detail": "Connexion impossible — vérifiez URL et clé"}
|
|
||||||
series = await self._get("/api/v3/series")
|
|
||||||
titles = await self.downloaded_titles()
|
|
||||||
return {
|
|
||||||
"ok": True,
|
|
||||||
"detail": (
|
|
||||||
f"Sonarr {payload.get('version', '?')} — "
|
|
||||||
f"{len(series) if isinstance(series, list) else 0} séries, "
|
|
||||||
f"{len(titles)} saisies récentes"
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
sonarr = SonarrService()
|
|
||||||
@@ -1,383 +0,0 @@
|
|||||||
"""Indexeur Torznab/Newznab : expose le catalogue OhmStreaming à Sonarr/Prowlarr.
|
|
||||||
|
|
||||||
OhmStreaming devient une source d'indexeur à part entière : Sonarr (ou Prowlarr,
|
|
||||||
qui relaiera vers Radarr/Lidarr…) interroge `/torznab/api` comme n'importe quel
|
|
||||||
indexer Jackett. Chaque « release » correspond à un épisode résolu depuis les
|
|
||||||
sources de scraping ; le grab (`/torznab/download`) déclenche l'extraction puis
|
|
||||||
l'ajout dans la file de téléchargements OhmStreaming — le fichier arrive donc
|
|
||||||
dans la bibliothèque locale, comme un téléchargement manuel.
|
|
||||||
|
|
||||||
Formats :
|
|
||||||
- `t=caps` → capacités du serveur (catégories TV/Anime, paramètres supportés)
|
|
||||||
- `t=tvsearch` → recherche par série + saison + épisode (Sonarr)
|
|
||||||
- `t=search` → recherche libre (Prowlarr, recherche manuelle)
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import hashlib
|
|
||||||
import logging
|
|
||||||
import time
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
from email.utils import format_datetime
|
|
||||||
from urllib.parse import urlencode
|
|
||||||
|
|
||||||
from app.scrapers.base import (
|
|
||||||
Episode,
|
|
||||||
ScrapeError,
|
|
||||||
SourceScraper,
|
|
||||||
VideoLink,
|
|
||||||
all_sources,
|
|
||||||
import_all_scrapers,
|
|
||||||
resolve_hoster,
|
|
||||||
)
|
|
||||||
from app.services.downloads import download_manager
|
|
||||||
from app.services.settings import is_source_enabled
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
import_all_scrapers()
|
|
||||||
|
|
||||||
TORZNAB_NS = "http://torznab.com/schemas/2015/feed"
|
|
||||||
CAT_TV = "5000"
|
|
||||||
CAT_ANIME = "5070"
|
|
||||||
_SIZE_ESTIMATE = 400 * 1024 * 1024 # estimation affichée (~400 Mo/épisode)
|
|
||||||
_MAX_SERIES_PER_SOURCE = 2 # fiches explorées par source lors d'une recherche
|
|
||||||
_MAX_RELEASES = 100 # borne du flux retourné
|
|
||||||
_SEARCH_TIMEOUT = 40.0 # scraping lent : garde-fou par source
|
|
||||||
_EPISODE_CACHE_TTL = 600.0 # listes d'épisodes re-scrapées au bout de 10 min
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Release:
|
|
||||||
"""Un épisode vu comme une release par Sonarr/Prowlarr."""
|
|
||||||
|
|
||||||
series: str
|
|
||||||
season: int
|
|
||||||
ep: int
|
|
||||||
source: str
|
|
||||||
source_id: str
|
|
||||||
episode_url: str
|
|
||||||
|
|
||||||
@property
|
|
||||||
def sonarr_title(self) -> str:
|
|
||||||
return f"{self.series} S{self.season:02d}E{self.ep:02d} VOSTFR WEB-DL"
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def _bencode(value) -> bytes:
|
|
||||||
if isinstance(value, int):
|
|
||||||
return f"i{value}e".encode()
|
|
||||||
if isinstance(value, str):
|
|
||||||
raw = value.encode()
|
|
||||||
return f"{len(raw)}:".encode() + raw
|
|
||||||
if isinstance(value, bytes):
|
|
||||||
return f"{len(value)}:".encode() + value
|
|
||||||
if isinstance(value, dict):
|
|
||||||
return b"d" + b"".join(
|
|
||||||
_bencode(k) + _bencode(v) for k, v in sorted(value.items())
|
|
||||||
) + b"e"
|
|
||||||
if isinstance(value, list):
|
|
||||||
return b"l" + b"".join(_bencode(v) for v in value) + b"e"
|
|
||||||
raise TypeError(f"type non encodable en bencode : {type(value)!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def torrent_stub(announce_url: str, name: str) -> bytes:
|
|
||||||
"""Fichier .torrent minimal (le vrai téléchargement est fait par OhmStreaming)."""
|
|
||||||
return build_stub(announce_url, name)[0]
|
|
||||||
|
|
||||||
|
|
||||||
def build_stub(announce_url: str, name: str) -> tuple[bytes, str]:
|
|
||||||
"""Fichier .torrent de service + infohash SHA-1 (identité côté Sonarr).
|
|
||||||
|
|
||||||
L'announce embarque les paramètres du grab (source, sid, season, ep, series) :
|
|
||||||
quand Sonarr renvoie ce .torrent à l'API compatible qBittorrent d'Ohm,
|
|
||||||
le grab est rejoué à l'identique.
|
|
||||||
"""
|
|
||||||
info = {"name": name + ".mp4", "length": 0, "piece length": 32768, "pieces": b"\x00" * 20}
|
|
||||||
data = _bencode(
|
|
||||||
{
|
|
||||||
"announce": announce_url,
|
|
||||||
"created by": "OhmStreaming",
|
|
||||||
"comment": name,
|
|
||||||
"info": info,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return data, hashlib.sha1(_bencode(info)).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def bdecode(data: bytes):
|
|
||||||
"""Décode un flux bencode (les clés dict reviennent en bytes)."""
|
|
||||||
|
|
||||||
def _parse(offset: int) -> tuple[object, int]:
|
|
||||||
char = data[offset : offset + 1]
|
|
||||||
if char == b"i":
|
|
||||||
end = data.index(b"e", offset)
|
|
||||||
return int(data[offset + 1 : end]), end + 1
|
|
||||||
if char in (b"d", b"l"):
|
|
||||||
is_dict = char == b"d"
|
|
||||||
items: dict | list = {} if is_dict else []
|
|
||||||
offset += 1
|
|
||||||
while data[offset : offset + 1] != b"e":
|
|
||||||
first, offset = _parse(offset)
|
|
||||||
if is_dict:
|
|
||||||
second, offset = _parse(offset)
|
|
||||||
items[first] = second
|
|
||||||
else:
|
|
||||||
items.append(first)
|
|
||||||
return items, offset + 1
|
|
||||||
if char.isdigit():
|
|
||||||
colon = data.index(b":", offset)
|
|
||||||
length = int(data[offset:colon])
|
|
||||||
start = colon + 1
|
|
||||||
return data[start : start + length], start + length
|
|
||||||
raise ValueError(f"bencode invalide à l'octet {offset}")
|
|
||||||
|
|
||||||
value, end = _parse(0)
|
|
||||||
if end != len(data):
|
|
||||||
raise ValueError("données après la fin du flux bencode")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
class TorznabService:
|
|
||||||
"""Recherche multi-sources mappée en releases Torznab + grab → file interne."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self._episodes_cache: dict[tuple[str, str], tuple[float, list[Episode]]] = {}
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ recherche
|
|
||||||
|
|
||||||
async def tvsearch(self, q: str, season: int | None = None, ep: int | None = None) -> list[Release]:
|
|
||||||
"""Recherche type Sonarr : série (+ saison/épisode optionnels)."""
|
|
||||||
releases: list[Release] = []
|
|
||||||
for source in await self._enabled_sources():
|
|
||||||
outcomes = await self._search_source(source, q)
|
|
||||||
for result in outcomes[:_MAX_SERIES_PER_SOURCE]:
|
|
||||||
try:
|
|
||||||
episodes = await self._episodes_of(source, result.source_id)
|
|
||||||
except (ScrapeError, TimeoutError):
|
|
||||||
continue
|
|
||||||
releases.extend(
|
|
||||||
self._releases_for(result.title, source, result.source_id, episodes, season, ep)
|
|
||||||
)
|
|
||||||
if len(releases) >= _MAX_RELEASES:
|
|
||||||
return releases[:_MAX_RELEASES]
|
|
||||||
return releases
|
|
||||||
|
|
||||||
async def search(self, q: str) -> list[Release]:
|
|
||||||
"""Recherche libre : tous les épisodes des fiches trouvées."""
|
|
||||||
return await self.tvsearch(q)
|
|
||||||
|
|
||||||
async def latest_releases(self) -> list[Release]:
|
|
||||||
"""Flux RSS (Sonarr) : dernier épisode de chaque nouveauté du catalogue."""
|
|
||||||
releases: list[Release] = []
|
|
||||||
for source in await self._enabled_sources():
|
|
||||||
try:
|
|
||||||
latest = await asyncio.wait_for(source.latest(), timeout=_SEARCH_TIMEOUT)
|
|
||||||
except (ScrapeError, TimeoutError) as exc:
|
|
||||||
logger.warning("Torznab : nouveautés %s KO : %s", source.name, exc)
|
|
||||||
continue
|
|
||||||
for result in latest[:_MAX_SERIES_PER_SOURCE]:
|
|
||||||
try:
|
|
||||||
episodes = await self._episodes_of(source, result.source_id)
|
|
||||||
except (ScrapeError, TimeoutError):
|
|
||||||
continue
|
|
||||||
whole = [e for e in episodes if e.number == int(e.number)]
|
|
||||||
if not whole:
|
|
||||||
continue
|
|
||||||
newest = max(whole, key=lambda e: (e.season, e.number))
|
|
||||||
releases.extend(
|
|
||||||
self._releases_for(result.title, source, result.source_id, [newest], None, None)
|
|
||||||
)
|
|
||||||
if len(releases) >= _MAX_RELEASES:
|
|
||||||
return releases[:_MAX_RELEASES]
|
|
||||||
return releases
|
|
||||||
|
|
||||||
async def _search_source(self, source: SourceScraper, q: str):
|
|
||||||
try:
|
|
||||||
return await asyncio.wait_for(source.search(q), timeout=_SEARCH_TIMEOUT)
|
|
||||||
except (ScrapeError, TimeoutError) as exc:
|
|
||||||
logger.warning("Torznab : recherche %s KO pour %r : %s", source.name, q, exc)
|
|
||||||
return []
|
|
||||||
|
|
||||||
async def _enabled_sources(self) -> list[SourceScraper]:
|
|
||||||
return [s for s in all_sources() if await is_source_enabled(s.name)]
|
|
||||||
|
|
||||||
async def _episodes_of(self, source: SourceScraper, source_id: str) -> list[Episode]:
|
|
||||||
"""Liste d'épisodes d'une fiche, avec cache mémoire TTL."""
|
|
||||||
key = (source.name, source_id)
|
|
||||||
cached = self._episodes_cache.get(key)
|
|
||||||
if cached is not None and cached[0] > time.monotonic():
|
|
||||||
return cached[1]
|
|
||||||
episodes = await asyncio.wait_for(source.list_episodes(source_id), timeout=_SEARCH_TIMEOUT)
|
|
||||||
self._episodes_cache[key] = (time.monotonic() + _EPISODE_CACHE_TTL, episodes)
|
|
||||||
return episodes
|
|
||||||
|
|
||||||
def _releases_for(
|
|
||||||
self,
|
|
||||||
series: str,
|
|
||||||
source: SourceScraper,
|
|
||||||
source_id: str,
|
|
||||||
episodes: list[Episode],
|
|
||||||
season: int | None,
|
|
||||||
ep: int | None,
|
|
||||||
) -> list[Release]:
|
|
||||||
"""Filtre les épisodes selon saison/épisode demandés (entiers uniquement)."""
|
|
||||||
out = []
|
|
||||||
for item in episodes:
|
|
||||||
if item.number != int(item.number): # OAV 2.5 → ignorée (inparseable Sonarr)
|
|
||||||
continue
|
|
||||||
if season is not None and item.season != season:
|
|
||||||
continue
|
|
||||||
if ep is not None and int(item.number) != ep:
|
|
||||||
continue
|
|
||||||
out.append(
|
|
||||||
Release(
|
|
||||||
series=series,
|
|
||||||
season=item.season,
|
|
||||||
ep=int(item.number),
|
|
||||||
source=source.name,
|
|
||||||
source_id=source_id,
|
|
||||||
episode_url=item.url,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return out
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ grab
|
|
||||||
|
|
||||||
async def grab(
|
|
||||||
self,
|
|
||||||
source: str,
|
|
||||||
source_id: str,
|
|
||||||
season: int,
|
|
||||||
ep: int,
|
|
||||||
series: str,
|
|
||||||
sonarr_hash: str | None = None,
|
|
||||||
) -> dict:
|
|
||||||
"""Résout l'épisode (embed → vidéo directe) puis l'ajoute à la file interne.
|
|
||||||
|
|
||||||
Retourne le dict du téléchargement (existant si doublon actif).
|
|
||||||
Lève ScrapeError si introuvable ou qu'aucun hébergeur n'a répondu.
|
|
||||||
sonarr_hash : infohash du .torrent de service — les téléchargements
|
|
||||||
Sonarr sont préfixés « sonarr:<hash>| » pour rester suivis via l'API
|
|
||||||
compatible qBittorrent.
|
|
||||||
"""
|
|
||||||
from app.scrapers.base import get_source
|
|
||||||
|
|
||||||
scraper = get_source(source)
|
|
||||||
episodes = await self._episodes_of(scraper, source_id)
|
|
||||||
match = next(
|
|
||||||
(
|
|
||||||
e
|
|
||||||
for e in episodes
|
|
||||||
if e.season == season and e.number == int(ep)
|
|
||||||
),
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
if match is None:
|
|
||||||
raise ScrapeError(f"Épisode S{season:02d}E{ep:02d} introuvable sur {source}")
|
|
||||||
|
|
||||||
link = await self._resolve_video(scraper, match.url)
|
|
||||||
title = f"{series} S{season:02d}E{ep:02d}"
|
|
||||||
key = f"sonarr:{sonarr_hash}|{link.url}" if sonarr_hash else link.url
|
|
||||||
result = await download_manager.enqueue(link.url, match.url, title, source_key=key)
|
|
||||||
if link.is_hls or link.headers.get("Referer"):
|
|
||||||
result["note"] = "HLS/proxy : OhmStreaming gère le téléchargement via ffmpeg"
|
|
||||||
logger.info("Torznab grab %s → download id=%s", title, result.get("id"))
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def _resolve_video(self, scraper: SourceScraper, episode_url: str) -> VideoLink:
|
|
||||||
"""Chaîne complète : page épisode → embeds → première URL directe valide."""
|
|
||||||
embeds = await asyncio.wait_for(
|
|
||||||
scraper.extract_embed_links(episode_url), timeout=_SEARCH_TIMEOUT
|
|
||||||
)
|
|
||||||
errors: list[str] = []
|
|
||||||
for embed in embeds:
|
|
||||||
extractor = resolve_hoster(embed)
|
|
||||||
if extractor is None:
|
|
||||||
errors.append(f"hébergeur non supporté : {embed}")
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
link = await asyncio.wait_for(extractor.extract(embed), timeout=_SEARCH_TIMEOUT)
|
|
||||||
if link and link.url:
|
|
||||||
return link
|
|
||||||
except (ScrapeError, TimeoutError) as exc:
|
|
||||||
errors.append(str(exc))
|
|
||||||
raise ScrapeError(f"Aucun hébergeur résolu pour {episode_url} ({'; '.join(errors[:3])})")
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ XML
|
|
||||||
|
|
||||||
def download_url(self, base_url: str, apikey: str, release: Release) -> str:
|
|
||||||
query = urlencode(
|
|
||||||
{
|
|
||||||
"apikey": apikey,
|
|
||||||
"source": release.source,
|
|
||||||
"sid": release.source_id,
|
|
||||||
"season": release.season,
|
|
||||||
"ep": release.ep,
|
|
||||||
"series": release.series,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return f"{base_url}/torznab/download?{query}"
|
|
||||||
|
|
||||||
def caps_xml(self, base_url: str) -> str:
|
|
||||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<caps>
|
|
||||||
<server version="1.0" title="OhmStreaming" url="{base_url}"
|
|
||||||
email="ohm@localhost" image="{base_url}/static/img/logo.png"/>
|
|
||||||
<searching>
|
|
||||||
<search available="yes" supportedParams="q"/>
|
|
||||||
<tv-search available="yes" supportedParams="q,season,ep"/>
|
|
||||||
<movie-search available="no" supportedParams=""/>
|
|
||||||
<audio-search available="no" supportedParams=""/>
|
|
||||||
</searching>
|
|
||||||
<categories>
|
|
||||||
<category id="{CAT_TV}" name="TV">
|
|
||||||
<subcat id="{CAT_ANIME}" name="Anime"/>
|
|
||||||
</category>
|
|
||||||
</categories>
|
|
||||||
</caps>"""
|
|
||||||
|
|
||||||
def results_xml(self, base_url: str, apikey: str, releases: list[Release]) -> str:
|
|
||||||
from xml.sax.saxutils import escape
|
|
||||||
|
|
||||||
items = []
|
|
||||||
for release in releases:
|
|
||||||
url = escape(self.download_url(base_url, apikey, release))
|
|
||||||
pubdate = escape(format_datetime(datetime.now(UTC)))
|
|
||||||
items.append(f""" <item>
|
|
||||||
<title>{escape(release.sonarr_title)}</title>
|
|
||||||
<guid isPermaLink="true">{url}</guid>
|
|
||||||
<link>{url}</link>
|
|
||||||
<comments>{escape(release.episode_url)}</comments>
|
|
||||||
<pubDate>{pubdate}</pubDate>
|
|
||||||
<category>{CAT_ANIME}</category>
|
|
||||||
<enclosure url="{url}" length="{_SIZE_ESTIMATE}" type="application/x-bittorrent"/>
|
|
||||||
<torznab:attr name="seeders" value="1"/>
|
|
||||||
<torznab:attr name="peers" value="1"/>
|
|
||||||
<torznab:attr name="downloadvolumefactor" value="0"/>
|
|
||||||
<torznab:attr name="uploadvolumefactor" value="0"/>
|
|
||||||
</item>""")
|
|
||||||
body = "\n".join(items)
|
|
||||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<rss version="2.0" xmlns:torznab="{TORZNAB_NS}">
|
|
||||||
<channel>
|
|
||||||
<title>OhmStreaming</title>
|
|
||||||
<link>{escape(base_url)}</link>
|
|
||||||
<description>Indexeur OhmStreaming — animes VOSTFR scrapés en direct</description>
|
|
||||||
<language>fr-FR</language>
|
|
||||||
{body}
|
|
||||||
</channel>
|
|
||||||
</rss>"""
|
|
||||||
|
|
||||||
def error_xml(self, code: int, description: str) -> str:
|
|
||||||
from xml.sax.saxutils import escape
|
|
||||||
|
|
||||||
return (
|
|
||||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
||||||
f'<error code="{code}" description="{escape(description)}"/>'
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
torznab = TorznabService()
|
|
||||||
@@ -1,110 +0,0 @@
|
|||||||
"""Mises à jour logicielles.
|
|
||||||
|
|
||||||
- Détection : dernier tag semver du dépôt Gitea public via son API (lecture anonyme),
|
|
||||||
avec le message du tag comme patchnote.
|
|
||||||
- Application : POST à Watchtower (compagnon docker-compose) qui tire la nouvelle
|
|
||||||
image et recrée le conteneur — quelques secondes d'indisponibilité.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import time
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.config import get_settings
|
|
||||||
from app.version import get_version
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_TAG_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
|
|
||||||
_CACHE_TTL = 300.0 # secondes
|
|
||||||
|
|
||||||
_cache: dict[str, object] = {"checked_at": 0.0, "latest": None, "notes": None}
|
|
||||||
|
|
||||||
|
|
||||||
class UpdateError(Exception):
|
|
||||||
"""Erreur de mise à jour (Gitea injoignable, Watchtower KO)."""
|
|
||||||
|
|
||||||
|
|
||||||
def parse_tag(tag: str) -> tuple[int, int, int] | None:
|
|
||||||
"""'v0.2.1' → (0, 2, 1) ; None si le tag n'est pas un semver strict."""
|
|
||||||
m = _TAG_RE.match(tag.strip())
|
|
||||||
return tuple(int(g) for g in m.groups()) if m else None # type: ignore[return-value]
|
|
||||||
|
|
||||||
|
|
||||||
def is_newer(latest: str, current: str) -> bool:
|
|
||||||
a, b = parse_tag(latest), parse_tag(current)
|
|
||||||
if a is None or b is None:
|
|
||||||
return False
|
|
||||||
return a > b
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_latest_version(*, force: bool = False) -> str | None:
|
|
||||||
"""Dernier tag semver du dépôt + patchnote (cache 5 min). None si erreur."""
|
|
||||||
now = time.monotonic()
|
|
||||||
latest_cache = _cache["latest"]
|
|
||||||
if not force and latest_cache and now - float(_cache["checked_at"]) < _CACHE_TTL:
|
|
||||||
return str(latest_cache) # type: ignore[arg-type]
|
|
||||||
|
|
||||||
settings = get_settings()
|
|
||||||
url = f"{settings.gitea_url}/api/v1/repos/{settings.gitea_repo}/tags?limit=20"
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(timeout=10, follow_redirects=True) as client:
|
|
||||||
resp = await client.get(url)
|
|
||||||
resp.raise_for_status()
|
|
||||||
tags = [
|
|
||||||
(t["name"], (t.get("message") or "").strip())
|
|
||||||
for t in resp.json()
|
|
||||||
if parse_tag(t.get("name", ""))
|
|
||||||
]
|
|
||||||
except (httpx.HTTPError, ValueError, KeyError) as exc:
|
|
||||||
logger.warning("Vérification de mise à jour impossible : %s", exc)
|
|
||||||
return None
|
|
||||||
|
|
||||||
latest, notes = max(tags, key=lambda t: parse_tag(t[0])) if tags else (None, None)
|
|
||||||
_cache.update(checked_at=now, latest=latest, notes=notes or None)
|
|
||||||
if latest and is_newer(latest, get_version()):
|
|
||||||
logger.info("Nouvelle version disponible : %s (courante %s)", latest, get_version())
|
|
||||||
return latest
|
|
||||||
|
|
||||||
|
|
||||||
def invalidate_cache() -> None:
|
|
||||||
_cache.update(checked_at=0.0, latest=None, notes=None)
|
|
||||||
|
|
||||||
|
|
||||||
async def status() -> dict[str, object]:
|
|
||||||
"""État complet : version courante, dernière dispo, patchnote."""
|
|
||||||
current = get_version()
|
|
||||||
latest = await fetch_latest_version()
|
|
||||||
return {
|
|
||||||
"current": current,
|
|
||||||
"latest": latest,
|
|
||||||
"notes": _cache["notes"],
|
|
||||||
"update_available": bool(latest and is_newer(latest, current)),
|
|
||||||
"docker": bool(get_settings().watchtower_url),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def trigger_update() -> dict[str, str]:
|
|
||||||
"""Demande à Watchtower de recréer le conteneur avec la dernière image.
|
|
||||||
|
|
||||||
Le conteneur courant (donc cette requête) disparaît quelques secondes après :
|
|
||||||
la réponse est renvoyée immédiatement, le frontend gère la reconnexion.
|
|
||||||
"""
|
|
||||||
settings = get_settings()
|
|
||||||
if not settings.watchtower_url:
|
|
||||||
raise UpdateError(
|
|
||||||
"Watchtower non configuré — mise à jour disponible uniquement en déploiement Docker"
|
|
||||||
)
|
|
||||||
headers = {}
|
|
||||||
if settings.watchtower_token:
|
|
||||||
headers["Authorization"] = f"Bearer {settings.watchtower_token}"
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(timeout=10) as client:
|
|
||||||
resp = await client.post(f"{settings.watchtower_url}/v1/update", headers=headers)
|
|
||||||
resp.raise_for_status()
|
|
||||||
except httpx.HTTPError as exc:
|
|
||||||
raise UpdateError(f"Watchtower injoignable : {exc}") from exc
|
|
||||||
logger.info("Mise à jour déclenchée via Watchtower (version courante %s)", get_version())
|
|
||||||
return {"status": "started"}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user