refactor: Restructure downloaders with clear separation
This commit implements a complete reorganization of the downloader system with a clear distinction between anime streaming sites and video hosting services. ## Structure Changes **New Organization:** - `app/downloaders/anime_sites/` - Anime streaming sites (catalogs + metadata) - `app/downloaders/video_players/` - Video hosting services (file downloads) **Base Classes:** - `BaseAnimeSite` - For anime providers (search, episodes, metadata) - `BaseVideoPlayer` - For video players (download link extraction) **Migrated Downloaders:** Anime Sites (4): - AnimeSama, NekoSama, AnimeUltime, Vostfree Video Players (8): - Doodstream, Sibnet, VidMoly, SendVid, Lpayer, 1fichier, Uptobox, Rapidfile ## Key Improvements 1. **Clear Separation**: Distinct base classes for different use cases 2. **Preserved Functionality**: All existing features maintained - VidMoly: M3U8 support, Playwright, multi-domains, target_filename param - SendVid: target_filename parameter support - All others: No behavioral changes 3. **Better Organization**: - Anime sites: search_anime(), get_episodes(), get_anime_metadata() - Video players: get_download_link(url, target_filename=None) 4. **Fixed Imports**: Updated cross-imports in AnimeSama - from ..video_players.vidmoly import - from ..video_players.sendvid import - from ..video_players.sibnet import - from ..video_players.lpayer import 5. **Updated Tests**: All test imports use new structure 6. **Updated Providers**: Added 4 missing file hosts to providers.py ## Backward Compatibility ✅ Main API unchanged: get_downloader() works identically ✅ All 23 tests passing ✅ Frontend fully functional ✅ No breaking changes for users ## Documentation - RESTRUCTURATION_SUMMARY.md - Technical details - FIX_IMPORT_ERROR.md - Import error resolution - IMPORT_VERIFICATION_REPORT.md - Complete import verification - FRONTEND_VERIFICATION_FINAL.md - Frontend validation Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude <noreply@anthropic.com> Co-Authored-By: Happy <yesreply@happy.engineering>
This commit is contained in:
root
@@ -0,0 +1,249 @@
|
||||
from .base import BaseAnimeSite
|
||||
from bs4 import BeautifulSoup
|
||||
import re
|
||||
from urllib.parse import urljoin
|
||||
|
||||
|
||||
class NekoSamaDownloader(BaseAnimeSite):
|
||||
"""Downloader for neko-sama.fr"""
|
||||
|
||||
BASE_DOMAINS = ["neko-sama.fr", "nekosama.fr", "www.neko-sama.fr"]
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
return any(domain in url.lower() for domain in self.BASE_DOMAINS)
|
||||
|
||||
async def get_download_link(self, url: str) -> tuple[str, str]:
|
||||
"""Extract download link from neko-sama URL"""
|
||||
try:
|
||||
response = await self.client.get(url, follow_redirects=True)
|
||||
soup = BeautifulSoup(response.text, 'lxml')
|
||||
|
||||
# Method 1: Look for iframes with video
|
||||
iframes = soup.find_all('iframe')
|
||||
for iframe in iframes:
|
||||
src = iframe.get('src', '')
|
||||
if src and any(p in src for p in ['video', 'player', 'stream']):
|
||||
if not src.startswith('http'):
|
||||
src = urljoin(str(response.url), src)
|
||||
filename = self._generate_filename(str(response.url))
|
||||
return src, filename
|
||||
|
||||
# Method 2: Look for video tags
|
||||
videos = soup.find_all('video')
|
||||
for video in videos:
|
||||
src = video.get('src') or video.get('data-src')
|
||||
if src:
|
||||
filename = self._generate_filename(str(response.url))
|
||||
return src, filename
|
||||
|
||||
sources = video.find_all('source')
|
||||
for source in sources:
|
||||
src = source.get('src', '')
|
||||
if src:
|
||||
filename = self._generate_filename(str(response.url))
|
||||
return src, filename
|
||||
|
||||
# Method 3: Look in scripts
|
||||
scripts = soup.find_all('script')
|
||||
for script in scripts:
|
||||
if script.string:
|
||||
patterns = [
|
||||
r'(https?://[^"\'>\s]+\.(?:mp4|m3u8)(?:\?[^"\'>\s]*)?)',
|
||||
r'"url":"([^"]+)"',
|
||||
r'"video":"([^"]+)"',
|
||||
]
|
||||
for pattern in patterns:
|
||||
matches = re.findall(pattern, script.string)
|
||||
for match in matches:
|
||||
match = match.replace('\\/', '/')
|
||||
if any(ext in match for ext in ['mp4', 'm3u8']):
|
||||
filename = self._generate_filename(str(response.url))
|
||||
return match, filename
|
||||
|
||||
raise Exception("Could not find video link")
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f"Error extracting NekoSama link: {str(e)}")
|
||||
|
||||
def _generate_filename(self, url: str) -> str:
|
||||
parts = url.split('/')
|
||||
anime_name = "anime"
|
||||
episode = "1"
|
||||
|
||||
for i, part in enumerate(parts):
|
||||
if 'episode' in part.lower():
|
||||
match = re.search(r'episode[-\s]*(\d+)', part, re.I)
|
||||
if match:
|
||||
episode = match.group(1)
|
||||
|
||||
filename = f"{anime_name} - Episode {episode}.mp4"
|
||||
return filename.title()
|
||||
|
||||
async def get_episodes(self, anime_url: str, lang: str = "vostfr") -> list[dict]:
|
||||
try:
|
||||
response = await self.client.get(anime_url)
|
||||
soup = BeautifulSoup(response.text, 'lxml')
|
||||
|
||||
episodes = []
|
||||
episode_links = soup.find_all('a', href=re.compile(r'episode'))
|
||||
|
||||
for link in episode_links:
|
||||
href = link.get('href', '')
|
||||
match = re.search(r'episode[-\s]*(\d+)', href, re.I)
|
||||
if match:
|
||||
episode_num = match.group(1)
|
||||
if not href.startswith('http'):
|
||||
href = urljoin(anime_url, href)
|
||||
|
||||
episodes.append({'episode': episode_num, 'url': href})
|
||||
|
||||
# Deduplicate and sort
|
||||
seen = set()
|
||||
unique_episodes = []
|
||||
for ep in episodes:
|
||||
if ep['episode'] not in seen:
|
||||
seen.add(ep['episode'])
|
||||
unique_episodes.append(ep)
|
||||
|
||||
unique_episodes.sort(key=lambda x: int(x['episode']))
|
||||
return unique_episodes
|
||||
|
||||
except Exception as e:
|
||||
return []
|
||||
|
||||
async def get_anime_metadata(self, anime_url: str) -> dict:
|
||||
"""
|
||||
Extract rich metadata from anime page
|
||||
Returns synopsis, genres, rating, release year, studio, etc.
|
||||
"""
|
||||
try:
|
||||
print(f"[NEKO-SAMA] Extracting metadata from: {anime_url}")
|
||||
response = await self.client.get(anime_url)
|
||||
soup = BeautifulSoup(response.text, 'lxml')
|
||||
|
||||
metadata = {
|
||||
'synopsis': None,
|
||||
'genres': [],
|
||||
'rating': None,
|
||||
'release_year': None,
|
||||
'studio': None,
|
||||
'poster_image': None,
|
||||
'banner_image': None,
|
||||
'total_episodes': None,
|
||||
'status': None,
|
||||
'alternative_titles': []
|
||||
}
|
||||
|
||||
# Extract synopsis
|
||||
synopsis_selectors = [
|
||||
'div.synopsis',
|
||||
'div.description',
|
||||
'div[class*="synopsis"]',
|
||||
'div[class*="desc"]',
|
||||
'p.synopsis',
|
||||
'.anime-synopsis',
|
||||
'.summary'
|
||||
]
|
||||
|
||||
for selector in synopsis_selectors:
|
||||
synopsis_elem = soup.select_one(selector)
|
||||
if synopsis_elem:
|
||||
synopsis = synopsis_elem.get_text(strip=True)
|
||||
if len(synopsis) > 50:
|
||||
metadata['synopsis'] = synopsis
|
||||
break
|
||||
|
||||
# Extract genres
|
||||
genre_links = soup.find_all('a', href=re.compile(r'genre|tag|type', re.I))
|
||||
if genre_links:
|
||||
metadata['genres'] = [link.get_text(strip=True) for link in genre_links[:5]]
|
||||
|
||||
# Extract rating
|
||||
rating_selectors = [
|
||||
'span.rating',
|
||||
'div.rating',
|
||||
'span.score',
|
||||
'div[class*="rating"]',
|
||||
'div[class*="score"]'
|
||||
]
|
||||
|
||||
for selector in rating_selectors:
|
||||
rating_elem = soup.select_one(selector)
|
||||
if rating_elem:
|
||||
rating_text = rating_elem.get_text(strip=True)
|
||||
rating_match = re.search(r'(\d+\.?\d*)\s*/\s*10', rating_text)
|
||||
if rating_match:
|
||||
metadata['rating'] = f"{rating_match.group(1)}/10"
|
||||
break
|
||||
|
||||
# Extract release year
|
||||
page_text = soup.get_text()
|
||||
year_matches = re.findall(r'\b(19\d{2}|20\d{2})\b', page_text)
|
||||
if year_matches:
|
||||
import datetime
|
||||
current_year = datetime.datetime.now().year + 2
|
||||
valid_years = [int(y) for y in year_matches if 1950 <= int(y) <= current_year]
|
||||
if valid_years:
|
||||
from collections import Counter
|
||||
metadata['release_year'] = Counter(valid_years).most_common(1)[0][0]
|
||||
|
||||
# Extract poster image
|
||||
poster_elem = soup.select_one('img.poster, img.cover, .anime-poster img')
|
||||
if poster_elem:
|
||||
metadata['poster_image'] = poster_elem.get('src') or poster_elem.get('data-src')
|
||||
|
||||
# Extract total episodes
|
||||
episodes_count = len(await self.get_episodes(anime_url))
|
||||
if episodes_count > 0:
|
||||
metadata['total_episodes'] = episodes_count
|
||||
|
||||
print(f"[NEKO-SAMA] Extracted metadata: {metadata}")
|
||||
return metadata
|
||||
|
||||
except Exception as e:
|
||||
print(f"[NEKO-SAMA] Error extracting metadata: {e}")
|
||||
return {}
|
||||
|
||||
async def search_anime(self, query: str, lang: str = "vostfr", include_metadata: bool = False) -> list[dict]:
|
||||
"""
|
||||
Search for anime on neko-sama
|
||||
|
||||
Args:
|
||||
query: Search query string
|
||||
lang: Language preference (vostfr, vf)
|
||||
include_metadata: Whether to fetch full metadata for each result (slower)
|
||||
"""
|
||||
try:
|
||||
import time
|
||||
start = time.time()
|
||||
print(f"[NEKO-SAMA] Searching for '{query}' ({lang})...")
|
||||
|
||||
# Neko-Sama URL pattern: https://neko-sama.fr/anime/{anime-name}
|
||||
search_url = f"https://neko-sama.fr/anime/{query.lower().replace(' ', '-')}"
|
||||
|
||||
response = await self.client.get(search_url)
|
||||
|
||||
elapsed = time.time() - start
|
||||
print(f"[NEKO-SAMA] Got response {response.status_code} in {elapsed:.2f}s")
|
||||
|
||||
if response.status_code == 200:
|
||||
print(f"[NEKO-SAMA] Found anime at {str(response.url)}")
|
||||
result = {
|
||||
'title': query,
|
||||
'url': str(response.url),
|
||||
'type': 'direct',
|
||||
'metadata': None
|
||||
}
|
||||
|
||||
if include_metadata:
|
||||
metadata = await self.get_anime_metadata(str(response.url))
|
||||
result['metadata'] = metadata
|
||||
|
||||
return [result]
|
||||
|
||||
print(f"[NEKO-SAMA] No anime found")
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
print(f"[NEKO-SAMA] Error: {str(e)}")
|
||||
return []
|
||||
Reference in New Issue
Block a user