Enhance project structure and add new files for enrichment and analysis
- Updated AGENTS.md to reflect changes in CLI commands and module organization, including the addition of an enrichment step and new functional modules. - Introduced analysis.json, identities.json, inventory.csv, and plan.json to support enriched metadata and execution planning. - Added CODE_IMPROVEMENTS.md to document identified code issues and proposed solutions for future enhancements. - Updated README.md to include new enrichment features and configuration options. - Removed unused dependency on ffmpeg-python from pyproject.toml. These changes improve the overall functionality and maintainability of the Video Library Manager project.
This commit is contained in:
windyboy
@@ -1,10 +1,12 @@
|
||||
"""Provider implementations for enrichment."""
|
||||
|
||||
from vlm.providers.base import EnrichmentProvider, ProviderResult
|
||||
from vlm.providers.tmdb import TMDBProvider
|
||||
from vlm.providers.tmdb import TMDBAuthError, TMDBProvider, TMDBProviderError
|
||||
|
||||
__all__ = [
|
||||
"EnrichmentProvider",
|
||||
"ProviderResult",
|
||||
"TMDBProvider",
|
||||
"TMDBAuthError",
|
||||
"TMDBProviderError",
|
||||
]
|
||||
|
||||
@@ -26,6 +26,7 @@ class EnrichmentProvider(Protocol):
|
||||
"""Protocol for title/score providers."""
|
||||
|
||||
name: str
|
||||
last_request_count: int # API request count for the last enrich() call (reset at start of each call)
|
||||
|
||||
def enrich(self, *, title: str, media_type: str, year: Optional[int] = None) -> Optional[ProviderResult]:
|
||||
"""Return normalized metadata for a single identity."""
|
||||
|
||||
+170
-18
@@ -3,14 +3,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from typing import Optional
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Any, Optional
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen, Request
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from vlm.providers.base import ProviderResult
|
||||
|
||||
|
||||
class TMDBAuthError(RuntimeError):
|
||||
"""Raised when TMDB credentials are invalid."""
|
||||
|
||||
|
||||
class TMDBProviderError(RuntimeError):
|
||||
"""Raised for TMDB errors that should be reported as provider failures."""
|
||||
|
||||
|
||||
class TMDBProvider:
|
||||
"""Fetch translations and reputation data from TMDB."""
|
||||
|
||||
@@ -19,48 +30,66 @@ class TMDBProvider:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: Optional[str],
|
||||
*,
|
||||
bearer_token: Optional[str] = None,
|
||||
language: str = "zh-CN",
|
||||
region: Optional[str] = None,
|
||||
include_adult: bool = False,
|
||||
timeout_seconds: int = 6,
|
||||
retries: int = 2,
|
||||
min_interval_seconds: float = 0.25,
|
||||
backoff_base_seconds: float = 0.5,
|
||||
backoff_max_seconds: float = 4.0,
|
||||
) -> None:
|
||||
self.api_key = api_key
|
||||
self.bearer_token = bearer_token
|
||||
self.language = language
|
||||
self.region = region
|
||||
self.include_adult = include_adult
|
||||
self.base_url = "https://api.themoviedb.org/3"
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.retries = retries
|
||||
self.min_interval_seconds = min_interval_seconds
|
||||
self.backoff_base_seconds = backoff_base_seconds
|
||||
self.backoff_max_seconds = backoff_max_seconds
|
||||
self._last_request_at = 0.0
|
||||
self.last_request_count = 0
|
||||
|
||||
def enrich(self, *, title: str, media_type: str, year: Optional[int] = None) -> Optional[ProviderResult]:
|
||||
if not self.api_key:
|
||||
self.last_request_count = 0
|
||||
if not (self.bearer_token or self.api_key):
|
||||
return None
|
||||
|
||||
search_type = "tv" if media_type in {"series", "anime", "tv"} else "movie"
|
||||
query_params = {
|
||||
"api_key": self.api_key,
|
||||
query_params: dict[str, Any] = {
|
||||
"query": title,
|
||||
"language": self.language,
|
||||
"include_adult": str(self.include_adult).lower(),
|
||||
}
|
||||
if self.region:
|
||||
query_params["region"] = self.region
|
||||
if year and search_type == "movie":
|
||||
query_params["year"] = year
|
||||
|
||||
search_data = self._get_json(f"{self.base_url}/search/{search_type}", query_params)
|
||||
search_data = self._get_json(f"/search/{search_type}", query_params)
|
||||
if not search_data:
|
||||
return None
|
||||
|
||||
results = search_data.get("results", [])
|
||||
if not results:
|
||||
if not isinstance(results, list) or not results:
|
||||
return None
|
||||
|
||||
candidate, match_score = self._pick_best_candidate(results, title, year)
|
||||
if not candidate:
|
||||
return None
|
||||
|
||||
candidate = results[0]
|
||||
tmdb_id = candidate.get("id")
|
||||
if tmdb_id is None:
|
||||
return None
|
||||
|
||||
details = self._get_json(
|
||||
f"{self.base_url}/{search_type}/{tmdb_id}",
|
||||
{"api_key": self.api_key, "language": self.language},
|
||||
f"/{search_type}/{tmdb_id}",
|
||||
{"language": self.language},
|
||||
)
|
||||
if not details:
|
||||
details = candidate
|
||||
@@ -79,10 +108,78 @@ class TMDBProvider:
|
||||
reputation_score=float(vote_average) if vote_average is not None else None,
|
||||
reputation_votes=int(vote_count) if vote_count is not None else None,
|
||||
reputation_source=self.name,
|
||||
match_score=float(candidate.get("popularity", 0.0)) if candidate.get("popularity") is not None else None,
|
||||
match_score=round(match_score, 3),
|
||||
raw_metadata={"media_type": search_type, "id": str(tmdb_id)},
|
||||
)
|
||||
|
||||
def _pick_best_candidate(
|
||||
self,
|
||||
results: list[dict[str, Any]],
|
||||
query_title: str,
|
||||
query_year: Optional[int],
|
||||
) -> tuple[Optional[dict[str, Any]], float]:
|
||||
query_norm = self._normalize_title(query_title)
|
||||
best_candidate: Optional[dict[str, Any]] = None
|
||||
best_score = -1.0
|
||||
|
||||
for result in results:
|
||||
candidates = [
|
||||
result.get("title"),
|
||||
result.get("name"),
|
||||
result.get("original_title"),
|
||||
result.get("original_name"),
|
||||
]
|
||||
title_score = 0.0
|
||||
for candidate_title in candidates:
|
||||
if not isinstance(candidate_title, str) or not candidate_title.strip():
|
||||
continue
|
||||
candidate_norm = self._normalize_title(candidate_title)
|
||||
if not candidate_norm:
|
||||
continue
|
||||
ratio = SequenceMatcher(None, query_norm, candidate_norm).ratio()
|
||||
if ratio > title_score:
|
||||
title_score = ratio
|
||||
|
||||
year_bonus = 0.0
|
||||
if query_year is not None:
|
||||
release = result.get("release_date") or result.get("first_air_date")
|
||||
candidate_year = self._extract_year(release)
|
||||
if candidate_year is None:
|
||||
year_bonus = -0.1
|
||||
else:
|
||||
delta = abs(candidate_year - query_year)
|
||||
if delta == 0:
|
||||
year_bonus = 0.2
|
||||
elif delta == 1:
|
||||
year_bonus = 0.1
|
||||
else:
|
||||
year_bonus = -0.2
|
||||
|
||||
popularity = result.get("popularity")
|
||||
popularity_bonus = 0.0
|
||||
if isinstance(popularity, (int, float)):
|
||||
popularity_bonus = min(float(popularity) / 1000.0, 0.1)
|
||||
|
||||
total_score = title_score + year_bonus + popularity_bonus
|
||||
if total_score > best_score:
|
||||
best_score = total_score
|
||||
best_candidate = result
|
||||
|
||||
return best_candidate, max(best_score, 0.0)
|
||||
|
||||
def _normalize_title(self, text: str) -> str:
|
||||
lowered = text.lower().strip()
|
||||
stripped = re.sub(r"[^\w\s]", " ", lowered)
|
||||
return " ".join(stripped.split())
|
||||
|
||||
def _extract_year(self, date_text: Any) -> Optional[int]:
|
||||
if not isinstance(date_text, str) or len(date_text) < 4:
|
||||
return None
|
||||
try:
|
||||
return int(date_text[:4])
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
def _wait_for_rate_limit(self) -> None:
|
||||
if self.min_interval_seconds <= 0:
|
||||
return
|
||||
@@ -91,18 +188,73 @@ class TMDBProvider:
|
||||
if elapsed < self.min_interval_seconds:
|
||||
time.sleep(self.min_interval_seconds - elapsed)
|
||||
|
||||
def _get_json(self, url: str, params: dict) -> Optional[dict]:
|
||||
full_url = f"{url}?{urlencode(params)}"
|
||||
request = Request(full_url, headers={"Accept": "application/json"})
|
||||
def _sleep_backoff(self, attempt: int, retry_after: Optional[float] = None) -> None:
|
||||
if retry_after is not None and retry_after > 0:
|
||||
time.sleep(min(retry_after, self.backoff_max_seconds))
|
||||
return
|
||||
delay = min(self.backoff_base_seconds * (2 ** attempt), self.backoff_max_seconds)
|
||||
time.sleep(delay)
|
||||
|
||||
for _ in range(max(self.retries + 1, 1)):
|
||||
def _get_json(self, path: str, params: dict[str, Any]) -> Optional[dict]:
|
||||
request_params = dict(params)
|
||||
if not self.bearer_token and self.api_key:
|
||||
request_params["api_key"] = self.api_key
|
||||
|
||||
full_url = f"{self.base_url}{path}?{urlencode(request_params)}"
|
||||
headers = {"Accept": "application/json"}
|
||||
if self.bearer_token:
|
||||
headers["Authorization"] = f"Bearer {self.bearer_token}"
|
||||
|
||||
for attempt in range(max(self.retries + 1, 1)):
|
||||
self._wait_for_rate_limit()
|
||||
self.last_request_count += 1
|
||||
request = Request(full_url, headers=headers)
|
||||
try:
|
||||
with urlopen(request, timeout=self.timeout_seconds) as response:
|
||||
payload = response.read().decode("utf-8")
|
||||
self._last_request_at = time.monotonic()
|
||||
return json.loads(payload)
|
||||
except Exception:
|
||||
parsed = json.loads(payload)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
return None
|
||||
except HTTPError as exc:
|
||||
self._last_request_at = time.monotonic()
|
||||
continue
|
||||
code = exc.code
|
||||
if code in (401, 403):
|
||||
raise TMDBAuthError(
|
||||
"TMDB authentication failed (401/403). "
|
||||
"Configure enrichment.api_keys.tmdb_bearer or enrichment.api_keys.tmdb."
|
||||
) from exc
|
||||
if code == 404:
|
||||
return None
|
||||
if code == 429:
|
||||
if attempt < self.retries:
|
||||
retry_after = None
|
||||
try:
|
||||
retry_after_header = exc.headers.get("Retry-After")
|
||||
retry_after = float(retry_after_header) if retry_after_header else None
|
||||
except Exception:
|
||||
retry_after = None
|
||||
self._sleep_backoff(attempt, retry_after=retry_after)
|
||||
continue
|
||||
raise TMDBProviderError("TMDB rate limit exceeded (429)") from exc
|
||||
if 500 <= code < 600 and attempt < self.retries:
|
||||
self._sleep_backoff(attempt)
|
||||
continue
|
||||
if 500 <= code < 600:
|
||||
raise TMDBProviderError(f"TMDB server error ({code})") from exc
|
||||
raise TMDBProviderError(f"TMDB request failed with HTTP {code}") from exc
|
||||
except URLError as exc:
|
||||
self._last_request_at = time.monotonic()
|
||||
if attempt < self.retries:
|
||||
self._sleep_backoff(attempt)
|
||||
continue
|
||||
raise TMDBProviderError(f"TMDB network error: {exc}") from exc
|
||||
except Exception as exc:
|
||||
self._last_request_at = time.monotonic()
|
||||
if attempt < self.retries:
|
||||
self._sleep_backoff(attempt)
|
||||
continue
|
||||
raise TMDBProviderError(f"TMDB unexpected error: {exc}") from exc
|
||||
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user