windy/dl-organizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor enrichment to tmdb-only and fix review findings

Browse Source
This commit is contained in:
windyboy
2026-02-10 08:29:31 +08:00
parent 53aaeeaedf
commit f0c951ad7f
8 changed files with 187 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vlm/config.py
+9 -8
View File
@@ -30,15 +30,13 @@ class Config:
enrichment_enabled: bool = True
enrichment_incremental: bool = True
enrichment_refresh_mode: str = "manual"
enrichment_providers: list[str] = field(default_factory=lambda: ["tmdb", "douban"])
enrichment_providers: list[str] = field(default_factory=lambda: ["tmdb"])
enrichment_cache_db: Path = field(default_factory=lambda: Path.home() / ".vlm" / "enrichment_cache.db")
enrichment_max_concurrency: int = 6
enrichment_min_match_score: float = 0.75
translation_mode: str = "bidirectional"
translation_fallback_machine: bool = True
tmdb_api_key: Optional[str] = None
douban_api_key: Optional[str] = None
douban_api_endpoint: Optional[str] = None
openai_api_key: Optional[str] = None
reputation_min_votes: int = 50
reputation_low_score_threshold: float = 6.0
@@ -103,7 +101,7 @@ def load_config(path: Path) -> Config:
enrichment_enabled=enrichment.get("enabled", True),
enrichment_incremental=enrichment.get("incremental", True),
enrichment_refresh_mode=enrichment.get("refresh_mode", "manual"),
enrichment_providers=enrichment.get("providers", ["tmdb", "douban"]),
enrichment_providers=enrichment.get("providers", ["tmdb"]),
enrichment_cache_db=Path(
enrichment.get("cache_db", str(Path.home() / ".vlm" / "enrichment_cache.db"))
).expanduser(),
@@ -112,8 +110,6 @@ def load_config(path: Path) -> Config:
translation_mode=translation.get("mode", "bidirectional"),
translation_fallback_machine=translation.get("fallback_machine", True),
tmdb_api_key=api_keys.get("tmdb"),
douban_api_key=api_keys.get("douban"),
douban_api_endpoint=enrichment.get("douban_endpoint"),
openai_api_key=api_keys.get("openai"),
reputation_min_votes=reputation.get("min_votes", 50),
reputation_low_score_threshold=reputation.get("low_score_threshold", 6.0),
@@ -155,10 +151,8 @@ def create_default_config(path: Path) -> Config:
},
"api_keys": {
"tmdb": default_config.tmdb_api_key,
"douban": default_config.douban_api_key,
"openai": default_config.openai_api_key,
},
"douban_endpoint": default_config.douban_api_endpoint,
"reputation": {
"min_votes": default_config.reputation_min_votes,
"low_score_threshold": default_config.reputation_low_score_threshold,
@@ -276,6 +270,13 @@ def validate_config(config: Config) -> list[str]:
errors.append("enrichment_cache_db must be a Path object")
if not isinstance(config.enrichment_providers, list) or not config.enrichment_providers:
errors.append("enrichment_providers must be a non-empty list")
else:
allowed_providers = {"tmdb"}
invalid = [provider for provider in config.enrichment_providers if provider.lower() not in allowed_providers]
if invalid:
errors.append(
f"enrichment_providers contains unsupported providers: {invalid}; supported providers: ['tmdb']"
)
if config.enrichment_max_concurrency < 1:
errors.append("enrichment_max_concurrency must be >= 1")
if not (0.0 <= config.enrichment_min_match_score <= 1.0):
src/vlm/enrichment.py
+21 -16
View File
@@ -12,7 +12,7 @@ from urllib.request import urlopen, Request
from vlm.cache import EnrichmentCache
from vlm.config import Config
from vlm.providers import ProviderResult, TMDBProvider, DoubanProvider
from vlm.providers import ProviderResult, TMDBProvider
from vlm.parser import normalize_title
RefreshMode = str
@@ -156,6 +156,7 @@ def _emit_progress(stats: dict[str, int | list[dict[str, str]]], progress_callba
def _build_providers(config: Config, *, request_timeout: int, retries: int) -> list:
providers = []
unsupported: list[str] = []
for name in config.enrichment_providers:
key = name.lower()
if key == "tmdb":
@@ -167,16 +168,14 @@ def _build_providers(config: Config, *, request_timeout: int, retries: int) -> l
min_interval_seconds=0.25,
)
)
elif key == "douban":
providers.append(
DoubanProvider(
config.douban_api_key,
endpoint=config.douban_api_endpoint,
timeout_seconds=request_timeout,
retries=retries,
min_interval_seconds=0.4,
)
)
else:
unsupported.append(name)
if unsupported:
raise ValueError(
f"Unsupported enrichment providers: {unsupported}. Supported providers: ['tmdb']"
)
return providers
@@ -390,7 +389,7 @@ def _apply_payload(record: dict, payload: dict) -> None:
"provider_metadata",
"display_title",
):
if key in payload and payload[key] is not None:
if key in payload:
record[key] = payload[key]
if "needs_review" in payload:
@@ -422,10 +421,16 @@ def _fallback_title_from_filename(filename: Optional[str]) -> Optional[str]:
def _build_display_title(record: dict, payload: dict, config: Config) -> str:
title_zh = payload.get("title_zh") or record.get("title")
title_en = payload.get("title_en") or record.get("title")
fallback_title = record.get("title")
title_zh = payload.get("title_zh") or fallback_title
title_en = payload.get("title_en") or fallback_title
if title_zh and title_en and title_zh == title_en:
title_en = ""
try:
formatted = config.naming_title_format.format(title_zh=title_zh, title_en=title_en).strip()
formatted = config.naming_title_format.format(
title_zh=title_zh or "",
title_en=title_en or "",
).strip()
except Exception:
formatted = f"{title_zh} {title_en}".strip()
formatted = f"{title_zh or ''} {title_en or ''}".strip()
return " ".join(formatted.split())
src/vlm/providers/__init__.py
-2
View File
@@ -2,11 +2,9 @@
from vlm.providers.base import EnrichmentProvider, ProviderResult
from vlm.providers.tmdb import TMDBProvider
from vlm.providers.douban import DoubanProvider
__all__ = [
"EnrichmentProvider",
"ProviderResult",
"TMDBProvider",
"DoubanProvider",
]
src/vlm/providers/douban.py
-97
View File
@@ -1,97 +0,0 @@
"""Douban provider implementation.
This provider is optional. If no endpoint is configured, it silently degrades.
"""
from __future__ import annotations
import json
import time
from typing import Optional
from urllib.parse import urlencode
from urllib.request import urlopen, Request
from vlm.providers.base import ProviderResult
class DoubanProvider:
"""Fetch reputation data from a configurable Douban-compatible API."""
name = "douban"
def __init__(
self,
api_key: Optional[str],
endpoint: Optional[str] = None,
timeout_seconds: int = 6,
retries: int = 2,
min_interval_seconds: float = 0.4,
) -> None:
self.api_key = api_key
self.endpoint = endpoint
self.timeout_seconds = timeout_seconds
self.retries = retries
self.min_interval_seconds = min_interval_seconds
self._last_request_at = 0.0
def enrich(self, *, title: str, media_type: str, year: Optional[int] = None) -> Optional[ProviderResult]:
if not self.endpoint:
return None
params = {
"q": title,
"type": media_type,
}
if year is not None:
params["year"] = year
if self.api_key:
params["api_key"] = self.api_key
data = self._get_json(self.endpoint, params)
if not data:
return None
items = data.get("items") or data.get("subjects") or []
if not items:
return None
item = items[0]
score = item.get("rating") or item.get("score")
votes = item.get("vote_count") or item.get("ratings_count")
title_zh = item.get("title")
title_en = item.get("original_title")
return ProviderResult(
provider=self.name,
canonical_id=f"douban:{item.get('id', 'unknown')}",
title_zh=title_zh,
title_en=title_en,
translation_source=self.name if title_zh or title_en else None,
reputation_score=float(score) if score is not None else None,
reputation_votes=int(votes) if votes is not None else None,
reputation_source=self.name,
raw_metadata={"id": str(item.get("id", ""))},
)
def _wait_for_rate_limit(self) -> None:
if self.min_interval_seconds <= 0:
return
now = time.monotonic()
elapsed = now - self._last_request_at
if elapsed < self.min_interval_seconds:
time.sleep(self.min_interval_seconds - elapsed)
def _get_json(self, url: str, params: dict) -> Optional[dict]:
full_url = f"{url}?{urlencode(params)}"
request = Request(full_url, headers={"Accept": "application/json"})
for _ in range(max(self.retries + 1, 1)):
self._wait_for_rate_limit()
try:
with urlopen(request, timeout=self.timeout_seconds) as response:
payload = response.read().decode("utf-8")
self._last_request_at = time.monotonic()
return json.loads(payload)
except Exception:
self._last_request_at = time.monotonic()
continue
return None