windy/dl-organizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enhance project structure and add new files for enrichment and analysis

Browse Source 
- Updated AGENTS.md to reflect changes in CLI commands and module organization, including the addition of an enrichment step and new functional modules.
- Introduced analysis.json, identities.json, inventory.csv, and plan.json to support enriched metadata and execution planning.
- Added CODE_IMPROVEMENTS.md to document identified code issues and proposed solutions for future enhancements.
- Updated README.md to include new enrichment features and configuration options.
- Removed unused dependency on ffmpeg-python from pyproject.toml.

These changes improve the overall functionality and maintainability of the Video Library Manager project.
This commit is contained in:
windyboy
2026-02-10 16:56:17 +08:00
parent f0c951ad7f
commit dcd87754cf
39 changed files with 145509 additions and 642 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vlm/enrichment.py
+112 -26
View File
@@ -12,7 +12,7 @@ from urllib.request import urlopen, Request
from vlm.cache import EnrichmentCache
from vlm.config import Config
from vlm.providers import ProviderResult, TMDBProvider
from vlm.providers import ProviderResult, TMDBAuthError, TMDBProvider, TMDBProviderError
from vlm.parser import normalize_title
RefreshMode = str
@@ -59,6 +59,7 @@ def enrich_identities_data(
"failed": 0,
"api_calls": 0,
"failed_items": [],
"skip_reasons": {},
}
refresh_all = refresh_mode == "refresh_all"
@@ -69,6 +70,7 @@ def enrich_identities_data(
title = record.get("title") or _fallback_title_from_filename(record.get("filename"))
if not title:
stats["skipped"] = int(stats["skipped"]) + 1
_increment_skip_reason(stats, "invalid_input")
stats["processed"] = int(stats["processed"]) + 1
_emit_progress(stats, progress_callback)
continue
@@ -92,7 +94,7 @@ def enrich_identities_data(
_emit_progress(stats, progress_callback)
continue
payload, api_calls, failures = _enrich_record(
payload, api_calls, failures, skip_reason = _enrich_record(
record,
media_type,
providers,
@@ -100,26 +102,12 @@ def enrich_identities_data(
request_timeout=request_timeout,
retries=retries,
)
stats["api_calls"] = int(stats["api_calls"]) + api_calls
if failures:
failed_items = stats["failed_items"]
assert isinstance(failed_items, list)
failed_items.extend(failures)
stats["failed"] = int(stats["failed"]) + len(failures)
_apply_payload(record, payload)
cache.put_identity(identity_key, fingerprint, payload)
if payload.get("enriched"):
stats["enriched"] = int(stats["enriched"]) + 1
else:
stats["skipped"] = int(stats["skipped"]) + 1
if record.get("needs_review"):
stats["needs_review"] = int(stats["needs_review"]) + 1
stats["processed"] = int(stats["processed"]) + 1
_emit_progress(stats, progress_callback)
_update_stats_after_enrich(
stats, payload, failures, api_calls, record, progress_callback, skip_reason
)
metadata = identities_data.setdefault("metadata", {})
metadata["enriched"] = True
@@ -154,6 +142,33 @@ def _emit_progress(stats: dict[str, int | list[dict[str, str]]], progress_callba
)
def _update_stats_after_enrich(
stats: dict[str, int | list[dict[str, str]]],
payload: dict,
failures: list[dict[str, str]],
api_calls: int,
record: dict,
progress_callback: Optional[ProgressCallback],
skip_reason: str,
) -> None:
"""Update stats and emit progress after enriching a single record."""
stats["api_calls"] = int(stats["api_calls"]) + api_calls
if failures:
failed_items = stats["failed_items"]
assert isinstance(failed_items, list)
failed_items.extend(failures)
stats["failed"] = int(stats["failed"]) + len(failures)
if payload.get("enriched"):
stats["enriched"] = int(stats["enriched"]) + 1
else:
stats["skipped"] = int(stats["skipped"]) + 1
_increment_skip_reason(stats, skip_reason or "no_match")
if record.get("needs_review"):
stats["needs_review"] = int(stats["needs_review"]) + 1
stats["processed"] = int(stats["processed"]) + 1
_emit_progress(stats, progress_callback)
def _build_providers(config: Config, *, request_timeout: int, retries: int) -> list:
providers = []
unsupported: list[str] = []
@@ -163,6 +178,10 @@ def _build_providers(config: Config, *, request_timeout: int, retries: int) -> l
providers.append(
TMDBProvider(
config.tmdb_api_key,
bearer_token=config.tmdb_bearer_token,
language=config.tmdb_language,
region=config.tmdb_region,
include_adult=config.tmdb_include_adult,
timeout_seconds=request_timeout,
retries=retries,
min_interval_seconds=0.25,
@@ -187,37 +206,57 @@ def _enrich_record(
*,
request_timeout: int,
retries: int,
) -> tuple[dict, int, list[dict[str, str]]]:
) -> tuple[dict, int, list[dict[str, str]], str]:
title = record.get("title")
year = record.get("year") if media_type == "movie" else None
provider_results: list[ProviderResult] = []
failures: list[dict[str, str]] = []
api_calls = 0
configured_provider_count = 0
for provider in providers:
api_calls += 1
if not _provider_is_configured(provider, config):
continue
configured_provider_count += 1
try:
result = provider.enrich(title=title, media_type=media_type, year=year)
except TMDBAuthError as exc:
raise RuntimeError(str(exc)) from exc
except TMDBProviderError as exc:
failures.append(
{
"path": str(record.get("path", "")),
"title": str(title),
"provider": provider.name,
"reason": str(exc),
}
)
api_calls += provider.last_request_count
continue
except Exception as exc:
failures.append(
{
"path": str(record.get("path", "")),
"title": str(title),
"provider": getattr(provider, "name", "unknown"),
"provider": provider.name,
"reason": str(exc),
}
)
api_calls += provider.last_request_count
continue
api_calls += provider.last_request_count
if result:
provider_results.append(result)
merged = _merge_provider_results(provider_results)
# Optional AI fallback for missing translated titles.
if config.translation_fallback_machine:
if config.translation_fallback_machine and config.openai_api_key:
if not merged.get("title_zh"):
api_calls += 1
translated = _translate_with_openai(
title,
target_language="Chinese (Simplified)",
@@ -225,12 +264,12 @@ def _enrich_record(
timeout_seconds=request_timeout,
retries=retries,
)
api_calls += 1
if translated:
merged["title_zh"] = translated
merged["translation_source"] = merged.get("translation_source") or "openai"
if not merged.get("title_en"):
api_calls += 1
translated = _translate_with_openai(
title,
target_language="English",
@@ -238,7 +277,6 @@ def _enrich_record(
timeout_seconds=request_timeout,
retries=retries,
)
api_calls += 1
if translated:
merged["title_en"] = translated
merged["translation_source"] = merged.get("translation_source") or "openai"
@@ -266,7 +304,55 @@ def _enrich_record(
merged["enriched"] = bool(provider_results or merged.get("translation_source"))
merged["display_title"] = _build_display_title(record, merged, config)
return merged, api_calls, failures
skip_reason = _determine_skip_reason(
provider_results=provider_results,
failures=failures,
configured_provider_count=configured_provider_count,
api_calls=api_calls,
)
return merged, api_calls, failures, skip_reason
def _determine_skip_reason(
*,
provider_results: list[ProviderResult],
failures: list[dict[str, str]],
configured_provider_count: int,
api_calls: int,
) -> str:
if provider_results:
return ""
if configured_provider_count == 0 and api_calls == 0:
return "no_key"
if failures:
for failure in failures:
reason = str(failure.get("reason", "")).lower()
if "rate limit" in reason or "(429)" in reason:
return "rate_limited"
if "authentication failed" in reason or "(401/403)" in reason:
return "auth_error"
return "provider_error"
return "no_match"
def _increment_skip_reason(stats: dict[str, int | list[dict[str, str]]], reason: str) -> None:
if not reason:
return
current = stats.get("skip_reasons")
if not isinstance(current, dict):
current = {}
stats["skip_reasons"] = current
current[reason] = int(current.get(reason, 0)) + 1
def _provider_is_configured(provider: object, config: Config) -> bool:
provider_name = provider.name.lower()
if provider_name == "tmdb":
return bool(config.tmdb_bearer_token or config.tmdb_api_key)
return True
def _merge_provider_results(results: list[ProviderResult]) -> dict: