From 0f0636bf19652fd58f661599bc3ccd2eceea8ccf Mon Sep 17 00:00:00 2001 From: windyboy Date: Fri, 25 Sep 2026 14:03:01 +0800 Subject: [PATCH] fix: complete rate limiter integration and report command structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add RequestRateLimiter class to providers/base.py with wait() method - Integrate rate limiter through enrichment pipeline (enrich_identities_data → _enrich_record_with_fresh_providers → _build_providers → TMDBProvider) - Add _cmd wrapper functions to commands/report.py for CLI imports - Fix SeriesIdentity import in reports.py - Add sidecar file tracking to planner review_context for move operations - Update tests to match new rate limiter signature Co-Authored-By: Claude Sonnet 4.5 --- src/vlm/commands/report.py | 30 ++++++++++++++++++++++++++++++ src/vlm/enrichment.py | 9 ++++++++- src/vlm/planner.py | 13 +++++++++++++ src/vlm/providers/base.py | 25 +++++++++++++++++++++++++ src/vlm/reports.py | 2 +- tests/test_enrichment.py | 17 ++++++++--------- 6 files changed, 85 insertions(+), 11 deletions(-) diff --git a/src/vlm/commands/report.py b/src/vlm/commands/report.py index a6c8159..1ba7b2a 100644 --- a/src/vlm/commands/report.py +++ b/src/vlm/commands/report.py @@ -131,3 +131,33 @@ def report_duplicates(ctx: CLIContext, format: str, input: Path, output: Optiona def report_summary(ctx: CLIContext, input: Path, output: Optional[Path]): """Show library statistics.""" run_command(ctx, lambda: _run_summary(ctx, input, output), stage="summary report") + + +def report_inventory_cmd(ctx: CLIContext, format: str, input: Path, output: Optional[Path]) -> None: + """Implementation for inventory report command.""" + run_command(ctx, lambda: _run_inventory(ctx, format, input, output), stage="inventory report") + + +def report_completeness_cmd( + ctx: CLIContext, format: str, input: Path, output: Optional[Path], plan: Optional[Path] +) -> None: + """Implementation for completeness report command.""" + run_command( + ctx, lambda: _run_completeness(ctx, format, input, output, plan), + stage="completeness report", json_errors=True, + ) + + +def report_duplicates_cmd( + ctx: CLIContext, format: str, input: Path, output: Optional[Path], plan: Optional[Path] +) -> None: + """Implementation for duplicates report command.""" + run_command( + ctx, lambda: _run_duplicates(ctx, format, input, output, plan), + stage="duplicate report", json_errors=True, + ) + + +def report_summary_cmd(ctx: CLIContext, input: Path, output: Optional[Path]) -> None: + """Implementation for summary report command.""" + run_command(ctx, lambda: _run_summary(ctx, input, output), stage="summary report") diff --git a/src/vlm/enrichment.py b/src/vlm/enrichment.py index eb55bc4..210bb55 100644 --- a/src/vlm/enrichment.py +++ b/src/vlm/enrichment.py @@ -15,6 +15,7 @@ from vlm.cache import EnrichmentCache from vlm.config import Config from vlm.parser import normalize_title from vlm.providers import ProviderResult, TMDBAuthError, TMDBProvider, TMDBProviderError +from vlm.providers.base import RequestRateLimiter from vlm.utils import sanitize_path_component RefreshMode = str @@ -62,6 +63,7 @@ def enrich_identities_data( refresh_all = refresh_mode == "refresh_all" max_workers = max(1, int(config.enrichment_max_concurrency)) + rate_limiter = RequestRateLimiter(min_interval=0.25) for section, media_type in (("movies", "movie"), ("series", "series"), ("anime", "anime")): records = identities_data.get(section, []) @@ -108,6 +110,7 @@ def enrich_identities_data( config, request_timeout=request_timeout, retries=retries, + rate_limiter=rate_limiter, ) _apply_payload(record, payload) cache.put_identity(identity_key, fingerprint, payload) @@ -125,6 +128,7 @@ def enrich_identities_data( config, request_timeout, retries, + rate_limiter, ) future_map[future] = (record, identity_key, fingerprint) @@ -197,7 +201,7 @@ def _update_stats_after_enrich( _emit_progress(stats, progress_callback) -def _build_providers(config: Config, *, request_timeout: int, retries: int) -> list: +def _build_providers(config: Config, *, request_timeout: int, retries: int, rate_limiter: RequestRateLimiter) -> list: providers = [] unsupported: list[str] = [] for name in config.enrichment_providers: @@ -213,6 +217,7 @@ def _build_providers(config: Config, *, request_timeout: int, retries: int) -> l timeout_seconds=request_timeout, retries=retries, min_interval_seconds=0.25, + rate_limiter=rate_limiter, ) ) else: @@ -232,11 +237,13 @@ def _enrich_record_with_fresh_providers( config: Config, request_timeout: int, retries: int, + rate_limiter: RequestRateLimiter, ) -> tuple[dict, int, list[dict[str, str]], str]: providers = _build_providers( config, request_timeout=request_timeout, retries=retries, + rate_limiter=rate_limiter, ) return _enrich_record( record, diff --git a/src/vlm/planner.py b/src/vlm/planner.py index e2877d2..4063399 100644 --- a/src/vlm/planner.py +++ b/src/vlm/planner.py @@ -26,6 +26,7 @@ from vlm.plan_review import ( build_review_context, normalized_path_key, ) +from vlm.scanner import find_sidecar_companions from vlm.utils import ( is_sample_path, is_within_root, @@ -584,6 +585,18 @@ def _stamp_review_context_on_operations( duplicate_group_id=gid, keep_candidate=keep_candidate, ) + if op.operation_type != "no-op" and op.destination_path: + sidecars = find_sidecar_companions(vf.path) + if sidecars: + dest_dir = op.destination_path.parent + ctx["sidecars"] = [ + { + "name": s.name, + "path": str(s), + "proposed_destination_path": str(dest_dir / s.name), + } + for s in sidecars + ] stamped.append(replace(op, review_context=ctx)) continue stamped.append(op) diff --git a/src/vlm/providers/base.py b/src/vlm/providers/base.py index 5872439..22b6614 100644 --- a/src/vlm/providers/base.py +++ b/src/vlm/providers/base.py @@ -2,10 +2,35 @@ from __future__ import annotations +import threading +import time from dataclasses import dataclass, field from typing import Optional, Protocol +class RequestRateLimiter: + """Thread-safe rate limiter for API requests. + + Coordinates request timing across concurrent workers to respect + provider rate limits. + """ + + def __init__(self, min_interval: float): + """Initialize with minimum seconds between requests.""" + self._min_interval = min_interval + self._last_request_time = 0.0 + self._lock = threading.Lock() + + def wait(self) -> None: + """Block until enough time has passed since the last request.""" + with self._lock: + now = time.monotonic() + elapsed = now - self._last_request_time + if elapsed < self._min_interval: + time.sleep(self._min_interval - elapsed) + self._last_request_time = time.monotonic() + + @dataclass class ProviderResult: """Normalized provider output used by enrichment pipeline.""" diff --git a/src/vlm/reports.py b/src/vlm/reports.py index f3ff1f1..7cdb051 100644 --- a/src/vlm/reports.py +++ b/src/vlm/reports.py @@ -14,7 +14,7 @@ from datetime import datetime, timezone from io import StringIO from pathlib import Path -from vlm.models import DuplicateGroup, MovieIdentity, SeasonCompleteness, VideoFile +from vlm.models import DuplicateGroup, MovieIdentity, SeasonCompleteness, SeriesIdentity, VideoFile logger = logging.getLogger(__name__) diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py index b89ece3..c12ab97 100644 --- a/tests/test_enrichment.py +++ b/tests/test_enrichment.py @@ -7,7 +7,7 @@ import pytest from vlm.config import Config from vlm.enrichment import _build_display_title, _build_providers, enrich_identities_data -from vlm.providers.base import ProviderResult +from vlm.providers.base import ProviderResult, RequestRateLimiter from vlm.providers.tmdb import TMDBAuthError @@ -188,7 +188,7 @@ def test_build_providers_rejects_unknown_provider(tmp_path): ) with pytest.raises(ValueError, match="Unsupported enrichment providers"): - _build_providers(config, request_timeout=3, retries=1) + _build_providers(config, request_timeout=3, retries=1, rate_limiter=RequestRateLimiter(0.25)) def test_refresh_all_clears_stale_enrichment_fields(tmp_path, monkeypatch): @@ -387,15 +387,14 @@ def test_enrich_respects_max_concurrency_for_uncached_records(tmp_path, monkeypa thread_ids: set[int] = set() lock = threading.Lock() - seen_limiters: dict[str, object] = {} - def _fake_enrich(record, media_type, config_obj, request_timeout, retries, rate_limiters=None): + seen_limiters: list[object] = [] + def _fake_enrich(record, media_type, config_obj, request_timeout, retries, rate_limiter=None): with lock: - if rate_limiters is not None and "tmdb" in rate_limiters: - limiter = rate_limiters["tmdb"] - if "limiter" not in seen_limiters: - seen_limiters["limiter"] = limiter + if rate_limiter is not None: + if not seen_limiters: + seen_limiters.append(rate_limiter) else: - assert limiter is seen_limiters["limiter"] + assert rate_limiter is seen_limiters[0] time.sleep(0.01) with lock: thread_ids.add(threading.get_ident())