windy/dl-organizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add incremental enrich controls with progress and retry limits

Browse Source
This commit is contained in:
windyboy
2026-02-09 23:55:13 +08:00
parent 259e7506d7
commit 59a3b52fee
15 changed files with 1653 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli_enrich.py
+136
View File
@@ -0,0 +1,136 @@
"""Tests for CLI enrich command."""
import json
from click.testing import CliRunner
from vlm.cli import main
def test_enrich_in_place_updates_identities(tmp_path):
"""`vlm enrich` should update identities file in place by default."""
library_root = tmp_path / "library"
library_root.mkdir(parents=True)
config_file = tmp_path / "config.yaml"
config_file.write_text(
f"""
library_root: {library_root}
categories:
movie: [movie, movies]
series: [series, tv, shows]
anime: [anime]
enrichment:
cache_db: {tmp_path / 'cache.db'}
""".strip()
)
identities_file = tmp_path / "identities.json"
identities_file.write_text(json.dumps({
"metadata": {},
"movies": [
{
"path": "/library/movie/Test.2024.mkv",
"filename": "Test.2024.mkv",
"category": "movie",
"title": "Test",
"year": 2024,
"confidence": 0.9,
"needs_review": False,
}
],
"series": [],
"anime": [],
"other": [],
}, ensure_ascii=False))
runner = CliRunner()
result = runner.invoke(main, ["--config", str(config_file), "enrich", "--input", str(identities_file)])
assert result.exit_code == 0
assert "Enrichment complete!" in result.output
updated = json.loads(identities_file.read_text(encoding="utf-8"))
assert updated["metadata"]["enriched"] is True
def test_enrich_refresh_all_option_runs_successfully(tmp_path):
"""`vlm enrich --refresh-all` should execute successfully."""
library_root = tmp_path / "library"
library_root.mkdir(parents=True)
config_file = tmp_path / "config.yaml"
config_file.write_text(
f"""
library_root: {library_root}
categories:
movie: [movie, movies]
series: [series, tv, shows]
anime: [anime]
enrichment:
cache_db: {tmp_path / 'cache.db'}
""".strip()
)
identities_file = tmp_path / "identities.json"
identities_file.write_text(json.dumps({
"metadata": {},
"movies": [],
"series": [],
"anime": [],
"other": [],
}))
runner = CliRunner()
result = runner.invoke(
main,
["--config", str(config_file), "enrich", "--input", str(identities_file), "--refresh-all"],
)
assert result.exit_code == 0
assert "Enrichment complete!" in result.output
def test_enrich_rejects_conflicting_refresh_flags(tmp_path):
"""Conflicting refresh flags should fail with a clear error."""
library_root = tmp_path / "library"
library_root.mkdir(parents=True)
config_file = tmp_path / "config.yaml"
config_file.write_text(
f"""
library_root: {library_root}
categories:
movie: [movie, movies]
series: [series, tv, shows]
anime: [anime]
enrichment:
cache_db: {tmp_path / 'cache.db'}
""".strip()
)
identities_file = tmp_path / "identities.json"
identities_file.write_text(json.dumps({
"metadata": {},
"movies": [],
"series": [],
"anime": [],
"other": [],
}))
runner = CliRunner()
result = runner.invoke(
main,
[
"--config",
str(config_file),
"enrich",
"--input",
str(identities_file),
"--refresh-all",
"--refresh-changed-only",
],
)
assert result.exit_code == 1
assert "mutually exclusive" in result.output
tests/test_config.py
+2 -2
View File
@@ -40,8 +40,8 @@ class TestConfig:
config = Config(library_root=Path("/test"))
assert config.categories == {
"movie": ["movie"],
"series": ["series"],
"movie": ["movie", "movies"],
"series": ["series", "tv", "shows"],
"anime": ["anime"]
}
tests/test_enrichment.py
+171
View File
@@ -0,0 +1,171 @@
"""Unit tests for enrichment pipeline."""
from vlm.config import Config
from vlm.enrichment import enrich_identities_data
from vlm.providers.base import ProviderResult
class DummyProvider:
name = "dummy"
def __init__(self):
self.calls = 0
def enrich(self, *, title: str, media_type: str, year=None):
self.calls += 1
return ProviderResult(
provider="dummy",
canonical_id=f"dummy:{title}",
title_zh="测试中文名",
title_en="Test English Title",
translation_source="dummy",
reputation_score=8.2,
reputation_votes=100,
reputation_source="dummy",
)
def test_enrich_incremental_cache_hit(tmp_path, monkeypatch):
"""Second enrichment run should hit cache when fingerprint is unchanged."""
config = Config(
library_root=tmp_path,
enrichment_cache_db=tmp_path / "cache.db",
enrichment_providers=["dummy"],
translation_fallback_machine=False,
)
provider = DummyProvider()
monkeypatch.setattr(
"vlm.enrichment._build_providers",
lambda _config, request_timeout, retries: [provider],
)
identities = {
"metadata": {},
"movies": [
{
"path": "/library/movie/The.Matrix.1999.mkv",
"filename": "The.Matrix.1999.mkv",
"category": "movie",
"title": "The Matrix",
"year": 1999,
"confidence": 0.9,
"needs_review": False,
}
],
"series": [],
"anime": [],
"other": [],
}
_, stats_first = enrich_identities_data(identities, config)
assert stats_first["enriched"] == 1
assert stats_first["cache_hits"] == 0
assert stats_first["api_calls"] == 1
assert provider.calls == 1
_, stats_second = enrich_identities_data(identities, config)
assert stats_second["cache_hits"] == 1
assert stats_second["api_calls"] == 0
assert provider.calls == 1
movie = identities["movies"][0]
assert movie["title_zh"] == "测试中文名"
assert movie["title_en"] == "Test English Title"
assert movie["display_title"] == "测试中文名 Test English Title"
def test_enrich_flags_low_reputation_for_review(tmp_path, monkeypatch):
"""Low reputation should set needs_review when vote count is high enough."""
class LowScoreProvider(DummyProvider):
def enrich(self, *, title: str, media_type: str, year=None):
self.calls += 1
return ProviderResult(
provider="dummy",
canonical_id=f"dummy:{title}",
title_zh="低分作品",
title_en="Low Score",
translation_source="dummy",
reputation_score=4.5,
reputation_votes=500,
reputation_source="dummy",
)
config = Config(
library_root=tmp_path,
enrichment_cache_db=tmp_path / "cache.db",
enrichment_providers=["dummy"],
translation_fallback_machine=False,
reputation_low_score_threshold=6.0,
reputation_min_votes=50,
)
provider = LowScoreProvider()
monkeypatch.setattr(
"vlm.enrichment._build_providers",
lambda _config, request_timeout, retries: [provider],
)
identities = {
"metadata": {},
"movies": [
{
"path": "/library/movie/Unknown.2020.mkv",
"filename": "Unknown.2020.mkv",
"category": "movie",
"title": "Unknown",
"year": 2020,
"confidence": 0.9,
"needs_review": False,
}
],
"series": [],
"anime": [],
"other": [],
}
enrich_identities_data(identities, config)
assert identities["movies"][0]["needs_review"] is True
def test_enrich_refresh_all_bypasses_cache(tmp_path, monkeypatch):
"""refresh_all should not use cache and should invoke provider again."""
config = Config(
library_root=tmp_path,
enrichment_cache_db=tmp_path / "cache.db",
enrichment_providers=["dummy"],
translation_fallback_machine=False,
)
provider = DummyProvider()
monkeypatch.setattr(
"vlm.enrichment._build_providers",
lambda _config, request_timeout, retries: [provider],
)
identities = {
"metadata": {},
"movies": [
{
"path": "/library/movie/Test.2020.mkv",
"filename": "Test.2020.mkv",
"category": "movie",
"title": "Test",
"year": 2020,
"confidence": 0.9,
"needs_review": False,
}
],
"series": [],
"anime": [],
"other": [],
}
enrich_identities_data(identities, config, refresh_mode="incremental")
assert provider.calls == 1
_, stats = enrich_identities_data(identities, config, refresh_mode="refresh_all")
assert provider.calls == 2
assert stats["cache_hits"] == 0
tests/test_planner.py
+49
View File
@@ -921,3 +921,52 @@ def test_plan_json_includes_all_required_fields(config, tmp_path):
required_op_fields = ["operation_type", "source_path", "destination_path", "reason", "has_conflict", "conflict_reason"]
for field in required_op_fields:
assert field in operation, f"Missing required operation field: {field}"
def test_movie_rejected_by_review_generates_noop(config):
"""Rejected movie should not generate move/rename operation."""
video_file = VideoFile(
path=Path("/mnt/nas/videos/movie/Movie.2020.mkv"),
filename="Movie.2020.mkv",
size_bytes=1000000,
modified_timestamp=datetime.now(),
category="movie"
)
identity = MovieIdentity(
title="Movie",
year=2020,
confidence=0.9,
needs_review=False,
original_filename="Movie.2020.mkv",
review_status="rejected"
)
plan = generate_plan([(video_file, identity)], config)
assert plan.operations[0].operation_type == "no-op"
assert "rejected" in plan.operations[0].reason.lower()
def test_series_rejected_by_review_generates_noop(config):
"""Rejected series should not generate move/rename operation."""
video_file = VideoFile(
path=Path("/mnt/nas/videos/series/Show.S01E01.mkv"),
filename="Show.S01E01.mkv",
size_bytes=1000000,
modified_timestamp=datetime.now(),
category="series"
)
identity = SeriesIdentity(
title="Show",
season=1,
episodes=[1],
confidence=0.9,
needs_review=False,
original_filename="Show.S01E01.mkv",
review_status="rejected"
)
plan = generate_plan([(video_file, identity)], config)
assert plan.operations[0].operation_type == "no-op"
assert "rejected" in plan.operations[0].reason.lower()