diff --git a/CLAUDE.md b/CLAUDE.md index 7a5e84e..9ee2115 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,6 +69,7 @@ VLM follows a read-first, multi-stage pipeline: - `io.py` - Load/save identities JSON/CSV, plan/analysis input helpers - `utils.py` - UTC time, format_size, shared helpers - `analysis.py` - Completeness checking (episode gaps) and duplicate detection +- `duplicate_resolve.py` - Duplicate group resolution (by_quality, by_reputation, first_seen, manual) - `planner.py` - Execution plan generation with conflict detection - `executor.py` - File operations (move/rename/quarantine) with rollback logging - `quarantine.py` - Quarantine management with manifest tracking @@ -116,6 +117,7 @@ Key settings: - `log_level` - logging verbosity - `categories` - mapping of category names to directory name lists - `enrichment` (or `enrich`) - TMDB/api_keys, cache_db, translation, reputation; see README for full schema +- `plan.duplicate_keep` - when using `vlm plan --analysis`: `by_quality` (resolution > source > codec > size), `by_reputation` (default), `first_seen`, or `manual` ### Category Mappings diff --git a/README.md b/README.md index b957dbc..9c3d278 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,11 @@ This creates `plan.json` with: - **Human summary** (中文): short narrative for quick review - **Metadata**: when using `--analysis`, duplicate groups considered and completeness gaps -Duplicate keep strategy is configurable in `~/.vlm/config.yaml` under `plan.duplicate_keep` (`by_reputation`, `first_seen`, or `manual`). Default is `by_reputation` (prefer external rating; fallback to first-seen). +Duplicate keep strategy is configurable in `~/.vlm/config.yaml` under `plan.duplicate_keep`: +- `by_quality` - Prefer highest quality (resolution > source > codec > file size). Best for automatic duplicate resolution. +- `by_reputation` - Prefer external rating (TMDB); fallback to first-seen. Default. +- `first_seen` - Keep the first file in each duplicate group. +- `manual` - Do not generate quarantine operations; duplicates are listed in analysis only. **Review the plan** by opening `plan.json` in your editor, or read the human summary when you run `vlm execute`. You can edit the plan JSON if needed. @@ -406,7 +410,8 @@ log_level: "INFO" # Plan behavior (e.g. when using vlm plan --analysis) plan: - # Duplicate keep strategy: "by_reputation" (default), "first_seen", or "manual" + # Duplicate keep strategy: "by_quality", "by_reputation" (default), "first_seen", or "manual" + # by_quality: resolution > source (BluRay > WEB-DL) > codec (x265 > x264) > file size duplicate_keep: "by_reputation" # Category mappings (directory name to category) @@ -762,7 +767,7 @@ src/vlm/ ├── io.py # JSON/CSV load/save, load_analysis_json, plan/analysis input helpers ├── utils.py # UTC time, format_size, etc. ├── analysis.py # Completeness and duplicate detection -├── duplicate_resolve.py # Duplicate group keep-index (by_reputation, first_seen, manual) +├── duplicate_resolve.py # Duplicate group keep-index (by_quality, by_reputation, first_seen, manual) ├── planner.py # Execution plan generation (optionally consumes analysis) ├── executor.py # File operations and rollback ├── quarantine.py # Quarantine management diff --git a/analysis.json b/analysis.json index 18c587e..285f58d 100644 --- a/analysis.json +++ b/analysis.json @@ -1,6 +1,6 @@ { "metadata": { - "generated": "2026-02-10T09:54:17", + "generated": "2026-02-10T12:54:42", "source_identities": "identities.json", "total_movies": 637, "total_series": 1492 diff --git a/plan.json b/plan.json index dbe2c09..0302187 100644 --- a/plan.json +++ b/plan.json @@ -1,6 +1,6 @@ { - "plan_id": "7e4827a4-3692-4ebe-87c7-b92950c87ea9", - "created_at": "2026-02-10T09:54:19.304610+00:00", + "plan_id": "059bd77a-4f2a-4bc7-860f-dd7fa4a09534", + "created_at": "2026-02-10T12:55:28.326159+00:00", "operations": [ { "operation_type": "no-op", diff --git a/src/vlm/config.py b/src/vlm/config.py index 5a89e82..c5396ce 100644 --- a/src/vlm/config.py +++ b/src/vlm/config.py @@ -324,9 +324,14 @@ def validate_config(config: Config) -> list[str]: errors.append("tmdb_region must be a string when set") if not isinstance(config.tmdb_include_adult, bool): errors.append("tmdb_include_adult must be a boolean") - if config.duplicate_keep not in ("by_reputation", "first_seen", "manual"): + if config.duplicate_keep not in ( + "by_reputation", + "first_seen", + "manual", + "by_quality", + ): errors.append( - f"duplicate_keep must be one of 'by_reputation', 'first_seen', 'manual', got: {config.duplicate_keep!r}" + f"duplicate_keep must be one of 'by_reputation', 'first_seen', 'manual', 'by_quality', got: {config.duplicate_keep!r}" ) return errors diff --git a/src/vlm/duplicate_resolve.py b/src/vlm/duplicate_resolve.py index 1ca8b7b..21720e7 100644 --- a/src/vlm/duplicate_resolve.py +++ b/src/vlm/duplicate_resolve.py @@ -1,7 +1,8 @@ """Duplicate group resolution: choose which file to keep when consuming analysis.""" +import re from pathlib import Path -from typing import Union +from typing import Optional, Union from vlm.models import MovieIdentity, SeriesIdentity @@ -9,10 +10,12 @@ from vlm.models import MovieIdentity, SeriesIdentity def choose_keep_index( items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]], strategy: str, + quality_comparison: Optional[list[dict]] = None, ) -> int | None: """Choose the index of the item to keep in a duplicate group. Strategies: + - by_quality: Prefer higher quality (resolution > source > codec > size). Requires quality_comparison. - by_reputation: Prefer items with reputation_score; then sort by score desc, then reputation_votes desc, then first_seen (input order). Keep index 0 after sort. - first_seen: Keep the first item (index 0). @@ -20,7 +23,9 @@ def choose_keep_index( Args: items: List of (path, identity) for the duplicate group. - strategy: One of "by_reputation", "first_seen", "manual". + strategy: One of "by_quality", "by_reputation", "first_seen", "manual". + quality_comparison: List of quality dicts (filename, path, size_bytes, resolution?, codec?) + aligned with items. Required when strategy is "by_quality". Returns: Index in [0, len(items)) to keep, or None for manual. @@ -29,11 +34,89 @@ def choose_keep_index( return None if strategy == "first_seen": return 0 + if strategy == "by_quality": + if quality_comparison is None or len(quality_comparison) != len(items): + return 0 # Fallback to first if quality data missing/mismatched + return _by_quality_index(items, quality_comparison) if strategy == "by_reputation": return _by_reputation_index(items) return 0 +def _parse_resolution_tier(resolution: Optional[str], path: Path) -> int: + """Parse resolution to tier. Higher = better. 2160=4, 1080=3, 720=2, 0=unknown.""" + if resolution: + m = re.search(r"(\d{3,4})[x×](\d{3,4})", resolution, re.I) + if m: + h = int(m.group(2)) + if h >= 2160: + return 4 + if h >= 1080: + return 3 + if h >= 720: + return 2 + return 1 + s = path.name.upper() + if "2160" in s or "4K" in s or "UHD" in s: + return 4 + if "1080" in s: + return 3 + if "720" in s: + return 2 + if "480" in s or "576" in s: + return 1 + return 0 + + +def _parse_source_tier(path: Path) -> int: + """Parse source type from path. BluRay=3, WEB-DL=2, HDTV=1, 0=other.""" + s = path.name.upper() + if "BLURAY" in s or "BLU-RAY" in s or "BD" in s: + return 3 + if "WEB-DL" in s or "WEBRIP" in s or "WEB" in s: + return 2 + if "HDTV" in s or "HDTVRIP" in s: + return 1 + return 0 + + +def _parse_codec_tier(codec: Optional[str], path: Path) -> int: + """Parse codec to tier. hevc/x265=2, h264/x264=1, 0=other.""" + if codec: + c = codec.lower() + if "hevc" in c or "h265" in c or "x265" in c: + return 2 + if "h264" in c or "x264" in c or "avc" in c: + return 1 + s = path.name.upper() + if "X265" in s or "H265" in s or "HEVC" in s: + return 2 + if "X264" in s or "H264" in s or "AVC" in s: + return 1 + return 0 + + +def _by_quality_index( + items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]], + quality_comparison: list[dict], +) -> int: + """Sort by: resolution desc, source desc, codec desc, size desc, index asc. Return best index.""" + def key(idx_item: tuple[int, tuple[Path, Union[MovieIdentity, SeriesIdentity]]]) -> tuple: + idx, (path, _) = idx_item + qc = quality_comparison[idx] if idx < len(quality_comparison) else {} + resolution = qc.get("resolution") + codec = qc.get("codec") + size = qc.get("size_bytes", 0) or 0 + res_tier = _parse_resolution_tier(resolution, path) + src_tier = _parse_source_tier(path) + codec_tier = _parse_codec_tier(codec, path) + return (-res_tier, -src_tier, -codec_tier, -size, idx) + + indexed = list(enumerate(items)) + indexed.sort(key=key) + return indexed[0][0] + + def _by_reputation_index( items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]], ) -> int: diff --git a/src/vlm/planner.py b/src/vlm/planner.py index 02c8d76..04dfe08 100644 --- a/src/vlm/planner.py +++ b/src/vlm/planner.py @@ -22,6 +22,7 @@ from vlm.models import ( ) QUARANTINE_REASON_DUPLICATE = "重复项（已按外部评分保留一条）" +QUARANTINE_REASON_DUPLICATE_BY_QUALITY = "重复项（已按画质保留最优）" def generate_plan( @@ -50,26 +51,40 @@ def generate_plan( for dup in analysis_data.get("duplicates", []): paths = dup.get("files", []) indices = [path_to_index[p] for p in paths if p in path_to_index] - items = [ - (identities[i][0].path, identities[i][1]) - for i in indices - if identities[i][1] is not None - and isinstance(identities[i][1], (MovieIdentity, SeriesIdentity)) - ] + path_to_qc = {qc.get("path"): qc for qc in dup.get("quality_comparison", [])} + items = [] + valid_indices = [] + for i in indices: + identity = identities[i][1] + if identity is not None and isinstance( + identity, (MovieIdentity, SeriesIdentity) + ): + items.append((identities[i][0].path, identity)) + valid_indices.append(i) if not items: continue - keep_idx = choose_keep_index(items, config.duplicate_keep) + quality_list = [ + path_to_qc.get(str(p), {}) for p, _ in items + ] + keep_idx = choose_keep_index( + items, config.duplicate_keep, quality_comparison=quality_list + ) if keep_idx is None: continue - keep_identity_index = indices[keep_idx] - quarantine_indices = set(indices) - {keep_identity_index} + keep_identity_index = valid_indices[keep_idx] + quarantine_indices = set(valid_indices) - {keep_identity_index} + reason = ( + QUARANTINE_REASON_DUPLICATE_BY_QUALITY + if config.duplicate_keep == "by_quality" + else QUARANTINE_REASON_DUPLICATE + ) for i in quarantine_indices: vf = identities[i][0] operations[i] = FileOperation( operation_type="quarantine", source_path=vf.path, destination_path=None, - reason=QUARANTINE_REASON_DUPLICATE, + reason=reason, has_conflict=False, conflict_reason=None, ) diff --git a/tests/test_duplicate_resolve.py b/tests/test_duplicate_resolve.py new file mode 100644 index 0000000..bd4023b --- /dev/null +++ b/tests/test_duplicate_resolve.py @@ -0,0 +1,119 @@ +"""Unit tests for duplicate resolution strategies.""" + +import pytest +from pathlib import Path + +from vlm.models import MovieIdentity +from vlm.duplicate_resolve import choose_keep_index + + +def _mi(title: str = "Test", year: int | None = 2020) -> MovieIdentity: + return MovieIdentity( + title=title, + year=year, + confidence=0.9, + needs_review=False, + original_filename="test.mkv", + ) + + +class TestByQuality: + """Tests for by_quality strategy.""" + + def test_by_quality_prefers_higher_resolution(self): + """1080p vs 720p -> keep 1080p.""" + p1 = Path("/movies/Test.2020.720p.BluRay.mkv") + p2 = Path("/movies/Test.2020.1080p.BluRay.mkv") + items = [(p1, _mi()), (p2, _mi())] + qc = [ + {"path": str(p1), "resolution": "1280x720", "size_bytes": 1000000}, + {"path": str(p2), "resolution": "1920x1080", "size_bytes": 2000000}, + ] + idx = choose_keep_index(items, "by_quality", quality_comparison=qc) + assert idx == 1 # 1080p + + def test_by_quality_prefers_bluray_over_webdl(self): + """Same resolution: BluRay > WEB-DL.""" + p1 = Path("/movies/Test.2020.1080p.WEB-DL.mkv") + p2 = Path("/movies/Test.2020.1080p.BluRay.mkv") + items = [(p1, _mi()), (p2, _mi())] + qc = [ + {"path": str(p1), "resolution": "1920x1080", "size_bytes": 2000000}, + {"path": str(p2), "resolution": "1920x1080", "size_bytes": 2000000}, + ] + idx = choose_keep_index(items, "by_quality", quality_comparison=qc) + assert idx == 1 # BluRay + + def test_by_quality_prefers_hevc_over_h264(self): + """Same resolution/source: x265 > x264.""" + p1 = Path("/movies/Test.2020.1080p.BluRay.x264.mkv") + p2 = Path("/movies/Test.2020.1080p.BluRay.x265.mkv") + items = [(p1, _mi()), (p2, _mi())] + qc = [ + {"path": str(p1), "resolution": "1920x1080", "codec": "h264", "size_bytes": 1500000}, + {"path": str(p2), "resolution": "1920x1080", "codec": "hevc", "size_bytes": 1200000}, + ] + idx = choose_keep_index(items, "by_quality", quality_comparison=qc) + assert idx == 1 # x265 + + def test_by_quality_uses_size_as_tiebreaker(self): + """Same resolution/source/codec: larger file wins.""" + p1 = Path("/movies/Test.2020.1080p.BluRay.x264.mkv") + p2 = Path("/movies/Test.2020.1080p.BluRay.x264.mkv") + items = [(p1, _mi()), (p2, _mi())] + qc = [ + {"path": str(p1), "resolution": "1920x1080", "size_bytes": 1000000}, + {"path": str(p2), "resolution": "1920x1080", "size_bytes": 2000000}, + ] + idx = choose_keep_index(items, "by_quality", quality_comparison=qc) + assert idx == 1 # larger + + def test_by_quality_fallback_to_filename(self): + """No resolution/codec in qc -> parse from filename.""" + p1 = Path("/movies/Test.2020.720p.WEB-DL.mkv") + p2 = Path("/movies/Test.2020.2160p.BluRay.x265.mkv") + items = [(p1, _mi()), (p2, _mi())] + qc = [ + {"path": str(p1), "size_bytes": 0}, + {"path": str(p2), "size_bytes": 0}, + ] + idx = choose_keep_index(items, "by_quality", quality_comparison=qc) + assert idx == 1 # 2160p from filename + + def test_by_quality_returns_first_when_all_equal(self): + """All equal -> keep first (index 0).""" + p1 = Path("/movies/Test.2020.1080p.BluRay.mkv") + p2 = Path("/movies/Test.2020.1080p.BluRay.mkv") + items = [(p1, _mi()), (p2, _mi())] + qc = [ + {"path": str(p1), "resolution": "1920x1080", "size_bytes": 1000000}, + {"path": str(p2), "resolution": "1920x1080", "size_bytes": 1000000}, + ] + idx = choose_keep_index(items, "by_quality", quality_comparison=qc) + assert idx == 0 + + +class TestOtherStrategies: + """Tests for by_reputation, first_seen, manual.""" + + def test_manual_returns_none(self): + items = [(Path("/a.mkv"), _mi()), (Path("/b.mkv"), _mi())] + assert choose_keep_index(items, "manual") is None + + def test_first_seen_returns_zero(self): + items = [(Path("/a.mkv"), _mi()), (Path("/b.mkv"), _mi())] + assert choose_keep_index(items, "first_seen") == 0 + + def test_by_reputation_prefers_higher_score(self): + mi_low = _mi() + mi_low.reputation_score = 6.0 + mi_low.reputation_votes = 100 + mi_high = _mi() + mi_high.reputation_score = 8.5 + mi_high.reputation_votes = 500 + items = [ + (Path("/a.mkv"), mi_low), + (Path("/b.mkv"), mi_high), + ] + idx = choose_keep_index(items, "by_reputation") + assert idx == 1 diff --git a/tests/test_planner.py b/tests/test_planner.py index 2611b6f..f5fd162 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -375,6 +375,54 @@ def test_generate_plan_summary(config): assert plan.summary["quarantine"] == 0 +def test_generate_plan_with_analysis_by_quality(config): + """With analysis duplicates and by_quality, higher-quality file is kept.""" + config.duplicate_keep = "by_quality" + p_720 = Path("/mnt/nas/videos/movie/Test.2020.720p.WEB-DL.mkv") + p_1080 = Path("/mnt/nas/videos/movie/Test.2020.1080p.BluRay.mkv") + vf_720 = VideoFile( + path=p_720, + filename="Test.2020.720p.WEB-DL.mkv", + size_bytes=1000000, + modified_timestamp=datetime.now(), + category="movie", + ) + vf_1080 = VideoFile( + path=p_1080, + filename="Test.2020.1080p.BluRay.mkv", + size_bytes=2000000, + modified_timestamp=datetime.now(), + category="movie", + ) + identity = MovieIdentity( + title="Test", + year=2020, + confidence=0.9, + needs_review=False, + original_filename="Test.2020.mkv", + ) + identities = [(vf_720, identity), (vf_1080, identity)] + analysis_data = { + "metadata": {"source_identities": "identities.json"}, + "completeness": [], + "duplicates": [ + { + "identity": {"type": "movie", "title": "Test", "year": 2020}, + "files": [str(p_720), str(p_1080)], + "quality_comparison": [ + {"path": str(p_720), "resolution": "1280x720", "size_bytes": 1000000}, + {"path": str(p_1080), "resolution": "1920x1080", "size_bytes": 2000000}, + ], + } + ], + } + plan = generate_plan(identities, config, analysis_data=analysis_data) + assert plan.summary["quarantine"] == 1 + assert plan.operations[0].operation_type == "quarantine" # 720p quarantined + assert plan.operations[1].operation_type == "move" # 1080p kept + assert "画质" in plan.operations[0].reason + + def test_generate_plan_with_different_extensions(config): """Test plan generation preserves file extensions.""" extensions = [".mp4", ".mkv", ".avi"]