Implement duplicate resolution strategy by quality and enhance related documentation
- Updated `duplicate_resolve.py` to introduce a new strategy for keeping files based on quality, considering resolution, source, codec, and size. - Enhanced `planner.py` to utilize the new quality-based strategy during plan generation, updating quarantine reasons accordingly. - Modified `README.md` to document the new `plan.duplicate_keep` options, including `by_quality`, and provided detailed descriptions of each strategy. - Added unit tests in `test_duplicate_resolve.py` to validate the new quality-based resolution logic. - Updated `analysis.json` and `plan.json` with new timestamps and IDs to reflect recent changes. These updates improve the Video Library Manager's ability to handle duplicate files more effectively, ensuring users retain the highest quality versions.
This commit is contained in:
@@ -69,6 +69,7 @@ VLM follows a read-first, multi-stage pipeline:
|
||||
- `io.py` - Load/save identities JSON/CSV, plan/analysis input helpers
|
||||
- `utils.py` - UTC time, format_size, shared helpers
|
||||
- `analysis.py` - Completeness checking (episode gaps) and duplicate detection
|
||||
- `duplicate_resolve.py` - Duplicate group resolution (by_quality, by_reputation, first_seen, manual)
|
||||
- `planner.py` - Execution plan generation with conflict detection
|
||||
- `executor.py` - File operations (move/rename/quarantine) with rollback logging
|
||||
- `quarantine.py` - Quarantine management with manifest tracking
|
||||
@@ -116,6 +117,7 @@ Key settings:
|
||||
- `log_level` - logging verbosity
|
||||
- `categories` - mapping of category names to directory name lists
|
||||
- `enrichment` (or `enrich`) - TMDB/api_keys, cache_db, translation, reputation; see README for full schema
|
||||
- `plan.duplicate_keep` - when using `vlm plan --analysis`: `by_quality` (resolution > source > codec > size), `by_reputation` (default), `first_seen`, or `manual`
|
||||
|
||||
### Category Mappings
|
||||
|
||||
|
||||
@@ -133,7 +133,11 @@ This creates `plan.json` with:
|
||||
- **Human summary** (中文): short narrative for quick review
|
||||
- **Metadata**: when using `--analysis`, duplicate groups considered and completeness gaps
|
||||
|
||||
Duplicate keep strategy is configurable in `~/.vlm/config.yaml` under `plan.duplicate_keep` (`by_reputation`, `first_seen`, or `manual`). Default is `by_reputation` (prefer external rating; fallback to first-seen).
|
||||
Duplicate keep strategy is configurable in `~/.vlm/config.yaml` under `plan.duplicate_keep`:
|
||||
- `by_quality` - Prefer highest quality (resolution > source > codec > file size). Best for automatic duplicate resolution.
|
||||
- `by_reputation` - Prefer external rating (TMDB); fallback to first-seen. Default.
|
||||
- `first_seen` - Keep the first file in each duplicate group.
|
||||
- `manual` - Do not generate quarantine operations; duplicates are listed in analysis only.
|
||||
|
||||
**Review the plan** by opening `plan.json` in your editor, or read the human summary when you run `vlm execute`. You can edit the plan JSON if needed.
|
||||
|
||||
@@ -406,7 +410,8 @@ log_level: "INFO"
|
||||
|
||||
# Plan behavior (e.g. when using vlm plan --analysis)
|
||||
plan:
|
||||
# Duplicate keep strategy: "by_reputation" (default), "first_seen", or "manual"
|
||||
# Duplicate keep strategy: "by_quality", "by_reputation" (default), "first_seen", or "manual"
|
||||
# by_quality: resolution > source (BluRay > WEB-DL) > codec (x265 > x264) > file size
|
||||
duplicate_keep: "by_reputation"
|
||||
|
||||
# Category mappings (directory name to category)
|
||||
@@ -762,7 +767,7 @@ src/vlm/
|
||||
├── io.py # JSON/CSV load/save, load_analysis_json, plan/analysis input helpers
|
||||
├── utils.py # UTC time, format_size, etc.
|
||||
├── analysis.py # Completeness and duplicate detection
|
||||
├── duplicate_resolve.py # Duplicate group keep-index (by_reputation, first_seen, manual)
|
||||
├── duplicate_resolve.py # Duplicate group keep-index (by_quality, by_reputation, first_seen, manual)
|
||||
├── planner.py # Execution plan generation (optionally consumes analysis)
|
||||
├── executor.py # File operations and rollback
|
||||
├── quarantine.py # Quarantine management
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"metadata": {
|
||||
"generated": "2026-02-10T09:54:17",
|
||||
"generated": "2026-02-10T12:54:42",
|
||||
"source_identities": "identities.json",
|
||||
"total_movies": 637,
|
||||
"total_series": 1492
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"plan_id": "7e4827a4-3692-4ebe-87c7-b92950c87ea9",
|
||||
"created_at": "2026-02-10T09:54:19.304610+00:00",
|
||||
"plan_id": "059bd77a-4f2a-4bc7-860f-dd7fa4a09534",
|
||||
"created_at": "2026-02-10T12:55:28.326159+00:00",
|
||||
"operations": [
|
||||
{
|
||||
"operation_type": "no-op",
|
||||
|
||||
+7
-2
@@ -324,9 +324,14 @@ def validate_config(config: Config) -> list[str]:
|
||||
errors.append("tmdb_region must be a string when set")
|
||||
if not isinstance(config.tmdb_include_adult, bool):
|
||||
errors.append("tmdb_include_adult must be a boolean")
|
||||
if config.duplicate_keep not in ("by_reputation", "first_seen", "manual"):
|
||||
if config.duplicate_keep not in (
|
||||
"by_reputation",
|
||||
"first_seen",
|
||||
"manual",
|
||||
"by_quality",
|
||||
):
|
||||
errors.append(
|
||||
f"duplicate_keep must be one of 'by_reputation', 'first_seen', 'manual', got: {config.duplicate_keep!r}"
|
||||
f"duplicate_keep must be one of 'by_reputation', 'first_seen', 'manual', 'by_quality', got: {config.duplicate_keep!r}"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Duplicate group resolution: choose which file to keep when consuming analysis."""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
from typing import Optional, Union
|
||||
|
||||
from vlm.models import MovieIdentity, SeriesIdentity
|
||||
|
||||
@@ -9,10 +10,12 @@ from vlm.models import MovieIdentity, SeriesIdentity
|
||||
def choose_keep_index(
|
||||
items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]],
|
||||
strategy: str,
|
||||
quality_comparison: Optional[list[dict]] = None,
|
||||
) -> int | None:
|
||||
"""Choose the index of the item to keep in a duplicate group.
|
||||
|
||||
Strategies:
|
||||
- by_quality: Prefer higher quality (resolution > source > codec > size). Requires quality_comparison.
|
||||
- by_reputation: Prefer items with reputation_score; then sort by score desc,
|
||||
then reputation_votes desc, then first_seen (input order). Keep index 0 after sort.
|
||||
- first_seen: Keep the first item (index 0).
|
||||
@@ -20,7 +23,9 @@ def choose_keep_index(
|
||||
|
||||
Args:
|
||||
items: List of (path, identity) for the duplicate group.
|
||||
strategy: One of "by_reputation", "first_seen", "manual".
|
||||
strategy: One of "by_quality", "by_reputation", "first_seen", "manual".
|
||||
quality_comparison: List of quality dicts (filename, path, size_bytes, resolution?, codec?)
|
||||
aligned with items. Required when strategy is "by_quality".
|
||||
|
||||
Returns:
|
||||
Index in [0, len(items)) to keep, or None for manual.
|
||||
@@ -29,11 +34,89 @@ def choose_keep_index(
|
||||
return None
|
||||
if strategy == "first_seen":
|
||||
return 0
|
||||
if strategy == "by_quality":
|
||||
if quality_comparison is None or len(quality_comparison) != len(items):
|
||||
return 0 # Fallback to first if quality data missing/mismatched
|
||||
return _by_quality_index(items, quality_comparison)
|
||||
if strategy == "by_reputation":
|
||||
return _by_reputation_index(items)
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_resolution_tier(resolution: Optional[str], path: Path) -> int:
|
||||
"""Parse resolution to tier. Higher = better. 2160=4, 1080=3, 720=2, 0=unknown."""
|
||||
if resolution:
|
||||
m = re.search(r"(\d{3,4})[x×](\d{3,4})", resolution, re.I)
|
||||
if m:
|
||||
h = int(m.group(2))
|
||||
if h >= 2160:
|
||||
return 4
|
||||
if h >= 1080:
|
||||
return 3
|
||||
if h >= 720:
|
||||
return 2
|
||||
return 1
|
||||
s = path.name.upper()
|
||||
if "2160" in s or "4K" in s or "UHD" in s:
|
||||
return 4
|
||||
if "1080" in s:
|
||||
return 3
|
||||
if "720" in s:
|
||||
return 2
|
||||
if "480" in s or "576" in s:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_source_tier(path: Path) -> int:
|
||||
"""Parse source type from path. BluRay=3, WEB-DL=2, HDTV=1, 0=other."""
|
||||
s = path.name.upper()
|
||||
if "BLURAY" in s or "BLU-RAY" in s or "BD" in s:
|
||||
return 3
|
||||
if "WEB-DL" in s or "WEBRIP" in s or "WEB" in s:
|
||||
return 2
|
||||
if "HDTV" in s or "HDTVRIP" in s:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_codec_tier(codec: Optional[str], path: Path) -> int:
|
||||
"""Parse codec to tier. hevc/x265=2, h264/x264=1, 0=other."""
|
||||
if codec:
|
||||
c = codec.lower()
|
||||
if "hevc" in c or "h265" in c or "x265" in c:
|
||||
return 2
|
||||
if "h264" in c or "x264" in c or "avc" in c:
|
||||
return 1
|
||||
s = path.name.upper()
|
||||
if "X265" in s or "H265" in s or "HEVC" in s:
|
||||
return 2
|
||||
if "X264" in s or "H264" in s or "AVC" in s:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _by_quality_index(
|
||||
items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]],
|
||||
quality_comparison: list[dict],
|
||||
) -> int:
|
||||
"""Sort by: resolution desc, source desc, codec desc, size desc, index asc. Return best index."""
|
||||
def key(idx_item: tuple[int, tuple[Path, Union[MovieIdentity, SeriesIdentity]]]) -> tuple:
|
||||
idx, (path, _) = idx_item
|
||||
qc = quality_comparison[idx] if idx < len(quality_comparison) else {}
|
||||
resolution = qc.get("resolution")
|
||||
codec = qc.get("codec")
|
||||
size = qc.get("size_bytes", 0) or 0
|
||||
res_tier = _parse_resolution_tier(resolution, path)
|
||||
src_tier = _parse_source_tier(path)
|
||||
codec_tier = _parse_codec_tier(codec, path)
|
||||
return (-res_tier, -src_tier, -codec_tier, -size, idx)
|
||||
|
||||
indexed = list(enumerate(items))
|
||||
indexed.sort(key=key)
|
||||
return indexed[0][0]
|
||||
|
||||
|
||||
def _by_reputation_index(
|
||||
items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]],
|
||||
) -> int:
|
||||
|
||||
+25
-10
@@ -22,6 +22,7 @@ from vlm.models import (
|
||||
)
|
||||
|
||||
QUARANTINE_REASON_DUPLICATE = "重复项(已按外部评分保留一条)"
|
||||
QUARANTINE_REASON_DUPLICATE_BY_QUALITY = "重复项(已按画质保留最优)"
|
||||
|
||||
|
||||
def generate_plan(
|
||||
@@ -50,26 +51,40 @@ def generate_plan(
|
||||
for dup in analysis_data.get("duplicates", []):
|
||||
paths = dup.get("files", [])
|
||||
indices = [path_to_index[p] for p in paths if p in path_to_index]
|
||||
items = [
|
||||
(identities[i][0].path, identities[i][1])
|
||||
for i in indices
|
||||
if identities[i][1] is not None
|
||||
and isinstance(identities[i][1], (MovieIdentity, SeriesIdentity))
|
||||
]
|
||||
path_to_qc = {qc.get("path"): qc for qc in dup.get("quality_comparison", [])}
|
||||
items = []
|
||||
valid_indices = []
|
||||
for i in indices:
|
||||
identity = identities[i][1]
|
||||
if identity is not None and isinstance(
|
||||
identity, (MovieIdentity, SeriesIdentity)
|
||||
):
|
||||
items.append((identities[i][0].path, identity))
|
||||
valid_indices.append(i)
|
||||
if not items:
|
||||
continue
|
||||
keep_idx = choose_keep_index(items, config.duplicate_keep)
|
||||
quality_list = [
|
||||
path_to_qc.get(str(p), {}) for p, _ in items
|
||||
]
|
||||
keep_idx = choose_keep_index(
|
||||
items, config.duplicate_keep, quality_comparison=quality_list
|
||||
)
|
||||
if keep_idx is None:
|
||||
continue
|
||||
keep_identity_index = indices[keep_idx]
|
||||
quarantine_indices = set(indices) - {keep_identity_index}
|
||||
keep_identity_index = valid_indices[keep_idx]
|
||||
quarantine_indices = set(valid_indices) - {keep_identity_index}
|
||||
reason = (
|
||||
QUARANTINE_REASON_DUPLICATE_BY_QUALITY
|
||||
if config.duplicate_keep == "by_quality"
|
||||
else QUARANTINE_REASON_DUPLICATE
|
||||
)
|
||||
for i in quarantine_indices:
|
||||
vf = identities[i][0]
|
||||
operations[i] = FileOperation(
|
||||
operation_type="quarantine",
|
||||
source_path=vf.path,
|
||||
destination_path=None,
|
||||
reason=QUARANTINE_REASON_DUPLICATE,
|
||||
reason=reason,
|
||||
has_conflict=False,
|
||||
conflict_reason=None,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Unit tests for duplicate resolution strategies."""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
|
||||
from vlm.models import MovieIdentity
|
||||
from vlm.duplicate_resolve import choose_keep_index
|
||||
|
||||
|
||||
def _mi(title: str = "Test", year: int | None = 2020) -> MovieIdentity:
|
||||
return MovieIdentity(
|
||||
title=title,
|
||||
year=year,
|
||||
confidence=0.9,
|
||||
needs_review=False,
|
||||
original_filename="test.mkv",
|
||||
)
|
||||
|
||||
|
||||
class TestByQuality:
|
||||
"""Tests for by_quality strategy."""
|
||||
|
||||
def test_by_quality_prefers_higher_resolution(self):
|
||||
"""1080p vs 720p -> keep 1080p."""
|
||||
p1 = Path("/movies/Test.2020.720p.BluRay.mkv")
|
||||
p2 = Path("/movies/Test.2020.1080p.BluRay.mkv")
|
||||
items = [(p1, _mi()), (p2, _mi())]
|
||||
qc = [
|
||||
{"path": str(p1), "resolution": "1280x720", "size_bytes": 1000000},
|
||||
{"path": str(p2), "resolution": "1920x1080", "size_bytes": 2000000},
|
||||
]
|
||||
idx = choose_keep_index(items, "by_quality", quality_comparison=qc)
|
||||
assert idx == 1 # 1080p
|
||||
|
||||
def test_by_quality_prefers_bluray_over_webdl(self):
|
||||
"""Same resolution: BluRay > WEB-DL."""
|
||||
p1 = Path("/movies/Test.2020.1080p.WEB-DL.mkv")
|
||||
p2 = Path("/movies/Test.2020.1080p.BluRay.mkv")
|
||||
items = [(p1, _mi()), (p2, _mi())]
|
||||
qc = [
|
||||
{"path": str(p1), "resolution": "1920x1080", "size_bytes": 2000000},
|
||||
{"path": str(p2), "resolution": "1920x1080", "size_bytes": 2000000},
|
||||
]
|
||||
idx = choose_keep_index(items, "by_quality", quality_comparison=qc)
|
||||
assert idx == 1 # BluRay
|
||||
|
||||
def test_by_quality_prefers_hevc_over_h264(self):
|
||||
"""Same resolution/source: x265 > x264."""
|
||||
p1 = Path("/movies/Test.2020.1080p.BluRay.x264.mkv")
|
||||
p2 = Path("/movies/Test.2020.1080p.BluRay.x265.mkv")
|
||||
items = [(p1, _mi()), (p2, _mi())]
|
||||
qc = [
|
||||
{"path": str(p1), "resolution": "1920x1080", "codec": "h264", "size_bytes": 1500000},
|
||||
{"path": str(p2), "resolution": "1920x1080", "codec": "hevc", "size_bytes": 1200000},
|
||||
]
|
||||
idx = choose_keep_index(items, "by_quality", quality_comparison=qc)
|
||||
assert idx == 1 # x265
|
||||
|
||||
def test_by_quality_uses_size_as_tiebreaker(self):
|
||||
"""Same resolution/source/codec: larger file wins."""
|
||||
p1 = Path("/movies/Test.2020.1080p.BluRay.x264.mkv")
|
||||
p2 = Path("/movies/Test.2020.1080p.BluRay.x264.mkv")
|
||||
items = [(p1, _mi()), (p2, _mi())]
|
||||
qc = [
|
||||
{"path": str(p1), "resolution": "1920x1080", "size_bytes": 1000000},
|
||||
{"path": str(p2), "resolution": "1920x1080", "size_bytes": 2000000},
|
||||
]
|
||||
idx = choose_keep_index(items, "by_quality", quality_comparison=qc)
|
||||
assert idx == 1 # larger
|
||||
|
||||
def test_by_quality_fallback_to_filename(self):
|
||||
"""No resolution/codec in qc -> parse from filename."""
|
||||
p1 = Path("/movies/Test.2020.720p.WEB-DL.mkv")
|
||||
p2 = Path("/movies/Test.2020.2160p.BluRay.x265.mkv")
|
||||
items = [(p1, _mi()), (p2, _mi())]
|
||||
qc = [
|
||||
{"path": str(p1), "size_bytes": 0},
|
||||
{"path": str(p2), "size_bytes": 0},
|
||||
]
|
||||
idx = choose_keep_index(items, "by_quality", quality_comparison=qc)
|
||||
assert idx == 1 # 2160p from filename
|
||||
|
||||
def test_by_quality_returns_first_when_all_equal(self):
|
||||
"""All equal -> keep first (index 0)."""
|
||||
p1 = Path("/movies/Test.2020.1080p.BluRay.mkv")
|
||||
p2 = Path("/movies/Test.2020.1080p.BluRay.mkv")
|
||||
items = [(p1, _mi()), (p2, _mi())]
|
||||
qc = [
|
||||
{"path": str(p1), "resolution": "1920x1080", "size_bytes": 1000000},
|
||||
{"path": str(p2), "resolution": "1920x1080", "size_bytes": 1000000},
|
||||
]
|
||||
idx = choose_keep_index(items, "by_quality", quality_comparison=qc)
|
||||
assert idx == 0
|
||||
|
||||
|
||||
class TestOtherStrategies:
|
||||
"""Tests for by_reputation, first_seen, manual."""
|
||||
|
||||
def test_manual_returns_none(self):
|
||||
items = [(Path("/a.mkv"), _mi()), (Path("/b.mkv"), _mi())]
|
||||
assert choose_keep_index(items, "manual") is None
|
||||
|
||||
def test_first_seen_returns_zero(self):
|
||||
items = [(Path("/a.mkv"), _mi()), (Path("/b.mkv"), _mi())]
|
||||
assert choose_keep_index(items, "first_seen") == 0
|
||||
|
||||
def test_by_reputation_prefers_higher_score(self):
|
||||
mi_low = _mi()
|
||||
mi_low.reputation_score = 6.0
|
||||
mi_low.reputation_votes = 100
|
||||
mi_high = _mi()
|
||||
mi_high.reputation_score = 8.5
|
||||
mi_high.reputation_votes = 500
|
||||
items = [
|
||||
(Path("/a.mkv"), mi_low),
|
||||
(Path("/b.mkv"), mi_high),
|
||||
]
|
||||
idx = choose_keep_index(items, "by_reputation")
|
||||
assert idx == 1
|
||||
@@ -375,6 +375,54 @@ def test_generate_plan_summary(config):
|
||||
assert plan.summary["quarantine"] == 0
|
||||
|
||||
|
||||
def test_generate_plan_with_analysis_by_quality(config):
|
||||
"""With analysis duplicates and by_quality, higher-quality file is kept."""
|
||||
config.duplicate_keep = "by_quality"
|
||||
p_720 = Path("/mnt/nas/videos/movie/Test.2020.720p.WEB-DL.mkv")
|
||||
p_1080 = Path("/mnt/nas/videos/movie/Test.2020.1080p.BluRay.mkv")
|
||||
vf_720 = VideoFile(
|
||||
path=p_720,
|
||||
filename="Test.2020.720p.WEB-DL.mkv",
|
||||
size_bytes=1000000,
|
||||
modified_timestamp=datetime.now(),
|
||||
category="movie",
|
||||
)
|
||||
vf_1080 = VideoFile(
|
||||
path=p_1080,
|
||||
filename="Test.2020.1080p.BluRay.mkv",
|
||||
size_bytes=2000000,
|
||||
modified_timestamp=datetime.now(),
|
||||
category="movie",
|
||||
)
|
||||
identity = MovieIdentity(
|
||||
title="Test",
|
||||
year=2020,
|
||||
confidence=0.9,
|
||||
needs_review=False,
|
||||
original_filename="Test.2020.mkv",
|
||||
)
|
||||
identities = [(vf_720, identity), (vf_1080, identity)]
|
||||
analysis_data = {
|
||||
"metadata": {"source_identities": "identities.json"},
|
||||
"completeness": [],
|
||||
"duplicates": [
|
||||
{
|
||||
"identity": {"type": "movie", "title": "Test", "year": 2020},
|
||||
"files": [str(p_720), str(p_1080)],
|
||||
"quality_comparison": [
|
||||
{"path": str(p_720), "resolution": "1280x720", "size_bytes": 1000000},
|
||||
{"path": str(p_1080), "resolution": "1920x1080", "size_bytes": 2000000},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
plan = generate_plan(identities, config, analysis_data=analysis_data)
|
||||
assert plan.summary["quarantine"] == 1
|
||||
assert plan.operations[0].operation_type == "quarantine" # 720p quarantined
|
||||
assert plan.operations[1].operation_type == "move" # 1080p kept
|
||||
assert "画质" in plan.operations[0].reason
|
||||
|
||||
|
||||
def test_generate_plan_with_different_extensions(config):
|
||||
"""Test plan generation preserves file extensions."""
|
||||
extensions = [".mp4", ".mkv", ".avi"]
|
||||
|
||||
Reference in New Issue
Block a user