Fix identity reconstruction metadata loss with v2 schema
Implements identities.json v2 schema with embedded video metadata to fix duplicate resolution by quality, which previously failed due to VideoFile objects being reconstructed with hardcoded defaults (size_bytes=0, resolution=None, codec=None). Changes: - Add --inventory flag to vlm parse command to embed video metadata - Update _video_file_from_record() to extract embedded metadata if present - Add vlm_schema_version field to identities.json (v1.0 or v2.0) - Maintain backward compatibility with v1 files (no metadata) Schema v2 format: - Embeds video_metadata object in each record (movies/series) - Contains: size_bytes, modified_timestamp, resolution, codec, duration_seconds, bitrate_kbps - Enables accurate quality comparison during duplicate analysis Testing: - Added comprehensive unit tests for io.py functions - Added CLI integration tests for parse command - Added end-to-end tests for duplicate quality comparison - All 437 existing tests still pass (1 pre-existing failure in executor) Documentation: - Updated README.md with --inventory usage examples - Updated CLAUDE.md with schema versioning details - Added workflow examples showing metadata embedding This fix resolves the critical P0 issue where duplicate resolution by_quality strategy failed completely due to missing video metadata in reconstructed VideoFile objects. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
windyboy
co-authored by
Claude Sonnet 4.5
Claude Sonnet 4.5
parent
1f55eab304
commit
065195b83b
+68
-20
@@ -169,16 +169,26 @@ def scan(
|
||||
default=Path('identities.json'),
|
||||
help='Output file for parsed identities (default: identities.json)'
|
||||
)
|
||||
@click.option(
|
||||
'--inventory',
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
default=None,
|
||||
help='Inventory CSV to embed video metadata (enables v2 schema with quality data)'
|
||||
)
|
||||
@pass_context
|
||||
def parse(ctx: CLIContext, input: Path, output: Path):
|
||||
def parse(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Path]):
|
||||
"""Parse identities from filenames.
|
||||
|
||||
|
||||
Extracts movie titles, years, series titles, seasons, and episodes
|
||||
from video filenames in the inventory.
|
||||
|
||||
|
||||
Use --inventory to embed video metadata (size, resolution, codec) in the output,
|
||||
which enables accurate duplicate resolution by quality in the analyze stage.
|
||||
|
||||
Example:
|
||||
|
||||
vlm parse # Use default files
|
||||
|
||||
vlm parse # Use default files (v1 schema)
|
||||
vlm parse --inventory inventory.csv # Embed metadata (v2 schema)
|
||||
vlm parse --input my_inventory.csv # Custom input
|
||||
vlm parse --output parsed_identities.json # Custom output
|
||||
"""
|
||||
@@ -186,15 +196,25 @@ def parse(ctx: CLIContext, input: Path, output: Path):
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from vlm.parser import parse_movie, parse_series
|
||||
|
||||
from vlm.io import load_inventory_csv
|
||||
|
||||
config = ctx.config
|
||||
logger = ctx.logger
|
||||
|
||||
|
||||
try:
|
||||
# Display parse start message
|
||||
click.echo(f"Parsing identities from: {input}")
|
||||
|
||||
# Load video metadata from inventory if provided
|
||||
path_to_metadata = {}
|
||||
if inventory:
|
||||
click.echo(f"Loading video metadata from: {inventory}")
|
||||
inventory_files = load_inventory_csv(inventory)
|
||||
path_to_metadata = {str(vf.path): vf for vf in inventory_files}
|
||||
click.echo(f"Loaded metadata for {len(path_to_metadata)} files")
|
||||
|
||||
click.echo()
|
||||
|
||||
|
||||
# Load inventory from CSV
|
||||
video_files = []
|
||||
with open(input, 'r', encoding='utf-8') as csvfile:
|
||||
@@ -203,7 +223,7 @@ def parse(ctx: CLIContext, input: Path, output: Path):
|
||||
for line in csvfile:
|
||||
if not line.startswith('#'):
|
||||
lines.append(line)
|
||||
|
||||
|
||||
# Parse CSV
|
||||
reader = csv.DictReader(lines)
|
||||
for row in reader:
|
||||
@@ -212,7 +232,7 @@ def parse(ctx: CLIContext, input: Path, output: Path):
|
||||
'filename': row['filename'],
|
||||
'category': row['category']
|
||||
})
|
||||
|
||||
|
||||
click.echo(f"Loaded {len(video_files)} files from inventory")
|
||||
click.echo()
|
||||
|
||||
@@ -221,27 +241,48 @@ def parse(ctx: CLIContext, input: Path, output: Path):
|
||||
series_identities = []
|
||||
anime_files = []
|
||||
other_files = []
|
||||
|
||||
|
||||
def get_video_metadata(file_path: str) -> dict:
|
||||
"""Extract video metadata from inventory if available."""
|
||||
if not path_to_metadata:
|
||||
return {}
|
||||
vf = path_to_metadata.get(file_path)
|
||||
if not vf:
|
||||
return {}
|
||||
return {
|
||||
'size_bytes': vf.size_bytes,
|
||||
'modified_timestamp': vf.modified_timestamp.isoformat(),
|
||||
'resolution': vf.resolution,
|
||||
'codec': vf.codec,
|
||||
'duration_seconds': vf.duration_seconds,
|
||||
'bitrate_kbps': vf.bitrate_kbps,
|
||||
}
|
||||
|
||||
for vf in video_files:
|
||||
filename = vf['filename']
|
||||
category = vf['category']
|
||||
|
||||
file_path = vf['path']
|
||||
video_metadata = get_video_metadata(file_path)
|
||||
|
||||
if category == 'movie':
|
||||
identity = parse_movie(filename, extensions=config.video_extensions)
|
||||
movie_identities.append({
|
||||
'path': vf['path'],
|
||||
record = {
|
||||
'path': file_path,
|
||||
'filename': filename,
|
||||
'category': category,
|
||||
'title': identity.title,
|
||||
'year': identity.year,
|
||||
'confidence': identity.confidence,
|
||||
'needs_review': identity.needs_review
|
||||
})
|
||||
|
||||
}
|
||||
if video_metadata:
|
||||
record['video_metadata'] = video_metadata
|
||||
movie_identities.append(record)
|
||||
|
||||
elif category == 'series':
|
||||
identity = parse_series(filename, extensions=config.video_extensions)
|
||||
series_identities.append({
|
||||
'path': vf['path'],
|
||||
record = {
|
||||
'path': file_path,
|
||||
'filename': filename,
|
||||
'category': category,
|
||||
'title': identity.title,
|
||||
@@ -249,7 +290,10 @@ def parse(ctx: CLIContext, input: Path, output: Path):
|
||||
'episodes': identity.episodes,
|
||||
'confidence': identity.confidence,
|
||||
'needs_review': identity.needs_review
|
||||
})
|
||||
}
|
||||
if video_metadata:
|
||||
record['video_metadata'] = video_metadata
|
||||
series_identities.append(record)
|
||||
|
||||
elif category == 'anime':
|
||||
# Anime files are not parsed in v1
|
||||
@@ -299,8 +343,12 @@ def parse(ctx: CLIContext, input: Path, output: Path):
|
||||
|
||||
# Build JSON structure
|
||||
generation_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
|
||||
# Use v2 schema if video metadata was embedded
|
||||
schema_version = "2.0" if path_to_metadata else "1.0"
|
||||
|
||||
identities_data = {
|
||||
'vlm_schema_version': schema_version,
|
||||
'metadata': {
|
||||
'generated': generation_timestamp,
|
||||
'source_inventory': str(input),
|
||||
|
||||
+21
-7
@@ -46,17 +46,31 @@ def save_identities_json(data: dict, path: Path) -> None:
|
||||
|
||||
|
||||
def _video_file_from_record(record: dict) -> VideoFile:
|
||||
"""Build a minimal VideoFile from an identities record (no inventory metadata)."""
|
||||
"""Build a VideoFile from an identities record.
|
||||
|
||||
If the record contains embedded video_metadata (v2 schema), use it.
|
||||
Otherwise, use defaults (v1 schema backward compatibility).
|
||||
"""
|
||||
vm = record.get("video_metadata", {})
|
||||
|
||||
# Parse modified_timestamp if provided (v2), otherwise use current time (v1)
|
||||
modified_timestamp_str = vm.get("modified_timestamp")
|
||||
if modified_timestamp_str:
|
||||
from datetime import datetime
|
||||
modified_timestamp = datetime.fromisoformat(modified_timestamp_str)
|
||||
else:
|
||||
modified_timestamp = utc_now()
|
||||
|
||||
return VideoFile(
|
||||
path=Path(record["path"]),
|
||||
filename=record["filename"],
|
||||
size_bytes=0,
|
||||
modified_timestamp=utc_now(),
|
||||
size_bytes=vm.get("size_bytes", 0),
|
||||
modified_timestamp=modified_timestamp,
|
||||
category=record["category"],
|
||||
resolution=None,
|
||||
codec=None,
|
||||
duration_seconds=None,
|
||||
bitrate_kbps=None,
|
||||
resolution=vm.get("resolution"),
|
||||
codec=vm.get("codec"),
|
||||
duration_seconds=vm.get("duration_seconds"),
|
||||
bitrate_kbps=vm.get("bitrate_kbps"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user