windy/dl-organizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enhance project structure and add new files for enrichment and analysis

Browse Source 
- Updated AGENTS.md to reflect changes in CLI commands and module organization, including the addition of an enrichment step and new functional modules.
- Introduced analysis.json, identities.json, inventory.csv, and plan.json to support enriched metadata and execution planning.
- Added CODE_IMPROVEMENTS.md to document identified code issues and proposed solutions for future enhancements.
- Updated README.md to include new enrichment features and configuration options.
- Removed unused dependency on ffmpeg-python from pyproject.toml.

These changes improve the overall functionality and maintainability of the Video Library Manager project.
This commit is contained in:
windyboy
2026-02-10 16:56:17 +08:00
parent f0c951ad7f
commit dcd87754cf
39 changed files with 145509 additions and 642 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vlm/analysis.py
+14 -40
View File
@@ -68,65 +68,39 @@ def analyze_series_completeness(episodes: list[SeriesIdentity]) -> list[SeasonCo
def detect_duplicates(
identities: list[MovieIdentity | SeriesIdentity],
files: list[VideoFile]
identity_file_pairs: list[tuple[MovieIdentity | SeriesIdentity, VideoFile]],
) -> list[DuplicateGroup]:
"""Detect duplicate video files and provide quality comparison data.
Groups files by normalized identity (title+year for movies, title+season+episode
for series) and identifies groups with multiple files as potential duplicates.
Uses (identity, file) pairs so that same filename under different paths are
not conflated.
Args:
identities: List of parsed identities (movies or series)
files: List of video files corresponding to the identities
identity_file_pairs: List of (identity, video_file) in matching order
Returns:
List of DuplicateGroup objects for files with duplicates
"""
# Create a mapping from original filename to VideoFile for quick lookup
file_map = {file.filename: file for file in files}
# Group identities by normalized identity
groups: dict[tuple, list[tuple[MovieIdentity | SeriesIdentity, VideoFile]]] = {}
for identity in identities:
# Create grouping key based on identity type
for identity, video_file in identity_file_pairs:
if isinstance(identity, MovieIdentity):
# For movies: group by (title, year)
# Skip if year is None (needs review)
if identity.year is None:
continue
key = ('movie', identity.title, identity.year)
key = ("movie", identity.title, identity.year)
if key not in groups:
groups[key] = []
groups[key].append((identity, video_file))
else: # SeriesIdentity
# For series: group by (title, season, episode)
# Skip if season is None or episodes is empty (needs review)
if identity.season is None or not identity.episodes:
continue
# For multi-episode files, use the first episode for grouping
# Each episode in the list should be treated separately
for episode in identity.episodes:
key = ('series', identity.title, identity.season, episode)
# Get the corresponding VideoFile
video_file = file_map.get(identity.original_filename)
if video_file is None:
continue
# Add to group
key = ("series", identity.title, identity.season, episode)
if key not in groups:
groups[key] = []
groups[key].append((identity, video_file))
continue
# Get the corresponding VideoFile for movies
video_file = file_map.get(identity.original_filename)
if video_file is None:
continue
# Add to group
if key not in groups:
groups[key] = []
groups[key].append((identity, video_file))
# Filter groups to only those with multiple files (duplicates)
duplicate_groups = []
src/vlm/cli.py
+52 -469
View File
@@ -5,6 +5,7 @@ It implements global options (--config, --log-level) and error handling.
"""
import sys
import traceback
from pathlib import Path
from typing import Optional
@@ -12,7 +13,9 @@ import click
import yaml
from vlm.config import Config, load_config, create_default_config, validate_config
from vlm.context import CLIContext, pass_context
from vlm.logging_config import setup_logging, get_logger
from vlm.utils import format_size
def default_config_path() -> Path:
@@ -20,17 +23,6 @@ def default_config_path() -> Path:
return Path.home() / ".vlm" / "config.yaml"
class CLIContext:
"""Context object to pass configuration and logger between commands."""
def __init__(self, config: Config, logger):
self.config = config
self.logger = logger
pass_context = click.make_pass_decorator(CLIContext)
@click.group()
@click.option(
'--config',
@@ -105,6 +97,7 @@ def main(ctx, config: Path, log_level: Optional[str]):
ctx.obj = CLIContext(config=cfg, logger=logger)
except Exception as e:
traceback.print_exc(file=sys.stderr)
click.echo(f"Error initializing VLM: {e}", err=True)
sys.exit(1)
@@ -154,118 +147,15 @@ def scan(
vlm scan --reuse-from old.csv # Reuse prior metadata cache
vlm scan --force-refresh-metadata # Re-run ffprobe for all files
"""
from vlm.scanner import scan_library, save_inventory_csv, load_inventory_csv
config = ctx.config
logger = ctx.logger
try:
# Display scan start message
click.echo(f"Scanning library at: {config.library_root}")
click.echo("This may take a while for large libraries...")
click.echo()
progress_state: dict[str, Optional[click.ProgressBar]] = {"bar": None}
progress_position = {"current": 0}
def _scan_progress(processed: int, total: int) -> None:
if total <= 0:
return
if progress_state["bar"] is None:
bar = click.progressbar(
length=total,
label="Scanning files",
show_pos=True,
)
progress_state["bar"] = bar.__enter__()
step = processed - progress_position["current"]
if step > 0 and progress_state["bar"] is not None:
progress_state["bar"].update(step)
progress_position["current"] = processed
metadata_cache = None
cache_source = None
if not force_refresh_metadata:
cache_source = reuse_from if reuse_from is not None else (output if output.exists() else None)
if metadata and force_refresh_metadata:
click.echo("Forcing metadata refresh for all files (cache disabled).")
click.echo()
if metadata and cache_source is not None:
click.echo(f"Loading metadata cache from: {cache_source}")
try:
cached_files = load_inventory_csv(cache_source)
metadata_cache = {str(vf.path): vf for vf in cached_files}
click.echo(f"Loaded metadata cache entries: {len(metadata_cache)}")
click.echo()
except Exception as e:
click.echo(f"Warning: could not load metadata cache: {e}")
click.echo("Continuing without cache.")
click.echo()
# Perform the scan
try:
video_files = scan_library(
config.library_root,
config,
progress_callback=_scan_progress,
include_video_metadata=metadata,
metadata_cache=metadata_cache
)
finally:
if progress_state["bar"] is not None:
progress_state["bar"].__exit__(None, None, None)
click.echo()
# Display summary
click.echo(f"Scan complete!")
click.echo(f" Total files found: {len(video_files)}")
# Count by category
categories = {}
total_size = 0
for vf in video_files:
categories[vf.category] = categories.get(vf.category, 0) + 1
total_size += vf.size_bytes
click.echo(f" Total size: {_format_size(total_size)}")
click.echo()
click.echo("Files by category:")
for category in sorted(categories.keys()):
click.echo(f" {category}: {categories[category]}")
# Save inventory to CSV
click.echo()
click.echo(f"Saving inventory to: {output}")
save_inventory_csv(video_files, output, config.library_root)
click.echo(f"Inventory saved successfully!")
logger.info(f"Scan completed: {len(video_files)} files found, saved to {output}")
from vlm.commands.scan import scan_cmd
scan_cmd(ctx, output, metadata, reuse_from, force_refresh_metadata)
except Exception as e:
click.echo(f"Error during scan: {e}", err=True)
logger.error(f"Scan failed: {e}", exc_info=True)
ctx.logger.error(f"Scan failed: {e}", exc_info=True)
sys.exit(1)
def _format_size(size_bytes: int) -> str:
"""Format file size in human-readable format.
Args:
size_bytes: Size in bytes
Returns:
Formatted string (e.g., "1.5 GB", "234.2 MB")
"""
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
if size_bytes < 1024.0:
return f"{size_bytes:.1f} {unit}"
size_bytes /= 1024.0
return f"{size_bytes:.1f} PB"
@main.command()
@click.option(
'--input',
@@ -337,7 +227,7 @@ def parse(ctx: CLIContext, input: Path, output: Path):
category = vf['category']
if category == 'movie':
identity = parse_movie(filename)
identity = parse_movie(filename, extensions=config.video_extensions)
movie_identities.append({
'path': vf['path'],
'filename': filename,
@@ -349,7 +239,7 @@ def parse(ctx: CLIContext, input: Path, output: Path):
})
elif category == 'series':
identity = parse_series(filename)
identity = parse_series(filename, extensions=config.video_extensions)
series_identities.append({
'path': vf['path'],
'filename': filename,
@@ -543,12 +433,14 @@ def enrich(
progress_state: dict[str, Optional[click.ProgressBar]] = {"bar": None}
progress_position = {"current": 0}
progress_bucket = {"value": -1}
is_tty = bool(getattr(sys.stderr, "isatty", lambda: False)())
def _enrich_progress(processed: int, total_count: int, _metrics: dict[str, int]) -> None:
def _enrich_progress(processed: int, total_count: int, metrics: dict[str, int]) -> None:
if total_count <= 0:
return
if progress_state["bar"] is None:
if is_tty and progress_state["bar"] is None:
bar = click.progressbar(
length=total_count,
label="Enriching records",
@@ -561,6 +453,19 @@ def enrich(
progress_state["bar"].update(step)
progress_position["current"] = processed
if not is_tty:
percent = int(processed * 100 / total_count)
bucket = percent // 5
if bucket > progress_bucket["value"] or processed == total_count:
progress_bucket["value"] = bucket
click.echo(
"Progress: "
f"{processed}/{total_count} ({percent}%) "
f"api_calls={metrics.get('api_calls', 0)} "
f"cache_hits={metrics.get('cache_hits', 0)} "
f"failed={metrics.get('failed', 0)}"
)
try:
if total == 0:
click.echo("No movie/series/anime records found to enrich.")
@@ -596,6 +501,11 @@ def enrich(
click.echo(f" Failed requests: {stats['failed']}")
click.echo(f" Skipped: {stats['skipped']}")
click.echo(f" Needs review: {stats['needs_review']}")
skip_reasons = stats.get("skip_reasons", {})
if isinstance(skip_reasons, dict):
non_zero = [f"{name}={count}" for name, count in sorted(skip_reasons.items()) if int(count) > 0]
if non_zero:
click.echo(f" Skip reasons: {' '.join(non_zero)}")
failed_items = stats.get('failed_items', [])
if isinstance(failed_items, list) and failed_items:
click.echo(" Failure sample:")
@@ -640,8 +550,14 @@ def enrich(
default=Path('analysis.json'),
help='Path to save analysis results (default: analysis.json)'
)
@click.option(
'--inventory',
type=click.Path(exists=True, path_type=Path),
default=None,
help='Optional inventory CSV to merge size/resolution/codec for duplicate quality comparison'
)
@pass_context
def analyze(ctx: CLIContext, input: Path, output: Path):
def analyze(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Path]):
"""Analyze completeness and duplicates.
Detects episode gaps in series and identifies potential duplicate files.
@@ -651,185 +567,23 @@ def analyze(ctx: CLIContext, input: Path, output: Path):
vlm analyze # Use default files
vlm analyze --input my_identities.json # Custom input
vlm analyze --inventory inventory.csv # Merge metadata for quality comparison
vlm analyze --output my_analysis.json # Custom output
"""
import json
from datetime import datetime, timezone
from vlm.analysis import analyze_series_completeness, detect_duplicates
from vlm.models import SeriesIdentity, MovieIdentity, VideoFile
config = ctx.config
logger = ctx.logger
try:
# Display analyze start message
click.echo(f"Analyzing identities from: {input}")
click.echo()
# Load identities from JSON
with open(input, 'r', encoding='utf-8') as jsonfile:
identities_data = json.load(jsonfile)
# Extract movies and series
movies_data = identities_data.get('movies', [])
series_data = identities_data.get('series', [])
click.echo(f"Loaded {len(movies_data)} movies and {len(series_data)} series")
click.echo()
# Convert to identity objects
movie_identities = []
for m in movies_data:
movie_identities.append(MovieIdentity(
title=m['title'],
year=m.get('year'),
confidence=m['confidence'],
needs_review=m['needs_review'],
original_filename=m['filename']
))
series_identities = []
for s in series_data:
series_identities.append(SeriesIdentity(
title=s['title'],
season=s.get('season'),
episodes=s.get('episodes', []),
confidence=s['confidence'],
needs_review=s['needs_review'],
original_filename=s['filename']
))
# Create VideoFile objects for duplicate detection
# We need to reconstruct basic VideoFile info from the identities data
video_files = []
for m in movies_data:
video_files.append(VideoFile(
path=Path(m['path']),
filename=m['filename'],
size_bytes=0, # Not available from identities file
modified_timestamp=datetime.now(timezone.utc),
category=m['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
))
for s in series_data:
video_files.append(VideoFile(
path=Path(s['path']),
filename=s['filename'],
size_bytes=0, # Not available from identities file
modified_timestamp=datetime.now(timezone.utc),
category=s['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
))
# Analyze series completeness
click.echo("Analyzing series completeness...")
completeness_results = analyze_series_completeness(series_identities)
# Detect duplicates
click.echo("Detecting duplicates...")
all_identities = movie_identities + series_identities
duplicate_groups = detect_duplicates(all_identities, video_files)
# Display analysis summary
click.echo()
click.echo("Analysis complete!")
click.echo()
click.echo("Results:")
click.echo(f" Series with episode gaps: {len(completeness_results)}")
if completeness_results:
total_missing = sum(len(c.episodes_missing) for c in completeness_results)
click.echo(f" - Total missing episodes: {total_missing}")
click.echo(f" Duplicate groups found: {len(duplicate_groups)}")
if duplicate_groups:
total_duplicates = sum(len(g.files) for g in duplicate_groups)
click.echo(f" - Total duplicate files: {total_duplicates}")
# Save analysis results to JSON
click.echo()
click.echo(f"Saving analysis results to: {output}")
# Ensure output directory exists
output.parent.mkdir(parents=True, exist_ok=True)
# Build JSON structure
generation_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
# Convert completeness results to dict
completeness_list = []
for c in completeness_results:
completeness_list.append({
'series_title': c.series_title,
'season': c.season,
'episodes_found': c.episodes_found,
'episodes_missing': c.episodes_missing
})
# Convert duplicate groups to dict
duplicates_list = []
for d in duplicate_groups:
# Get identity info
if isinstance(d.identity, MovieIdentity):
identity_info = {
'type': 'movie',
'title': d.identity.title,
'year': d.identity.year
}
else: # SeriesIdentity
identity_info = {
'type': 'series',
'title': d.identity.title,
'season': d.identity.season,
'episodes': d.identity.episodes
}
duplicates_list.append({
'identity': identity_info,
'files': [str(f.path) for f in d.files],
'quality_comparison': d.quality_comparison
})
analysis_data = {
'metadata': {
'generated': generation_timestamp,
'source_identities': str(input),
'total_movies': len(movies_data),
'total_series': len(series_data)
},
'completeness': completeness_list,
'duplicates': duplicates_list
}
# Write JSON file with pretty formatting
with open(output, 'w', encoding='utf-8') as jsonfile:
json.dump(analysis_data, jsonfile, indent=2, ensure_ascii=False)
click.echo(f"Analysis results saved successfully!")
logger.info(f"Analysis completed: {len(completeness_results)} incomplete series, {len(duplicate_groups)} duplicate groups, saved to {output}")
from vlm.commands.analyze import analyze_cmd
analyze_cmd(ctx, input, output, inventory)
except FileNotFoundError:
click.echo(f"Error: Input file not found: {input}", err=True)
logger.error(f"Input file not found: {input}")
ctx.logger.error(f"Input file not found: {input}")
sys.exit(1)
except json.JSONDecodeError as e:
click.echo(f"Error: Failed to parse JSON file: {e}", err=True)
logger.error(f"JSON parsing failed: {e}", exc_info=True)
ctx.logger.error(f"JSON parsing failed: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error during analysis: {e}", err=True)
logger.error(f"Analysis failed: {e}", exc_info=True)
ctx.logger.error(f"Analysis failed: {e}", exc_info=True)
sys.exit(1)
@@ -859,192 +613,20 @@ def plan(ctx: CLIContext, input: Path, output: Path):
vlm plan --input my_identities.json # Custom input
vlm plan --output my_plan.json # Custom output
"""
import json
from vlm.planner import generate_plan, save_plan
from vlm.models import MovieIdentity, SeriesIdentity, VideoFile
from datetime import datetime, timezone
config = ctx.config
logger = ctx.logger
try:
# Display plan start message
click.echo(f"Generating execution plan from: {input}")
click.echo()
# Load identities from JSON
with open(input, 'r', encoding='utf-8') as jsonfile:
identities_data = json.load(jsonfile)
# Extract movies and series
movies_data = identities_data.get('movies', [])
series_data = identities_data.get('series', [])
anime_data = identities_data.get('anime', [])
other_data = identities_data.get('other', [])
click.echo(f"Loaded {len(movies_data)} movies, {len(series_data)} series, {len(anime_data)} anime, {len(other_data)} other")
click.echo()
# Build list of (VideoFile, Identity) tuples for plan generator
identities_list = []
# Process movies
for m in movies_data:
is_approved = m.get('review_status') == 'approved'
video_file = VideoFile(
path=Path(m['path']),
filename=m['filename'],
size_bytes=0, # Not available from identities file
modified_timestamp=datetime.now(timezone.utc),
category=m['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
)
movie_identity = MovieIdentity(
title=m.get('display_title', m['title']),
year=m.get('year'),
confidence=m['confidence'],
needs_review=(m['needs_review'] and not is_approved),
original_filename=m['filename'],
canonical_id=m.get('canonical_id'),
title_zh=m.get('title_zh'),
title_en=m.get('title_en'),
translation_source=m.get('translation_source'),
reputation_score=m.get('reputation_score'),
reputation_votes=m.get('reputation_votes'),
reputation_source=m.get('reputation_source'),
review_status=m.get('review_status', 'pending'),
enrichment_confidence=m.get('enrichment_confidence'),
provider_metadata=m.get('provider_metadata', {})
)
identities_list.append((video_file, movie_identity))
# Process series
for s in series_data:
is_approved = s.get('review_status') == 'approved'
video_file = VideoFile(
path=Path(s['path']),
filename=s['filename'],
size_bytes=0, # Not available from identities file
modified_timestamp=datetime.now(timezone.utc),
category=s['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
)
series_identity = SeriesIdentity(
title=s.get('display_title', s['title']),
season=s.get('season'),
episodes=s.get('episodes', []),
confidence=s['confidence'],
needs_review=(s['needs_review'] and not is_approved),
original_filename=s['filename'],
canonical_id=s.get('canonical_id'),
title_zh=s.get('title_zh'),
title_en=s.get('title_en'),
translation_source=s.get('translation_source'),
reputation_score=s.get('reputation_score'),
reputation_votes=s.get('reputation_votes'),
reputation_source=s.get('reputation_source'),
review_status=s.get('review_status', 'pending'),
enrichment_confidence=s.get('enrichment_confidence'),
provider_metadata=s.get('provider_metadata', {})
)
identities_list.append((video_file, series_identity))
# Process anime (no identity in v1)
for a in anime_data:
video_file = VideoFile(
path=Path(a['path']),
filename=a['filename'],
size_bytes=0,
modified_timestamp=datetime.now(timezone.utc),
category=a['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
)
identities_list.append((video_file, None))
# Process other (no identity)
for o in other_data:
video_file = VideoFile(
path=Path(o['path']),
filename=o['filename'],
size_bytes=0,
modified_timestamp=datetime.now(timezone.utc),
category=o['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
)
identities_list.append((video_file, None))
# Generate execution plan
click.echo("Generating execution plan...")
execution_plan = generate_plan(identities_list, config)
# Display plan summary
click.echo()
click.echo("Plan generation complete!")
click.echo()
click.echo("Operation summary:")
click.echo(f" Total operations: {execution_plan.summary['total']}")
click.echo(f" Move operations: {execution_plan.summary['move']}")
click.echo(f" Rename operations: {execution_plan.summary['rename']}")
click.echo(f" Quarantine operations: {execution_plan.summary['quarantine']}")
click.echo(f" No-op (skipped): {execution_plan.summary['no-op']}")
# Count conflicts
conflicts = sum(1 for op in execution_plan.operations if op.has_conflict)
if conflicts > 0:
click.echo()
click.echo(f" ⚠️ Conflicts detected: {conflicts}")
click.echo(" Review the plan file for details on conflicting operations.")
# Save execution plan to JSON
click.echo()
click.echo(f"Saving execution plan to: {output}")
# Ensure output directory exists
output.parent.mkdir(parents=True, exist_ok=True)
save_plan(execution_plan, output)
click.echo(f"Execution plan saved successfully!")
click.echo()
click.echo("Next steps:")
click.echo(f" 1. Review the plan: {output}")
click.echo(f" 2. Edit the plan if needed (it's JSON)")
click.echo(f" 3. Dry-run: vlm execute --plan {output}")
click.echo(f" 4. Execute: vlm execute --plan {output} --confirm")
logger.info(f"Plan generated: {execution_plan.summary['total']} operations, {conflicts} conflicts, saved to {output}")
from vlm.commands.plan import plan_cmd
plan_cmd(ctx, input, output)
except FileNotFoundError:
click.echo(f"Error: Input file not found: {input}", err=True)
logger.error(f"Input file not found: {input}")
ctx.logger.error(f"Input file not found: {input}")
sys.exit(1)
except json.JSONDecodeError as e:
click.echo(f"Error: Failed to parse JSON file: {e}", err=True)
logger.error(f"JSON parsing failed: {e}", exc_info=True)
ctx.logger.error(f"JSON parsing failed: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error during plan generation: {e}", err=True)
logger.error(f"Plan generation failed: {e}", exc_info=True)
ctx.logger.error(f"Plan generation failed: {e}", exc_info=True)
sys.exit(1)
@@ -1266,7 +848,7 @@ def quarantine_list(ctx: CLIContext, category: Optional[str]):
click.echo(f" Category: {entry.category}")
click.echo(f" Original: {entry.original_path}")
click.echo(f" Quarantine: {entry.quarantine_path}")
click.echo(f" Size: {_format_size(entry.size_bytes)}")
click.echo(f" Size: {format_size(entry.size_bytes)}")
click.echo(f" Quarantined: {entry.quarantined_at.strftime('%Y-%m-%d %H:%M:%S')}")
if entry.reason:
click.echo(f" Reason: {entry.reason}")
@@ -2246,7 +1828,8 @@ def config_cmd(ctx: CLIContext):
default=default_config_path,
help='Path where configuration file should be created'
)
def config_init(path: Path):
@pass_context
def config_init(ctx: CLIContext, path: Path):
"""Initialize configuration file with defaults."""
try:
if path.exists():
src/vlm/commands/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""CLI command implementations.
Each module provides *_cmd(ctx, ...) functions that are invoked by cli.py
after Click parses options and passes context.
"""
src/vlm/commands/analyze.py
+116
View File
@@ -0,0 +1,116 @@
"""Analyze command implementation."""
import json
from pathlib import Path
from typing import Optional
import click
from vlm.analysis import analyze_series_completeness, detect_duplicates
from vlm.context import CLIContext
from vlm.io import identities_to_analysis_input, load_identities_json, load_inventory_csv
from vlm.models import MovieIdentity, SeriesIdentity
from vlm.utils import utc_now
def analyze_cmd(
ctx: CLIContext,
input: Path,
output: Path,
inventory: Optional[Path],
) -> None:
"""Run analysis: completeness and duplicate detection."""
config = ctx.config
logger = ctx.logger
click.echo(f"Analyzing identities from: {input}")
if inventory:
click.echo(f"Merging metadata from inventory: {inventory}")
click.echo()
identities_data = load_identities_json(input)
movies_data = identities_data.get("movies", [])
series_data = identities_data.get("series", [])
click.echo(f"Loaded {len(movies_data)} movies and {len(series_data)} series")
click.echo()
inventory_files = load_inventory_csv(inventory) if inventory else None
movie_identities, series_identities, video_files = identities_to_analysis_input(
identities_data, inventory_files=inventory_files
)
click.echo("Analyzing series completeness...")
completeness_results = analyze_series_completeness(series_identities)
click.echo("Detecting duplicates...")
n_movies = len(movie_identities)
identity_file_pairs = (
list(zip(movie_identities, video_files[:n_movies]))
+ list(zip(series_identities, video_files[n_movies:]))
)
duplicate_groups = detect_duplicates(identity_file_pairs)
click.echo()
click.echo("Analysis complete!")
click.echo()
click.echo("Results:")
click.echo(f" Series with episode gaps: {len(completeness_results)}")
if completeness_results:
total_missing = sum(len(c.episodes_missing) for c in completeness_results)
click.echo(f" - Total missing episodes: {total_missing}")
click.echo(f" Duplicate groups found: {len(duplicate_groups)}")
if duplicate_groups:
total_duplicates = sum(len(g.files) for g in duplicate_groups)
click.echo(f" - Total duplicate files: {total_duplicates}")
click.echo()
click.echo(f"Saving analysis results to: {output}")
output.parent.mkdir(parents=True, exist_ok=True)
generation_timestamp = utc_now().strftime("%Y-%m-%dT%H:%M:%S")
completeness_list = [
{
"series_title": c.series_title,
"season": c.season,
"episodes_found": c.episodes_found,
"episodes_missing": c.episodes_missing,
}
for c in completeness_results
]
duplicates_list = []
for d in duplicate_groups:
if isinstance(d.identity, MovieIdentity):
identity_info = {"type": "movie", "title": d.identity.title, "year": d.identity.year}
else:
identity_info = {
"type": "series",
"title": d.identity.title,
"season": d.identity.season,
"episodes": d.identity.episodes,
}
duplicates_list.append(
{
"identity": identity_info,
"files": [str(f.path) for f in d.files],
"quality_comparison": d.quality_comparison,
}
)
analysis_data = {
"metadata": {
"generated": generation_timestamp,
"source_identities": str(input),
"total_movies": len(movies_data),
"total_series": len(series_data),
},
"completeness": completeness_list,
"duplicates": duplicates_list,
}
with open(output, "w", encoding="utf-8") as jsonfile:
json.dump(analysis_data, jsonfile, indent=2, ensure_ascii=False)
click.echo("Analysis results saved successfully!")
logger.info(
f"Analysis completed: {len(completeness_results)} incomplete series, "
f"{len(duplicate_groups)} duplicate groups, saved to {output}"
)
src/vlm/commands/plan.py
+69
View File
@@ -0,0 +1,69 @@
"""Plan command implementation."""
from pathlib import Path
import click
from vlm.context import CLIContext
from vlm.io import identities_to_plan_input, load_identities_json
from vlm.planner import generate_plan, save_plan
def plan_cmd(ctx: CLIContext, input: Path, output: Path) -> None:
"""Generate execution plan from identities."""
config = ctx.config
logger = ctx.logger
click.echo(f"Generating execution plan from: {input}")
click.echo()
identities_data = load_identities_json(input)
movies_data = identities_data.get("movies", [])
series_data = identities_data.get("series", [])
anime_data = identities_data.get("anime", [])
other_data = identities_data.get("other", [])
click.echo(
f"Loaded {len(movies_data)} movies, {len(series_data)} series, "
f"{len(anime_data)} anime, {len(other_data)} other"
)
click.echo()
identities_list = identities_to_plan_input(identities_data)
click.echo("Generating execution plan...")
execution_plan = generate_plan(identities_list, config)
click.echo()
click.echo("Plan generation complete!")
click.echo()
click.echo("Operation summary:")
click.echo(f" Total operations: {execution_plan.summary['total']}")
click.echo(f" Move operations: {execution_plan.summary['move']}")
click.echo(f" Rename operations: {execution_plan.summary['rename']}")
click.echo(f" Quarantine operations: {execution_plan.summary['quarantine']}")
click.echo(f" No-op (skipped): {execution_plan.summary['no-op']}")
conflicts = sum(1 for op in execution_plan.operations if op.has_conflict)
if conflicts > 0:
click.echo()
click.echo(f" ⚠️ Conflicts detected: {conflicts}")
click.echo(" Review the plan file for details on conflicting operations.")
click.echo()
click.echo(f"Saving execution plan to: {output}")
output.parent.mkdir(parents=True, exist_ok=True)
save_plan(execution_plan, output)
click.echo("Execution plan saved successfully!")
click.echo()
click.echo("Next steps:")
click.echo(f" 1. Review the plan: {output}")
click.echo(" 2. Edit the plan if needed (it's JSON)")
click.echo(f" 3. Dry-run: vlm execute --plan {output}")
click.echo(f" 4. Execute: vlm execute --plan {output} --confirm")
logger.info(
f"Plan generated: {execution_plan.summary['total']} operations, "
f"{conflicts} conflicts, saved to {output}"
)
src/vlm/commands/scan.py
+97
View File
@@ -0,0 +1,97 @@
"""Scan command implementation."""
import sys
from pathlib import Path
from typing import Optional
import click
from vlm.context import CLIContext
from vlm.scanner import load_inventory_csv, save_inventory_csv, scan_library
from vlm.utils import format_size
def scan_cmd(
ctx: CLIContext,
output: Path,
metadata: bool,
reuse_from: Optional[Path],
force_refresh_metadata: bool,
) -> None:
"""Run scan: discover video files and save inventory."""
config = ctx.config
logger = ctx.logger
click.echo(f"Scanning library at: {config.library_root}")
click.echo("This may take a while for large libraries...")
click.echo()
progress_state: dict[str, Optional[click.ProgressBar]] = {"bar": None}
progress_position = {"current": 0}
def _scan_progress(processed: int, total: int) -> None:
if total <= 0:
return
if progress_state["bar"] is None:
bar = click.progressbar(
length=total,
label="Scanning files",
show_pos=True,
)
progress_state["bar"] = bar.__enter__()
step = processed - progress_position["current"]
if step > 0 and progress_state["bar"] is not None:
progress_state["bar"].update(step)
progress_position["current"] = processed
metadata_cache = None
cache_source = None
if not force_refresh_metadata:
cache_source = reuse_from if reuse_from is not None else (output if output.exists() else None)
if metadata and force_refresh_metadata:
click.echo("Forcing metadata refresh for all files (cache disabled).")
click.echo()
if metadata and cache_source is not None:
click.echo(f"Loading metadata cache from: {cache_source}")
try:
cached_files = load_inventory_csv(cache_source)
metadata_cache = {str(vf.path): vf for vf in cached_files}
click.echo(f"Loaded metadata cache entries: {len(metadata_cache)}")
click.echo()
except Exception as e:
click.echo(f"Warning: could not load metadata cache: {e}")
click.echo("Continuing without cache.")
click.echo()
try:
video_files = scan_library(
config.library_root,
config,
progress_callback=_scan_progress,
include_video_metadata=metadata,
metadata_cache=metadata_cache,
)
finally:
if progress_state["bar"] is not None:
progress_state["bar"].__exit__(None, None, None)
click.echo()
click.echo("Scan complete!")
click.echo(f" Total files found: {len(video_files)}")
categories = {}
total_size = 0
for vf in video_files:
categories[vf.category] = categories.get(vf.category, 0) + 1
total_size += vf.size_bytes
click.echo(f" Total size: {format_size(total_size)}")
click.echo()
click.echo("Files by category:")
for category in sorted(categories.keys()):
click.echo(f" {category}: {categories[category]}")
click.echo()
click.echo(f"Saving inventory to: {output}")
save_inventory_csv(video_files, output, config.library_root)
click.echo("Inventory saved successfully!")
logger.info(f"Scan completed: {len(video_files)} files found, saved to {output}")
src/vlm/config.py
+53 -26
View File
@@ -37,6 +37,10 @@ class Config:
translation_mode: str = "bidirectional"
translation_fallback_machine: bool = True
tmdb_api_key: Optional[str] = None
tmdb_bearer_token: Optional[str] = None
tmdb_language: str = "zh-CN"
tmdb_region: Optional[str] = None
tmdb_include_adult: bool = False
openai_api_key: Optional[str] = None
reputation_min_votes: int = 50
reputation_low_score_threshold: float = 6.0
@@ -82,11 +86,14 @@ def load_config(path: Path) -> Config:
"anime": ["anime"]
})
enrichment = data.get("enrichment", {})
enrichment = data.get("enrichment")
if enrichment is None:
enrichment = data.get("enrich", {})
translation = enrichment.get("translation", {})
api_keys = enrichment.get("api_keys", {})
reputation = enrichment.get("reputation", {})
naming = enrichment.get("naming", {})
tmdb = enrichment.get("tmdb", {})
return Config(
library_root=library_root,
@@ -110,6 +117,10 @@ def load_config(path: Path) -> Config:
translation_mode=translation.get("mode", "bidirectional"),
translation_fallback_machine=translation.get("fallback_machine", True),
tmdb_api_key=api_keys.get("tmdb"),
tmdb_bearer_token=api_keys.get("tmdb_bearer"),
tmdb_language=tmdb.get("language", "zh-CN"),
tmdb_region=tmdb.get("region"),
tmdb_include_adult=tmdb.get("include_adult", False),
openai_api_key=api_keys.get("openai"),
reputation_min_votes=reputation.get("min_votes", 50),
reputation_low_score_threshold=reputation.get("low_score_threshold", 6.0),
@@ -125,6 +136,38 @@ def create_default_config(path: Path) -> Config:
video_extensions=[".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".m4v"],
)
enrichment_content = {
"enabled": default_config.enrichment_enabled,
"incremental": default_config.enrichment_incremental,
"refresh_mode": default_config.enrichment_refresh_mode,
"providers": default_config.enrichment_providers,
"cache_db": str(default_config.enrichment_cache_db),
"max_concurrency": default_config.enrichment_max_concurrency,
"min_match_score": default_config.enrichment_min_match_score,
"translation": {
"mode": default_config.translation_mode,
"fallback_machine": default_config.translation_fallback_machine,
},
"api_keys": {
"tmdb": default_config.tmdb_api_key,
"tmdb_bearer": default_config.tmdb_bearer_token,
"openai": default_config.openai_api_key,
},
"tmdb": {
"language": default_config.tmdb_language,
"region": default_config.tmdb_region,
"include_adult": default_config.tmdb_include_adult,
},
"reputation": {
"min_votes": default_config.reputation_min_votes,
"low_score_threshold": default_config.reputation_low_score_threshold,
"policy": default_config.reputation_policy,
},
"naming": {
"title_format": default_config.naming_title_format,
},
}
yaml_content = {
"library_root": str(default_config.library_root),
"video_extensions": default_config.video_extensions,
@@ -137,31 +180,9 @@ def create_default_config(path: Path) -> Config:
"quarantine_dir": default_config.quarantine_dir,
"log_level": default_config.log_level,
"categories": default_config.categories,
"enrichment": {
"enabled": default_config.enrichment_enabled,
"incremental": default_config.enrichment_incremental,
"refresh_mode": default_config.enrichment_refresh_mode,
"providers": default_config.enrichment_providers,
"cache_db": str(default_config.enrichment_cache_db),
"max_concurrency": default_config.enrichment_max_concurrency,
"min_match_score": default_config.enrichment_min_match_score,
"translation": {
"mode": default_config.translation_mode,
"fallback_machine": default_config.translation_fallback_machine,
},
"api_keys": {
"tmdb": default_config.tmdb_api_key,
"openai": default_config.openai_api_key,
},
"reputation": {
"min_votes": default_config.reputation_min_votes,
"low_score_threshold": default_config.reputation_low_score_threshold,
"policy": default_config.reputation_policy,
},
"naming": {
"title_format": default_config.naming_title_format,
},
},
"enrichment": enrichment_content,
# Backward-compatible alias for users who prefer `enrich`.
"enrich": enrichment_content,
}
path.parent.mkdir(parents=True, exist_ok=True)
@@ -287,5 +308,11 @@ def validate_config(config: Config) -> list[str]:
errors.append("reputation_min_votes must be >= 0")
if not (0.0 <= config.reputation_low_score_threshold <= 10.0):
errors.append("reputation_low_score_threshold must be between 0.0 and 10.0")
if not isinstance(config.tmdb_language, str) or not config.tmdb_language.strip():
errors.append("tmdb_language must be a non-empty string")
if config.tmdb_region is not None and not isinstance(config.tmdb_region, str):
errors.append("tmdb_region must be a string when set")
if not isinstance(config.tmdb_include_adult, bool):
errors.append("tmdb_include_adult must be a boolean")
return errors
src/vlm/context.py
+16
View File
@@ -0,0 +1,16 @@
"""CLI context shared by cli.py and command modules."""
import click
from vlm.config import Config
class CLIContext:
"""Context object to pass configuration and logger between commands."""
def __init__(self, config: Config, logger):
self.config = config
self.logger = logger
pass_context = click.make_pass_decorator(CLIContext)
src/vlm/enrichment.py
+112 -26
View File
@@ -12,7 +12,7 @@ from urllib.request import urlopen, Request
from vlm.cache import EnrichmentCache
from vlm.config import Config
from vlm.providers import ProviderResult, TMDBProvider
from vlm.providers import ProviderResult, TMDBAuthError, TMDBProvider, TMDBProviderError
from vlm.parser import normalize_title
RefreshMode = str
@@ -59,6 +59,7 @@ def enrich_identities_data(
"failed": 0,
"api_calls": 0,
"failed_items": [],
"skip_reasons": {},
}
refresh_all = refresh_mode == "refresh_all"
@@ -69,6 +70,7 @@ def enrich_identities_data(
title = record.get("title") or _fallback_title_from_filename(record.get("filename"))
if not title:
stats["skipped"] = int(stats["skipped"]) + 1
_increment_skip_reason(stats, "invalid_input")
stats["processed"] = int(stats["processed"]) + 1
_emit_progress(stats, progress_callback)
continue
@@ -92,7 +94,7 @@ def enrich_identities_data(
_emit_progress(stats, progress_callback)
continue
payload, api_calls, failures = _enrich_record(
payload, api_calls, failures, skip_reason = _enrich_record(
record,
media_type,
providers,
@@ -100,26 +102,12 @@ def enrich_identities_data(
request_timeout=request_timeout,
retries=retries,
)
stats["api_calls"] = int(stats["api_calls"]) + api_calls
if failures:
failed_items = stats["failed_items"]
assert isinstance(failed_items, list)
failed_items.extend(failures)
stats["failed"] = int(stats["failed"]) + len(failures)
_apply_payload(record, payload)
cache.put_identity(identity_key, fingerprint, payload)
if payload.get("enriched"):
stats["enriched"] = int(stats["enriched"]) + 1
else:
stats["skipped"] = int(stats["skipped"]) + 1
if record.get("needs_review"):
stats["needs_review"] = int(stats["needs_review"]) + 1
stats["processed"] = int(stats["processed"]) + 1
_emit_progress(stats, progress_callback)
_update_stats_after_enrich(
stats, payload, failures, api_calls, record, progress_callback, skip_reason
)
metadata = identities_data.setdefault("metadata", {})
metadata["enriched"] = True
@@ -154,6 +142,33 @@ def _emit_progress(stats: dict[str, int | list[dict[str, str]]], progress_callba
)
def _update_stats_after_enrich(
stats: dict[str, int | list[dict[str, str]]],
payload: dict,
failures: list[dict[str, str]],
api_calls: int,
record: dict,
progress_callback: Optional[ProgressCallback],
skip_reason: str,
) -> None:
"""Update stats and emit progress after enriching a single record."""
stats["api_calls"] = int(stats["api_calls"]) + api_calls
if failures:
failed_items = stats["failed_items"]
assert isinstance(failed_items, list)
failed_items.extend(failures)
stats["failed"] = int(stats["failed"]) + len(failures)
if payload.get("enriched"):
stats["enriched"] = int(stats["enriched"]) + 1
else:
stats["skipped"] = int(stats["skipped"]) + 1
_increment_skip_reason(stats, skip_reason or "no_match")
if record.get("needs_review"):
stats["needs_review"] = int(stats["needs_review"]) + 1
stats["processed"] = int(stats["processed"]) + 1
_emit_progress(stats, progress_callback)
def _build_providers(config: Config, *, request_timeout: int, retries: int) -> list:
providers = []
unsupported: list[str] = []
@@ -163,6 +178,10 @@ def _build_providers(config: Config, *, request_timeout: int, retries: int) -> l
providers.append(
TMDBProvider(
config.tmdb_api_key,
bearer_token=config.tmdb_bearer_token,
language=config.tmdb_language,
region=config.tmdb_region,
include_adult=config.tmdb_include_adult,
timeout_seconds=request_timeout,
retries=retries,
min_interval_seconds=0.25,
@@ -187,37 +206,57 @@ def _enrich_record(
*,
request_timeout: int,
retries: int,
) -> tuple[dict, int, list[dict[str, str]]]:
) -> tuple[dict, int, list[dict[str, str]], str]:
title = record.get("title")
year = record.get("year") if media_type == "movie" else None
provider_results: list[ProviderResult] = []
failures: list[dict[str, str]] = []
api_calls = 0
configured_provider_count = 0
for provider in providers:
api_calls += 1
if not _provider_is_configured(provider, config):
continue
configured_provider_count += 1
try:
result = provider.enrich(title=title, media_type=media_type, year=year)
except TMDBAuthError as exc:
raise RuntimeError(str(exc)) from exc
except TMDBProviderError as exc:
failures.append(
{
"path": str(record.get("path", "")),
"title": str(title),
"provider": provider.name,
"reason": str(exc),
}
)
api_calls += provider.last_request_count
continue
except Exception as exc:
failures.append(
{
"path": str(record.get("path", "")),
"title": str(title),
"provider": getattr(provider, "name", "unknown"),
"provider": provider.name,
"reason": str(exc),
}
)
api_calls += provider.last_request_count
continue
api_calls += provider.last_request_count
if result:
provider_results.append(result)
merged = _merge_provider_results(provider_results)
# Optional AI fallback for missing translated titles.
if config.translation_fallback_machine:
if config.translation_fallback_machine and config.openai_api_key:
if not merged.get("title_zh"):
api_calls += 1
translated = _translate_with_openai(
title,
target_language="Chinese (Simplified)",
@@ -225,12 +264,12 @@ def _enrich_record(
timeout_seconds=request_timeout,
retries=retries,
)
api_calls += 1
if translated:
merged["title_zh"] = translated
merged["translation_source"] = merged.get("translation_source") or "openai"
if not merged.get("title_en"):
api_calls += 1
translated = _translate_with_openai(
title,
target_language="English",
@@ -238,7 +277,6 @@ def _enrich_record(
timeout_seconds=request_timeout,
retries=retries,
)
api_calls += 1
if translated:
merged["title_en"] = translated
merged["translation_source"] = merged.get("translation_source") or "openai"
@@ -266,7 +304,55 @@ def _enrich_record(
merged["enriched"] = bool(provider_results or merged.get("translation_source"))
merged["display_title"] = _build_display_title(record, merged, config)
return merged, api_calls, failures
skip_reason = _determine_skip_reason(
provider_results=provider_results,
failures=failures,
configured_provider_count=configured_provider_count,
api_calls=api_calls,
)
return merged, api_calls, failures, skip_reason
def _determine_skip_reason(
*,
provider_results: list[ProviderResult],
failures: list[dict[str, str]],
configured_provider_count: int,
api_calls: int,
) -> str:
if provider_results:
return ""
if configured_provider_count == 0 and api_calls == 0:
return "no_key"
if failures:
for failure in failures:
reason = str(failure.get("reason", "")).lower()
if "rate limit" in reason or "(429)" in reason:
return "rate_limited"
if "authentication failed" in reason or "(401/403)" in reason:
return "auth_error"
return "provider_error"
return "no_match"
def _increment_skip_reason(stats: dict[str, int | list[dict[str, str]]], reason: str) -> None:
if not reason:
return
current = stats.get("skip_reasons")
if not isinstance(current, dict):
current = {}
stats["skip_reasons"] = current
current[reason] = int(current.get(reason, 0)) + 1
def _provider_is_configured(provider: object, config: Config) -> bool:
provider_name = provider.name.lower()
if provider_name == "tmdb":
return bool(config.tmdb_bearer_token or config.tmdb_api_key)
return True
def _merge_provider_results(results: list[ProviderResult]) -> dict:
src/vlm/executor.py
+8 -7
View File
@@ -16,6 +16,7 @@ from uuid import uuid4
from .logging_config import get_logger, log_operation
from .models import ExecutionPlan, FileOperation, OperationResult, RollbackLog
from .utils import ensure_utc, utc_now
class ExecutionEngine:
@@ -86,7 +87,7 @@ class ExecutionEngine:
rollback_log = RollbackLog(
log_id=str(uuid4()),
execution_plan_id=plan.plan_id,
executed_at=datetime.now(),
executed_at=utc_now(),
operations=successful_operations
)
@@ -116,8 +117,8 @@ class ExecutionEngine:
Returns:
OperationResult with success status and any error message
"""
executed_at = datetime.now()
executed_at = utc_now()
# Handle no-op operations
if operation.operation_type == "no-op":
log_operation(
@@ -385,19 +386,19 @@ class ExecutionEngine:
conflict_reason=None
)
# Reconstruct OperationResult
# Reconstruct OperationResult (normalize naive datetime to UTC)
op_result = OperationResult(
operation=file_op,
success=op_data["success"],
error_message=op_data["error_message"],
executed_at=datetime.fromisoformat(op_data["executed_at"])
executed_at=ensure_utc(datetime.fromisoformat(op_data["executed_at"]))
)
operations.append(op_result)
rollback_log = RollbackLog(
log_id=log_data["log_id"],
execution_plan_id=log_data["execution_plan_id"],
executed_at=datetime.fromisoformat(log_data["executed_at"]),
executed_at=ensure_utc(datetime.fromisoformat(log_data["executed_at"])),
operations=operations
)
@@ -468,7 +469,7 @@ class ExecutionEngine:
OperationResult indicating success or failure of the rollback
"""
operation = original_result.operation
executed_at = datetime.now()
executed_at = utc_now()
# Skip no-op operations
if operation.operation_type == "no-op":
src/vlm/io.py
+188
View File
@@ -0,0 +1,188 @@
"""Unified I/O layer for inventory and identities data."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Union
from vlm.models import MovieIdentity, SeriesIdentity, VideoFile
from vlm.utils import utc_now
# Re-export scanner CSV functions so CLI and others use a single I/O entry point
from vlm.scanner import load_inventory_csv, save_inventory_csv
__all__ = [
"load_inventory_csv",
"save_inventory_csv",
"load_identities_json",
"save_identities_json",
"identities_to_plan_input",
"identities_to_analysis_input",
]
def load_identities_json(path: Path) -> dict:
"""Load identities from JSON file."""
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def save_identities_json(data: dict, path: Path) -> None:
"""Save identities dict to JSON file."""
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def _video_file_from_record(record: dict) -> VideoFile:
"""Build a minimal VideoFile from an identities record (no inventory metadata)."""
return VideoFile(
path=Path(record["path"]),
filename=record["filename"],
size_bytes=0,
modified_timestamp=utc_now(),
category=record["category"],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None,
)
def _movie_identity_from_record(m: dict) -> MovieIdentity:
"""Build MovieIdentity from identities JSON record."""
is_approved = m.get("review_status") == "approved"
return MovieIdentity(
title=m.get("display_title", m["title"]),
year=m.get("year"),
confidence=m["confidence"],
needs_review=(m["needs_review"] and not is_approved),
original_filename=m["filename"],
canonical_id=m.get("canonical_id"),
title_zh=m.get("title_zh"),
title_en=m.get("title_en"),
translation_source=m.get("translation_source"),
reputation_score=m.get("reputation_score"),
reputation_votes=m.get("reputation_votes"),
reputation_source=m.get("reputation_source"),
review_status=m.get("review_status", "pending"),
enrichment_confidence=m.get("enrichment_confidence"),
provider_metadata=m.get("provider_metadata", {}),
)
def _series_identity_from_record(s: dict) -> SeriesIdentity:
"""Build SeriesIdentity from identities JSON record."""
is_approved = s.get("review_status") == "approved"
return SeriesIdentity(
title=s.get("display_title", s["title"]),
season=s.get("season"),
episodes=s.get("episodes", []),
confidence=s["confidence"],
needs_review=(s["needs_review"] and not is_approved),
original_filename=s["filename"],
canonical_id=s.get("canonical_id"),
title_zh=s.get("title_zh"),
title_en=s.get("title_en"),
translation_source=s.get("translation_source"),
reputation_score=s.get("reputation_score"),
reputation_votes=s.get("reputation_votes"),
reputation_source=s.get("reputation_source"),
review_status=s.get("review_status", "pending"),
enrichment_confidence=s.get("enrichment_confidence"),
provider_metadata=s.get("provider_metadata", {}),
)
def identities_to_plan_input(
data: dict,
) -> list[tuple[VideoFile, Union[MovieIdentity, SeriesIdentity, None]]]:
"""Convert identities JSON dict to list of (VideoFile, Identity) for plan generator."""
result: list[tuple[VideoFile, Union[MovieIdentity, SeriesIdentity, None]]] = []
movies_data = data.get("movies", [])
series_data = data.get("series", [])
anime_data = data.get("anime", [])
other_data = data.get("other", [])
for m in movies_data:
result.append((_video_file_from_record(m), _movie_identity_from_record(m)))
for s in series_data:
result.append((_video_file_from_record(s), _series_identity_from_record(s)))
for a in anime_data:
result.append((_video_file_from_record(a), None))
for o in other_data:
result.append((_video_file_from_record(o), None))
return result
def identities_to_analysis_input(
data: dict,
inventory_files: list[VideoFile] | None = None,
) -> tuple[list[MovieIdentity], list[SeriesIdentity], list[VideoFile]]:
"""Convert identities JSON dict to analysis inputs; optionally merge inventory metadata by path."""
movies_data = data.get("movies", [])
series_data = data.get("series", [])
movie_identities = []
for m in movies_data:
movie_identities.append(
MovieIdentity(
title=m["title"],
year=m.get("year"),
confidence=m["confidence"],
needs_review=m["needs_review"],
original_filename=m["filename"],
)
)
series_identities = []
for s in series_data:
series_identities.append(
SeriesIdentity(
title=s["title"],
season=s.get("season"),
episodes=s.get("episodes", []),
confidence=s["confidence"],
needs_review=s["needs_review"],
original_filename=s["filename"],
)
)
video_files: list[VideoFile] = []
path_to_inventory: dict[str, VideoFile] = {}
if inventory_files:
path_to_inventory = {str(vf.path): vf for vf in inventory_files}
for m in movies_data:
vf = _video_file_from_record(m)
if path_to_inventory:
inv = path_to_inventory.get(str(vf.path))
if inv:
vf = VideoFile(
path=inv.path,
filename=inv.filename,
size_bytes=inv.size_bytes,
modified_timestamp=inv.modified_timestamp,
category=inv.category,
resolution=inv.resolution,
codec=inv.codec,
duration_seconds=inv.duration_seconds,
bitrate_kbps=inv.bitrate_kbps,
)
video_files.append(vf)
for s in series_data:
vf = _video_file_from_record(s)
if path_to_inventory:
inv = path_to_inventory.get(str(vf.path))
if inv:
vf = VideoFile(
path=inv.path,
filename=inv.filename,
size_bytes=inv.size_bytes,
modified_timestamp=inv.modified_timestamp,
category=inv.category,
resolution=inv.resolution,
codec=inv.codec,
duration_seconds=inv.duration_seconds,
bitrate_kbps=inv.bitrate_kbps,
)
video_files.append(vf)
return movie_identities, series_identities, video_files
src/vlm/models.py
+11 -2
View File
@@ -41,7 +41,12 @@ class VideoFile:
@dataclass
class MovieIdentity:
"""Represents the parsed identity of a movie file.
review_status (pending/approved/rejected) and needs_review overlap in meaning:
needs_review is True when (review_status == 'pending') and parsing or
enrichment indicates the record should be reviewed; once approved, needs_review
is typically False.
Attributes:
title: Extracted movie title (normalized)
year: Extracted release year (None if not found)
@@ -69,7 +74,11 @@ class MovieIdentity:
@dataclass
class SeriesIdentity:
"""Represents the parsed identity of a TV series episode file.
review_status (pending/approved/rejected) and needs_review overlap in meaning:
needs_review is True when (review_status == 'pending') and parsing or
enrichment indicates the record should be reviewed.
Attributes:
title: Extracted series title (normalized)
season: Extracted season number (None if not found)
src/vlm/parser.py
+23 -6
View File
@@ -9,6 +9,11 @@ from typing import Optional
from vlm.models import MovieIdentity, SeriesIdentity
# Default extensions used when extensions param is not provided (matches config default)
DEFAULT_VIDEO_EXTENSIONS = [
".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".m4v"
]
# Quality tags to remove from titles
QUALITY_TAGS = [
@@ -80,7 +85,10 @@ def normalize_title(title: str) -> str:
return title.strip()
def parse_movie(filename: str) -> MovieIdentity:
def parse_movie(
filename: str,
extensions: Optional[list[str]] = None,
) -> MovieIdentity:
"""Parse a movie filename to extract title and year.
Supports patterns:
@@ -91,14 +99,17 @@ def parse_movie(filename: str) -> MovieIdentity:
Args:
filename: Movie filename to parse
extensions: Video extensions to strip (default: DEFAULT_VIDEO_EXTENSIONS)
Returns:
MovieIdentity with extracted information
"""
if extensions is None:
extensions = DEFAULT_VIDEO_EXTENSIONS
# Remove file extension
name_without_ext = filename
for ext in ['.mp4', '.mkv', '.avi', '.mov', '.wmv', '.flv', '.webm', '.m4v']:
if name_without_ext.lower().endswith(ext):
for ext in extensions:
if name_without_ext.lower().endswith(ext.lower()):
name_without_ext = name_without_ext[:-len(ext)]
break
@@ -148,7 +159,10 @@ def parse_movie(filename: str) -> MovieIdentity:
)
def parse_series(filename: str) -> SeriesIdentity:
def parse_series(
filename: str,
extensions: Optional[list[str]] = None,
) -> SeriesIdentity:
"""Parse a series filename to extract title, season, and episode numbers.
Supports patterns:
@@ -160,14 +174,17 @@ def parse_series(filename: str) -> SeriesIdentity:
Args:
filename: Series filename to parse
extensions: Video extensions to strip (default: DEFAULT_VIDEO_EXTENSIONS)
Returns:
SeriesIdentity with extracted information
"""
if extensions is None:
extensions = DEFAULT_VIDEO_EXTENSIONS
# Remove file extension
name_without_ext = filename
for ext in ['.mp4', '.mkv', '.avi', '.mov', '.wmv', '.flv', '.webm', '.m4v']:
if name_without_ext.lower().endswith(ext):
for ext in extensions:
if name_without_ext.lower().endswith(ext.lower()):
name_without_ext = name_without_ext[:-len(ext)]
break
src/vlm/planner.py
+5 -3
View File
@@ -11,6 +11,7 @@ from pathlib import Path
from typing import Union
from vlm.config import Config
from vlm.utils import ensure_utc, utc_now
from vlm.models import (
ExecutionPlan,
FileOperation,
@@ -48,7 +49,7 @@ def generate_plan(
return ExecutionPlan(
plan_id=str(uuid.uuid4()),
created_at=datetime.now(),
created_at=utc_now(),
operations=operations,
summary=summary
)
@@ -391,10 +392,11 @@ def load_plan(input_path: Path) -> ExecutionPlan:
for op in plan_dict["operations"]
]
# Reconstruct ExecutionPlan
# Reconstruct ExecutionPlan (normalize naive datetime to UTC for backward compatibility)
created_at = ensure_utc(datetime.fromisoformat(plan_dict["created_at"]))
return ExecutionPlan(
plan_id=plan_dict["plan_id"],
created_at=datetime.fromisoformat(plan_dict["created_at"]),
created_at=created_at,
operations=operations,
summary=plan_dict["summary"]
)
src/vlm/providers/__init__.py
+3 -1
View File
@@ -1,10 +1,12 @@
"""Provider implementations for enrichment."""
from vlm.providers.base import EnrichmentProvider, ProviderResult
from vlm.providers.tmdb import TMDBProvider
from vlm.providers.tmdb import TMDBAuthError, TMDBProvider, TMDBProviderError
__all__ = [
"EnrichmentProvider",
"ProviderResult",
"TMDBProvider",
"TMDBAuthError",
"TMDBProviderError",
]
src/vlm/providers/base.py
+1
View File
@@ -26,6 +26,7 @@ class EnrichmentProvider(Protocol):
"""Protocol for title/score providers."""
name: str
last_request_count: int # API request count for the last enrich() call (reset at start of each call)
def enrich(self, *, title: str, media_type: str, year: Optional[int] = None) -> Optional[ProviderResult]:
"""Return normalized metadata for a single identity."""
src/vlm/providers/tmdb.py
+170 -18
View File
@@ -3,14 +3,25 @@
from __future__ import annotations
import json
import re
import time
from typing import Optional
from difflib import SequenceMatcher
from typing import Any, Optional
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import urlopen, Request
from urllib.request import Request, urlopen
from vlm.providers.base import ProviderResult
class TMDBAuthError(RuntimeError):
"""Raised when TMDB credentials are invalid."""
class TMDBProviderError(RuntimeError):
"""Raised for TMDB errors that should be reported as provider failures."""
class TMDBProvider:
"""Fetch translations and reputation data from TMDB."""
@@ -19,48 +30,66 @@ class TMDBProvider:
def __init__(
self,
api_key: Optional[str],
*,
bearer_token: Optional[str] = None,
language: str = "zh-CN",
region: Optional[str] = None,
include_adult: bool = False,
timeout_seconds: int = 6,
retries: int = 2,
min_interval_seconds: float = 0.25,
backoff_base_seconds: float = 0.5,
backoff_max_seconds: float = 4.0,
) -> None:
self.api_key = api_key
self.bearer_token = bearer_token
self.language = language
self.region = region
self.include_adult = include_adult
self.base_url = "https://api.themoviedb.org/3"
self.timeout_seconds = timeout_seconds
self.retries = retries
self.min_interval_seconds = min_interval_seconds
self.backoff_base_seconds = backoff_base_seconds
self.backoff_max_seconds = backoff_max_seconds
self._last_request_at = 0.0
self.last_request_count = 0
def enrich(self, *, title: str, media_type: str, year: Optional[int] = None) -> Optional[ProviderResult]:
if not self.api_key:
self.last_request_count = 0
if not (self.bearer_token or self.api_key):
return None
search_type = "tv" if media_type in {"series", "anime", "tv"} else "movie"
query_params = {
"api_key": self.api_key,
query_params: dict[str, Any] = {
"query": title,
"language": self.language,
"include_adult": str(self.include_adult).lower(),
}
if self.region:
query_params["region"] = self.region
if year and search_type == "movie":
query_params["year"] = year
search_data = self._get_json(f"{self.base_url}/search/{search_type}", query_params)
search_data = self._get_json(f"/search/{search_type}", query_params)
if not search_data:
return None
results = search_data.get("results", [])
if not results:
if not isinstance(results, list) or not results:
return None
candidate, match_score = self._pick_best_candidate(results, title, year)
if not candidate:
return None
candidate = results[0]
tmdb_id = candidate.get("id")
if tmdb_id is None:
return None
details = self._get_json(
f"{self.base_url}/{search_type}/{tmdb_id}",
{"api_key": self.api_key, "language": self.language},
f"/{search_type}/{tmdb_id}",
{"language": self.language},
)
if not details:
details = candidate
@@ -79,10 +108,78 @@ class TMDBProvider:
reputation_score=float(vote_average) if vote_average is not None else None,
reputation_votes=int(vote_count) if vote_count is not None else None,
reputation_source=self.name,
match_score=float(candidate.get("popularity", 0.0)) if candidate.get("popularity") is not None else None,
match_score=round(match_score, 3),
raw_metadata={"media_type": search_type, "id": str(tmdb_id)},
)
def _pick_best_candidate(
self,
results: list[dict[str, Any]],
query_title: str,
query_year: Optional[int],
) -> tuple[Optional[dict[str, Any]], float]:
query_norm = self._normalize_title(query_title)
best_candidate: Optional[dict[str, Any]] = None
best_score = -1.0
for result in results:
candidates = [
result.get("title"),
result.get("name"),
result.get("original_title"),
result.get("original_name"),
]
title_score = 0.0
for candidate_title in candidates:
if not isinstance(candidate_title, str) or not candidate_title.strip():
continue
candidate_norm = self._normalize_title(candidate_title)
if not candidate_norm:
continue
ratio = SequenceMatcher(None, query_norm, candidate_norm).ratio()
if ratio > title_score:
title_score = ratio
year_bonus = 0.0
if query_year is not None:
release = result.get("release_date") or result.get("first_air_date")
candidate_year = self._extract_year(release)
if candidate_year is None:
year_bonus = -0.1
else:
delta = abs(candidate_year - query_year)
if delta == 0:
year_bonus = 0.2
elif delta == 1:
year_bonus = 0.1
else:
year_bonus = -0.2
popularity = result.get("popularity")
popularity_bonus = 0.0
if isinstance(popularity, (int, float)):
popularity_bonus = min(float(popularity) / 1000.0, 0.1)
total_score = title_score + year_bonus + popularity_bonus
if total_score > best_score:
best_score = total_score
best_candidate = result
return best_candidate, max(best_score, 0.0)
def _normalize_title(self, text: str) -> str:
lowered = text.lower().strip()
stripped = re.sub(r"[^\w\s]", " ", lowered)
return " ".join(stripped.split())
def _extract_year(self, date_text: Any) -> Optional[int]:
if not isinstance(date_text, str) or len(date_text) < 4:
return None
try:
return int(date_text[:4])
except (ValueError, TypeError):
return None
def _wait_for_rate_limit(self) -> None:
if self.min_interval_seconds <= 0:
return
@@ -91,18 +188,73 @@ class TMDBProvider:
if elapsed < self.min_interval_seconds:
time.sleep(self.min_interval_seconds - elapsed)
def _get_json(self, url: str, params: dict) -> Optional[dict]:
full_url = f"{url}?{urlencode(params)}"
request = Request(full_url, headers={"Accept": "application/json"})
def _sleep_backoff(self, attempt: int, retry_after: Optional[float] = None) -> None:
if retry_after is not None and retry_after > 0:
time.sleep(min(retry_after, self.backoff_max_seconds))
return
delay = min(self.backoff_base_seconds * (2 ** attempt), self.backoff_max_seconds)
time.sleep(delay)
for _ in range(max(self.retries + 1, 1)):
def _get_json(self, path: str, params: dict[str, Any]) -> Optional[dict]:
request_params = dict(params)
if not self.bearer_token and self.api_key:
request_params["api_key"] = self.api_key
full_url = f"{self.base_url}{path}?{urlencode(request_params)}"
headers = {"Accept": "application/json"}
if self.bearer_token:
headers["Authorization"] = f"Bearer {self.bearer_token}"
for attempt in range(max(self.retries + 1, 1)):
self._wait_for_rate_limit()
self.last_request_count += 1
request = Request(full_url, headers=headers)
try:
with urlopen(request, timeout=self.timeout_seconds) as response:
payload = response.read().decode("utf-8")
self._last_request_at = time.monotonic()
return json.loads(payload)
except Exception:
parsed = json.loads(payload)
if isinstance(parsed, dict):
return parsed
return None
except HTTPError as exc:
self._last_request_at = time.monotonic()
continue
code = exc.code
if code in (401, 403):
raise TMDBAuthError(
"TMDB authentication failed (401/403). "
"Configure enrichment.api_keys.tmdb_bearer or enrichment.api_keys.tmdb."
) from exc
if code == 404:
return None
if code == 429:
if attempt < self.retries:
retry_after = None
try:
retry_after_header = exc.headers.get("Retry-After")
retry_after = float(retry_after_header) if retry_after_header else None
except Exception:
retry_after = None
self._sleep_backoff(attempt, retry_after=retry_after)
continue
raise TMDBProviderError("TMDB rate limit exceeded (429)") from exc
if 500 <= code < 600 and attempt < self.retries:
self._sleep_backoff(attempt)
continue
if 500 <= code < 600:
raise TMDBProviderError(f"TMDB server error ({code})") from exc
raise TMDBProviderError(f"TMDB request failed with HTTP {code}") from exc
except URLError as exc:
self._last_request_at = time.monotonic()
if attempt < self.retries:
self._sleep_backoff(attempt)
continue
raise TMDBProviderError(f"TMDB network error: {exc}") from exc
except Exception as exc:
self._last_request_at = time.monotonic()
if attempt < self.retries:
self._sleep_backoff(attempt)
continue
raise TMDBProviderError(f"TMDB unexpected error: {exc}") from exc
return None
src/vlm/quarantine.py
+3 -2
View File
@@ -16,6 +16,7 @@ from pathlib import Path
from typing import Optional
from .config import Config
from .utils import utc_now
from .logging_config import get_logger, log_operation
from .models import QuarantineEntry, QuarantineManifest, OperationResult, FileOperation
@@ -58,7 +59,7 @@ class QuarantineManager:
Raises:
ValueError: If file is in anime or other category (not supported in v1)
"""
executed_at = datetime.now()
executed_at = utc_now()
# Verify file exists
if not file_path.exists():
@@ -526,7 +527,7 @@ class QuarantineManager:
Returns:
OperationResult indicating success or failure
"""
executed_at = datetime.now()
executed_at = utc_now()
# Verify quarantine file exists
if not quarantine_path.exists():
src/vlm/state.py
+7 -6
View File
@@ -10,6 +10,7 @@ from pathlib import Path
from typing import Optional
from vlm.models import FileState, StateStore
from vlm.utils import ensure_utc, utc_now
# Valid status values
@@ -32,20 +33,20 @@ def load_state(path: Path) -> StateStore:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
# Parse states dictionary
# Parse states dictionary (normalize naive datetime to UTC)
states = {}
for file_path_str, state_data in data.get('states', {}).items():
states[file_path_str] = FileState(
file_path=Path(state_data['file_path']),
status=state_data['status'],
reason=state_data.get('reason'),
updated_at=datetime.fromisoformat(state_data['updated_at'])
updated_at=ensure_utc(datetime.fromisoformat(state_data['updated_at']))
)
return StateStore(
states=states,
version=data.get('version', '1.0'),
last_updated=datetime.fromisoformat(data['last_updated'])
last_updated=ensure_utc(datetime.fromisoformat(data['last_updated']))
)
@@ -101,7 +102,7 @@ class StateManager:
self.store = StateStore(
states={},
version='1.0',
last_updated=datetime.now()
last_updated=utc_now()
)
def get_file_state(self, file_path: Path) -> Optional[FileState]:
@@ -136,7 +137,7 @@ class StateManager:
)
file_path_str = str(file_path)
now = datetime.now()
now = utc_now()
self.store.states[file_path_str] = FileState(
file_path=file_path,
@@ -170,7 +171,7 @@ class StateManager:
file_path_str = str(file_path)
if file_path_str in self.store.states:
del self.store.states[file_path_str]
self.store.last_updated = datetime.now()
self.store.last_updated = utc_now()
def save(self) -> None:
"""Save the current state store to disk."""
src/vlm/utils.py
+24
View File
@@ -0,0 +1,24 @@
"""Shared utilities for Video Library Manager."""
from datetime import datetime, timezone
def utc_now() -> datetime:
"""Return current UTC time (timezone-aware)."""
return datetime.now(timezone.utc)
def ensure_utc(dt: datetime) -> datetime:
"""Ensure datetime is timezone-aware UTC (for backward compatibility with naive ISO strings)."""
if dt.tzinfo is None:
return dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc)
def format_size(size_bytes: int) -> str:
"""Format file size in human-readable format (e.g. 1.5 GB, 234.2 MB)."""
for unit in ["B", "KB", "MB", "GB", "TB"]:
if size_bytes < 1024.0:
return f"{size_bytes:.1f} {unit}"
size_bytes /= 1024.0
return f"{size_bytes:.1f} PB"