windy/dl-organizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enhance project structure and add new files for enrichment and analysis

Browse Source 
- Updated AGENTS.md to reflect changes in CLI commands and module organization, including the addition of an enrichment step and new functional modules.
- Introduced analysis.json, identities.json, inventory.csv, and plan.json to support enriched metadata and execution planning.
- Added CODE_IMPROVEMENTS.md to document identified code issues and proposed solutions for future enhancements.
- Updated README.md to include new enrichment features and configuration options.
- Removed unused dependency on ffmpeg-python from pyproject.toml.

These changes improve the overall functionality and maintainability of the Video Library Manager project.
This commit is contained in:
windyboy
2026-02-10 16:56:17 +08:00
parent f0c951ad7f
commit dcd87754cf
39 changed files with 145509 additions and 642 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vlm/cli.py
+52 -469
View File
@@ -5,6 +5,7 @@ It implements global options (--config, --log-level) and error handling.
"""
import sys
import traceback
from pathlib import Path
from typing import Optional
@@ -12,7 +13,9 @@ import click
import yaml
from vlm.config import Config, load_config, create_default_config, validate_config
from vlm.context import CLIContext, pass_context
from vlm.logging_config import setup_logging, get_logger
from vlm.utils import format_size
def default_config_path() -> Path:
@@ -20,17 +23,6 @@ def default_config_path() -> Path:
return Path.home() / ".vlm" / "config.yaml"
class CLIContext:
"""Context object to pass configuration and logger between commands."""
def __init__(self, config: Config, logger):
self.config = config
self.logger = logger
pass_context = click.make_pass_decorator(CLIContext)
@click.group()
@click.option(
'--config',
@@ -105,6 +97,7 @@ def main(ctx, config: Path, log_level: Optional[str]):
ctx.obj = CLIContext(config=cfg, logger=logger)
except Exception as e:
traceback.print_exc(file=sys.stderr)
click.echo(f"Error initializing VLM: {e}", err=True)
sys.exit(1)
@@ -154,118 +147,15 @@ def scan(
vlm scan --reuse-from old.csv # Reuse prior metadata cache
vlm scan --force-refresh-metadata # Re-run ffprobe for all files
"""
from vlm.scanner import scan_library, save_inventory_csv, load_inventory_csv
config = ctx.config
logger = ctx.logger
try:
# Display scan start message
click.echo(f"Scanning library at: {config.library_root}")
click.echo("This may take a while for large libraries...")
click.echo()
progress_state: dict[str, Optional[click.ProgressBar]] = {"bar": None}
progress_position = {"current": 0}
def _scan_progress(processed: int, total: int) -> None:
if total <= 0:
return
if progress_state["bar"] is None:
bar = click.progressbar(
length=total,
label="Scanning files",
show_pos=True,
)
progress_state["bar"] = bar.__enter__()
step = processed - progress_position["current"]
if step > 0 and progress_state["bar"] is not None:
progress_state["bar"].update(step)
progress_position["current"] = processed
metadata_cache = None
cache_source = None
if not force_refresh_metadata:
cache_source = reuse_from if reuse_from is not None else (output if output.exists() else None)
if metadata and force_refresh_metadata:
click.echo("Forcing metadata refresh for all files (cache disabled).")
click.echo()
if metadata and cache_source is not None:
click.echo(f"Loading metadata cache from: {cache_source}")
try:
cached_files = load_inventory_csv(cache_source)
metadata_cache = {str(vf.path): vf for vf in cached_files}
click.echo(f"Loaded metadata cache entries: {len(metadata_cache)}")
click.echo()
except Exception as e:
click.echo(f"Warning: could not load metadata cache: {e}")
click.echo("Continuing without cache.")
click.echo()
# Perform the scan
try:
video_files = scan_library(
config.library_root,
config,
progress_callback=_scan_progress,
include_video_metadata=metadata,
metadata_cache=metadata_cache
)
finally:
if progress_state["bar"] is not None:
progress_state["bar"].__exit__(None, None, None)
click.echo()
# Display summary
click.echo(f"Scan complete!")
click.echo(f" Total files found: {len(video_files)}")
# Count by category
categories = {}
total_size = 0
for vf in video_files:
categories[vf.category] = categories.get(vf.category, 0) + 1
total_size += vf.size_bytes
click.echo(f" Total size: {_format_size(total_size)}")
click.echo()
click.echo("Files by category:")
for category in sorted(categories.keys()):
click.echo(f" {category}: {categories[category]}")
# Save inventory to CSV
click.echo()
click.echo(f"Saving inventory to: {output}")
save_inventory_csv(video_files, output, config.library_root)
click.echo(f"Inventory saved successfully!")
logger.info(f"Scan completed: {len(video_files)} files found, saved to {output}")
from vlm.commands.scan import scan_cmd
scan_cmd(ctx, output, metadata, reuse_from, force_refresh_metadata)
except Exception as e:
click.echo(f"Error during scan: {e}", err=True)
logger.error(f"Scan failed: {e}", exc_info=True)
ctx.logger.error(f"Scan failed: {e}", exc_info=True)
sys.exit(1)
def _format_size(size_bytes: int) -> str:
"""Format file size in human-readable format.
Args:
size_bytes: Size in bytes
Returns:
Formatted string (e.g., "1.5 GB", "234.2 MB")
"""
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
if size_bytes < 1024.0:
return f"{size_bytes:.1f} {unit}"
size_bytes /= 1024.0
return f"{size_bytes:.1f} PB"
@main.command()
@click.option(
'--input',
@@ -337,7 +227,7 @@ def parse(ctx: CLIContext, input: Path, output: Path):
category = vf['category']
if category == 'movie':
identity = parse_movie(filename)
identity = parse_movie(filename, extensions=config.video_extensions)
movie_identities.append({
'path': vf['path'],
'filename': filename,
@@ -349,7 +239,7 @@ def parse(ctx: CLIContext, input: Path, output: Path):
})
elif category == 'series':
identity = parse_series(filename)
identity = parse_series(filename, extensions=config.video_extensions)
series_identities.append({
'path': vf['path'],
'filename': filename,
@@ -543,12 +433,14 @@ def enrich(
progress_state: dict[str, Optional[click.ProgressBar]] = {"bar": None}
progress_position = {"current": 0}
progress_bucket = {"value": -1}
is_tty = bool(getattr(sys.stderr, "isatty", lambda: False)())
def _enrich_progress(processed: int, total_count: int, _metrics: dict[str, int]) -> None:
def _enrich_progress(processed: int, total_count: int, metrics: dict[str, int]) -> None:
if total_count <= 0:
return
if progress_state["bar"] is None:
if is_tty and progress_state["bar"] is None:
bar = click.progressbar(
length=total_count,
label="Enriching records",
@@ -561,6 +453,19 @@ def enrich(
progress_state["bar"].update(step)
progress_position["current"] = processed
if not is_tty:
percent = int(processed * 100 / total_count)
bucket = percent // 5
if bucket > progress_bucket["value"] or processed == total_count:
progress_bucket["value"] = bucket
click.echo(
"Progress: "
f"{processed}/{total_count} ({percent}%) "
f"api_calls={metrics.get('api_calls', 0)} "
f"cache_hits={metrics.get('cache_hits', 0)} "
f"failed={metrics.get('failed', 0)}"
)
try:
if total == 0:
click.echo("No movie/series/anime records found to enrich.")
@@ -596,6 +501,11 @@ def enrich(
click.echo(f" Failed requests: {stats['failed']}")
click.echo(f" Skipped: {stats['skipped']}")
click.echo(f" Needs review: {stats['needs_review']}")
skip_reasons = stats.get("skip_reasons", {})
if isinstance(skip_reasons, dict):
non_zero = [f"{name}={count}" for name, count in sorted(skip_reasons.items()) if int(count) > 0]
if non_zero:
click.echo(f" Skip reasons: {' '.join(non_zero)}")
failed_items = stats.get('failed_items', [])
if isinstance(failed_items, list) and failed_items:
click.echo(" Failure sample:")
@@ -640,8 +550,14 @@ def enrich(
default=Path('analysis.json'),
help='Path to save analysis results (default: analysis.json)'
)
@click.option(
'--inventory',
type=click.Path(exists=True, path_type=Path),
default=None,
help='Optional inventory CSV to merge size/resolution/codec for duplicate quality comparison'
)
@pass_context
def analyze(ctx: CLIContext, input: Path, output: Path):
def analyze(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Path]):
"""Analyze completeness and duplicates.
Detects episode gaps in series and identifies potential duplicate files.
@@ -651,185 +567,23 @@ def analyze(ctx: CLIContext, input: Path, output: Path):
vlm analyze # Use default files
vlm analyze --input my_identities.json # Custom input
vlm analyze --inventory inventory.csv # Merge metadata for quality comparison
vlm analyze --output my_analysis.json # Custom output
"""
import json
from datetime import datetime, timezone
from vlm.analysis import analyze_series_completeness, detect_duplicates
from vlm.models import SeriesIdentity, MovieIdentity, VideoFile
config = ctx.config
logger = ctx.logger
try:
# Display analyze start message
click.echo(f"Analyzing identities from: {input}")
click.echo()
# Load identities from JSON
with open(input, 'r', encoding='utf-8') as jsonfile:
identities_data = json.load(jsonfile)
# Extract movies and series
movies_data = identities_data.get('movies', [])
series_data = identities_data.get('series', [])
click.echo(f"Loaded {len(movies_data)} movies and {len(series_data)} series")
click.echo()
# Convert to identity objects
movie_identities = []
for m in movies_data:
movie_identities.append(MovieIdentity(
title=m['title'],
year=m.get('year'),
confidence=m['confidence'],
needs_review=m['needs_review'],
original_filename=m['filename']
))
series_identities = []
for s in series_data:
series_identities.append(SeriesIdentity(
title=s['title'],
season=s.get('season'),
episodes=s.get('episodes', []),
confidence=s['confidence'],
needs_review=s['needs_review'],
original_filename=s['filename']
))
# Create VideoFile objects for duplicate detection
# We need to reconstruct basic VideoFile info from the identities data
video_files = []
for m in movies_data:
video_files.append(VideoFile(
path=Path(m['path']),
filename=m['filename'],
size_bytes=0, # Not available from identities file
modified_timestamp=datetime.now(timezone.utc),
category=m['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
))
for s in series_data:
video_files.append(VideoFile(
path=Path(s['path']),
filename=s['filename'],
size_bytes=0, # Not available from identities file
modified_timestamp=datetime.now(timezone.utc),
category=s['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
))
# Analyze series completeness
click.echo("Analyzing series completeness...")
completeness_results = analyze_series_completeness(series_identities)
# Detect duplicates
click.echo("Detecting duplicates...")
all_identities = movie_identities + series_identities
duplicate_groups = detect_duplicates(all_identities, video_files)
# Display analysis summary
click.echo()
click.echo("Analysis complete!")
click.echo()
click.echo("Results:")
click.echo(f" Series with episode gaps: {len(completeness_results)}")
if completeness_results:
total_missing = sum(len(c.episodes_missing) for c in completeness_results)
click.echo(f" - Total missing episodes: {total_missing}")
click.echo(f" Duplicate groups found: {len(duplicate_groups)}")
if duplicate_groups:
total_duplicates = sum(len(g.files) for g in duplicate_groups)
click.echo(f" - Total duplicate files: {total_duplicates}")
# Save analysis results to JSON
click.echo()
click.echo(f"Saving analysis results to: {output}")
# Ensure output directory exists
output.parent.mkdir(parents=True, exist_ok=True)
# Build JSON structure
generation_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
# Convert completeness results to dict
completeness_list = []
for c in completeness_results:
completeness_list.append({
'series_title': c.series_title,
'season': c.season,
'episodes_found': c.episodes_found,
'episodes_missing': c.episodes_missing
})
# Convert duplicate groups to dict
duplicates_list = []
for d in duplicate_groups:
# Get identity info
if isinstance(d.identity, MovieIdentity):
identity_info = {
'type': 'movie',
'title': d.identity.title,
'year': d.identity.year
}
else: # SeriesIdentity
identity_info = {
'type': 'series',
'title': d.identity.title,
'season': d.identity.season,
'episodes': d.identity.episodes
}
duplicates_list.append({
'identity': identity_info,
'files': [str(f.path) for f in d.files],
'quality_comparison': d.quality_comparison
})
analysis_data = {
'metadata': {
'generated': generation_timestamp,
'source_identities': str(input),
'total_movies': len(movies_data),
'total_series': len(series_data)
},
'completeness': completeness_list,
'duplicates': duplicates_list
}
# Write JSON file with pretty formatting
with open(output, 'w', encoding='utf-8') as jsonfile:
json.dump(analysis_data, jsonfile, indent=2, ensure_ascii=False)
click.echo(f"Analysis results saved successfully!")
logger.info(f"Analysis completed: {len(completeness_results)} incomplete series, {len(duplicate_groups)} duplicate groups, saved to {output}")
from vlm.commands.analyze import analyze_cmd
analyze_cmd(ctx, input, output, inventory)
except FileNotFoundError:
click.echo(f"Error: Input file not found: {input}", err=True)
logger.error(f"Input file not found: {input}")
ctx.logger.error(f"Input file not found: {input}")
sys.exit(1)
except json.JSONDecodeError as e:
click.echo(f"Error: Failed to parse JSON file: {e}", err=True)
logger.error(f"JSON parsing failed: {e}", exc_info=True)
ctx.logger.error(f"JSON parsing failed: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error during analysis: {e}", err=True)
logger.error(f"Analysis failed: {e}", exc_info=True)
ctx.logger.error(f"Analysis failed: {e}", exc_info=True)
sys.exit(1)
@@ -859,192 +613,20 @@ def plan(ctx: CLIContext, input: Path, output: Path):
vlm plan --input my_identities.json # Custom input
vlm plan --output my_plan.json # Custom output
"""
import json
from vlm.planner import generate_plan, save_plan
from vlm.models import MovieIdentity, SeriesIdentity, VideoFile
from datetime import datetime, timezone
config = ctx.config
logger = ctx.logger
try:
# Display plan start message
click.echo(f"Generating execution plan from: {input}")
click.echo()
# Load identities from JSON
with open(input, 'r', encoding='utf-8') as jsonfile:
identities_data = json.load(jsonfile)
# Extract movies and series
movies_data = identities_data.get('movies', [])
series_data = identities_data.get('series', [])
anime_data = identities_data.get('anime', [])
other_data = identities_data.get('other', [])
click.echo(f"Loaded {len(movies_data)} movies, {len(series_data)} series, {len(anime_data)} anime, {len(other_data)} other")
click.echo()
# Build list of (VideoFile, Identity) tuples for plan generator
identities_list = []
# Process movies
for m in movies_data:
is_approved = m.get('review_status') == 'approved'
video_file = VideoFile(
path=Path(m['path']),
filename=m['filename'],
size_bytes=0, # Not available from identities file
modified_timestamp=datetime.now(timezone.utc),
category=m['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
)
movie_identity = MovieIdentity(
title=m.get('display_title', m['title']),
year=m.get('year'),
confidence=m['confidence'],
needs_review=(m['needs_review'] and not is_approved),
original_filename=m['filename'],
canonical_id=m.get('canonical_id'),
title_zh=m.get('title_zh'),
title_en=m.get('title_en'),
translation_source=m.get('translation_source'),
reputation_score=m.get('reputation_score'),
reputation_votes=m.get('reputation_votes'),
reputation_source=m.get('reputation_source'),
review_status=m.get('review_status', 'pending'),
enrichment_confidence=m.get('enrichment_confidence'),
provider_metadata=m.get('provider_metadata', {})
)
identities_list.append((video_file, movie_identity))
# Process series
for s in series_data:
is_approved = s.get('review_status') == 'approved'
video_file = VideoFile(
path=Path(s['path']),
filename=s['filename'],
size_bytes=0, # Not available from identities file
modified_timestamp=datetime.now(timezone.utc),
category=s['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
)
series_identity = SeriesIdentity(
title=s.get('display_title', s['title']),
season=s.get('season'),
episodes=s.get('episodes', []),
confidence=s['confidence'],
needs_review=(s['needs_review'] and not is_approved),
original_filename=s['filename'],
canonical_id=s.get('canonical_id'),
title_zh=s.get('title_zh'),
title_en=s.get('title_en'),
translation_source=s.get('translation_source'),
reputation_score=s.get('reputation_score'),
reputation_votes=s.get('reputation_votes'),
reputation_source=s.get('reputation_source'),
review_status=s.get('review_status', 'pending'),
enrichment_confidence=s.get('enrichment_confidence'),
provider_metadata=s.get('provider_metadata', {})
)
identities_list.append((video_file, series_identity))
# Process anime (no identity in v1)
for a in anime_data:
video_file = VideoFile(
path=Path(a['path']),
filename=a['filename'],
size_bytes=0,
modified_timestamp=datetime.now(timezone.utc),
category=a['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
)
identities_list.append((video_file, None))
# Process other (no identity)
for o in other_data:
video_file = VideoFile(
path=Path(o['path']),
filename=o['filename'],
size_bytes=0,
modified_timestamp=datetime.now(timezone.utc),
category=o['category'],
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
)
identities_list.append((video_file, None))
# Generate execution plan
click.echo("Generating execution plan...")
execution_plan = generate_plan(identities_list, config)
# Display plan summary
click.echo()
click.echo("Plan generation complete!")
click.echo()
click.echo("Operation summary:")
click.echo(f" Total operations: {execution_plan.summary['total']}")
click.echo(f" Move operations: {execution_plan.summary['move']}")
click.echo(f" Rename operations: {execution_plan.summary['rename']}")
click.echo(f" Quarantine operations: {execution_plan.summary['quarantine']}")
click.echo(f" No-op (skipped): {execution_plan.summary['no-op']}")
# Count conflicts
conflicts = sum(1 for op in execution_plan.operations if op.has_conflict)
if conflicts > 0:
click.echo()
click.echo(f" ⚠️ Conflicts detected: {conflicts}")
click.echo(" Review the plan file for details on conflicting operations.")
# Save execution plan to JSON
click.echo()
click.echo(f"Saving execution plan to: {output}")
# Ensure output directory exists
output.parent.mkdir(parents=True, exist_ok=True)
save_plan(execution_plan, output)
click.echo(f"Execution plan saved successfully!")
click.echo()
click.echo("Next steps:")
click.echo(f" 1. Review the plan: {output}")
click.echo(f" 2. Edit the plan if needed (it's JSON)")
click.echo(f" 3. Dry-run: vlm execute --plan {output}")
click.echo(f" 4. Execute: vlm execute --plan {output} --confirm")
logger.info(f"Plan generated: {execution_plan.summary['total']} operations, {conflicts} conflicts, saved to {output}")
from vlm.commands.plan import plan_cmd
plan_cmd(ctx, input, output)
except FileNotFoundError:
click.echo(f"Error: Input file not found: {input}", err=True)
logger.error(f"Input file not found: {input}")
ctx.logger.error(f"Input file not found: {input}")
sys.exit(1)
except json.JSONDecodeError as e:
click.echo(f"Error: Failed to parse JSON file: {e}", err=True)
logger.error(f"JSON parsing failed: {e}", exc_info=True)
ctx.logger.error(f"JSON parsing failed: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error during plan generation: {e}", err=True)
logger.error(f"Plan generation failed: {e}", exc_info=True)
ctx.logger.error(f"Plan generation failed: {e}", exc_info=True)
sys.exit(1)
@@ -1266,7 +848,7 @@ def quarantine_list(ctx: CLIContext, category: Optional[str]):
click.echo(f" Category: {entry.category}")
click.echo(f" Original: {entry.original_path}")
click.echo(f" Quarantine: {entry.quarantine_path}")
click.echo(f" Size: {_format_size(entry.size_bytes)}")
click.echo(f" Size: {format_size(entry.size_bytes)}")
click.echo(f" Quarantined: {entry.quarantined_at.strftime('%Y-%m-%d %H:%M:%S')}")
if entry.reason:
click.echo(f" Reason: {entry.reason}")
@@ -2246,7 +1828,8 @@ def config_cmd(ctx: CLIContext):
default=default_config_path,
help='Path where configuration file should be created'
)
def config_init(path: Path):
@pass_context
def config_init(ctx: CLIContext, path: Path):
"""Initialize configuration file with defaults."""
try:
if path.exists():