refactor CLI command modules and synchronize docs

This commit is contained in:
windyboy
2026-02-16 12:31:26 +08:00
parent 0be802eac5
commit 2dd329cba9
41 changed files with 1066 additions and 753 deletions
+55 -550
View File
@@ -193,180 +193,20 @@ def parse(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Path])
vlm parse --input my_inventory.csv # Custom input
vlm parse --output parsed_identities.json # Custom output
"""
from datetime import datetime, timezone
from vlm.parser import parse_movie, parse_series
from vlm.io import load_inventory_csv, save_identities_json
config = ctx.config
logger = ctx.logger
try:
# Display parse start message
click.echo(f"Parsing identities from: {input}")
# Load video metadata from inventory if provided
path_to_metadata = {}
if inventory:
click.echo(f"Loading video metadata from: {inventory}")
inventory_files = load_inventory_csv(inventory)
path_to_metadata = {str(vf.path): vf for vf in inventory_files}
click.echo(f"Loaded metadata for {len(path_to_metadata)} files")
click.echo()
# Load inventory via unified I/O layer
inventory_files = load_inventory_csv(input)
video_files = [
{
'path': str(vf.path),
'filename': vf.filename,
'category': vf.category,
}
for vf in inventory_files
]
click.echo(f"Loaded {len(video_files)} files from inventory")
click.echo()
# Parse identities based on category
movie_identities = []
series_identities = []
anime_files = []
other_files = []
def get_video_metadata(file_path: str) -> dict:
"""Extract video metadata from inventory if available."""
if not path_to_metadata:
return {}
vf = path_to_metadata.get(file_path)
if not vf:
return {}
return {
'size_bytes': vf.size_bytes,
'modified_timestamp': vf.modified_timestamp.isoformat(),
'resolution': vf.resolution,
'codec': vf.codec,
'duration_seconds': vf.duration_seconds,
'bitrate_kbps': vf.bitrate_kbps,
}
for vf in video_files:
filename = vf['filename']
category = vf['category']
file_path = vf['path']
video_metadata = get_video_metadata(file_path)
if category == 'movie':
identity = parse_movie(filename, extensions=config.video_extensions)
record = {
'path': file_path,
'filename': filename,
'category': category,
'title': identity.title,
'year': identity.year,
'confidence': identity.confidence,
'needs_review': identity.needs_review
}
if video_metadata:
record['video_metadata'] = video_metadata
movie_identities.append(record)
elif category == 'series':
identity = parse_series(filename, extensions=config.video_extensions)
record = {
'path': file_path,
'filename': filename,
'category': category,
'title': identity.title,
'season': identity.season,
'episodes': identity.episodes,
'confidence': identity.confidence,
'needs_review': identity.needs_review
}
if video_metadata:
record['video_metadata'] = video_metadata
series_identities.append(record)
elif category == 'anime':
# Anime files are not parsed in v1
anime_files.append({
'path': vf['path'],
'filename': filename,
'category': category,
'note': 'Anime parsing deferred in v1'
})
else:
# Other files are not parsed
other_files.append({
'path': vf['path'],
'filename': filename,
'category': category,
'note': 'Not categorized for parsing'
})
# Display parsing statistics
click.echo("Parsing complete!")
click.echo()
click.echo("Results by category:")
click.echo(f" Movies: {len(movie_identities)}")
# Count movies needing review
movies_need_review = sum(1 for m in movie_identities if m['needs_review'])
if movies_need_review > 0:
click.echo(f" - Need review: {movies_need_review}")
click.echo(f" Series: {len(series_identities)}")
# Count series needing review
series_need_review = sum(1 for s in series_identities if s['needs_review'])
if series_need_review > 0:
click.echo(f" - Need review: {series_need_review}")
click.echo(f" Anime: {len(anime_files)} (not parsed in v1)")
click.echo(f" Other: {len(other_files)} (not parsed)")
# Save parsed identities to JSON
click.echo()
click.echo(f"Saving parsed identities to: {output}")
# Ensure output directory exists
output.parent.mkdir(parents=True, exist_ok=True)
# Build JSON structure
generation_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
# Use v2 schema if video metadata was embedded
schema_version = "2.0" if path_to_metadata else "1.0"
identities_data = {
'vlm_schema_version': schema_version,
'metadata': {
'generated': generation_timestamp,
'source_inventory': str(input),
'total_files': len(video_files)
},
'movies': movie_identities,
'series': series_identities,
'anime': anime_files,
'other': other_files
}
# Write JSON via unified I/O layer
save_identities_json(identities_data, output)
click.echo(f"Parsed identities saved successfully!")
logger.info(f"Parse completed: {len(movie_identities)} movies, {len(series_identities)} series, saved to {output}")
from vlm.commands.parse import parse_cmd
parse_cmd(ctx, input, output, inventory)
except FileNotFoundError:
click.echo(f"Error: Input file not found: {input}", err=True)
logger.error(f"Input file not found: {input}")
ctx.logger.error(f"Input file not found: {input}")
sys.exit(1)
except Exception as e:
click.echo(f"Error during parsing: {e}", err=True)
logger.error(f"Parse failed: {e}", exc_info=True)
except ValueError as e:
click.echo(f"Error: {e}", err=True)
ctx.logger.error(f"Parse failed: {e}")
sys.exit(1)
except OSError as e:
click.echo(f"Error reading/writing files: {e}", err=True)
ctx.logger.error(f"Parse file I/O failed: {e}", exc_info=True)
sys.exit(1)
@@ -424,150 +264,32 @@ def enrich(
Applies incremental cache-backed enrichment to parsed identities and writes
results back into identities JSON.
"""
import json
from vlm.enrichment import enrich_identities_data
config = ctx.config
logger = ctx.logger
if output is None:
output = input
try:
click.echo(f"Enriching identities from: {input}")
click.echo(f"Output file: {output}")
click.echo()
with open(input, 'r', encoding='utf-8') as jsonfile:
identities_data = json.load(jsonfile)
if refresh_all and refresh_changed_only:
click.echo("Error: --refresh-all and --refresh-changed-only are mutually exclusive.", err=True)
sys.exit(1)
if timeout < 1:
click.echo("Error: --timeout must be >= 1", err=True)
sys.exit(1)
if retries < 0:
click.echo("Error: --retries must be >= 0", err=True)
sys.exit(1)
refresh_mode = "incremental"
if refresh_all:
refresh_mode = "refresh_all"
elif refresh_changed_only:
refresh_mode = "refresh_changed_only"
total = (
len(identities_data.get('movies', []))
+ len(identities_data.get('series', []))
+ len(identities_data.get('anime', []))
)
progress_state: dict[str, Optional[click.ProgressBar]] = {"bar": None}
progress_position = {"current": 0}
progress_bucket = {"value": -1}
is_tty = bool(getattr(sys.stderr, "isatty", lambda: False)())
def _enrich_progress(processed: int, total_count: int, metrics: dict[str, int]) -> None:
if total_count <= 0:
return
if is_tty and progress_state["bar"] is None:
bar = click.progressbar(
length=total_count,
label="Enriching records",
show_pos=True,
)
progress_state["bar"] = bar.__enter__()
step = processed - progress_position["current"]
if step > 0 and progress_state["bar"] is not None:
progress_state["bar"].update(step)
progress_position["current"] = processed
if not is_tty:
percent = int(processed * 100 / total_count)
bucket = percent // 5
if bucket > progress_bucket["value"] or processed == total_count:
progress_bucket["value"] = bucket
click.echo(
"Progress: "
f"{processed}/{total_count} ({percent}%) "
f"api_calls={metrics.get('api_calls', 0)} "
f"cache_hits={metrics.get('cache_hits', 0)} "
f"failed={metrics.get('failed', 0)}"
)
try:
if total == 0:
click.echo("No movie/series/anime records found to enrich.")
enriched_data, stats = enrich_identities_data(
identities_data,
config,
refresh_mode=refresh_mode,
request_timeout=timeout,
retries=retries,
logger=logger,
progress_callback=_enrich_progress,
)
finally:
if progress_state["bar"] is not None:
progress_state["bar"].__exit__(None, None, None)
click.echo()
output.parent.mkdir(parents=True, exist_ok=True)
with open(output, 'w', encoding='utf-8') as jsonfile:
json.dump(enriched_data, jsonfile, indent=2, ensure_ascii=False)
total_records = int(stats['total']) if stats['total'] else 0
cache_hits = int(stats['cache_hits'])
hit_rate = (cache_hits / total_records * 100.0) if total_records else 0.0
click.echo("Enrichment complete!")
click.echo(f" Total records: {total_records}")
click.echo(f" Refresh mode: {refresh_mode}")
click.echo(f" Enriched now: {stats['enriched']}")
click.echo(f" Cache hits: {stats['cache_hits']}")
click.echo(f" Cache hit rate: {hit_rate:.1f}%")
click.echo(f" API calls: {stats['api_calls']}")
click.echo(f" Failed requests: {stats['failed']}")
click.echo(f" Skipped: {stats['skipped']}")
click.echo(f" Needs review: {stats['needs_review']}")
skip_reasons = stats.get("skip_reasons", {})
if isinstance(skip_reasons, dict):
non_zero = [f"{name}={count}" for name, count in sorted(skip_reasons.items()) if int(count) > 0]
if non_zero:
click.echo(f" Skip reasons: {' '.join(non_zero)}")
failed_items = stats.get('failed_items', [])
if isinstance(failed_items, list) and failed_items:
click.echo(" Failure sample:")
for item in failed_items[:3]:
click.echo(
f" - [{item.get('provider', 'unknown')}] {item.get('title', '')}: {item.get('reason', '')}"
)
logger.info(
"Enrich completed: total=%s enriched=%s cache_hits=%s failed=%s skipped=%s mode=%s",
stats['total'],
stats['enriched'],
stats['cache_hits'],
stats['failed'],
stats['skipped'],
refresh_mode,
from vlm.commands.enrich import enrich_cmd
enrich_cmd(
ctx,
input,
output,
refresh_changed_only,
refresh_all,
timeout,
retries,
)
except FileNotFoundError:
click.echo(f"Error: Input file not found: {input}", err=True)
logger.error(f"Input file not found: {input}")
ctx.logger.error(f"Input file not found: {input}")
sys.exit(1)
except json.JSONDecodeError as e:
click.echo(f"Error: Failed to parse JSON file: {e}", err=True)
logger.error(f"JSON parsing failed during enrich: {e}", exc_info=True)
ctx.logger.error(f"JSON parsing failed during enrich: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error during enrichment: {e}", err=True)
logger.error(f"Enrich failed: {e}", exc_info=True)
except ValueError as e:
click.echo(f"Error: {e}", err=True)
ctx.logger.error(f"Enrich validation failed: {e}")
sys.exit(1)
except OSError as e:
click.echo(f"Error reading/writing files: {e}", err=True)
ctx.logger.error(f"Enrich file I/O failed: {e}", exc_info=True)
sys.exit(1)
@@ -791,8 +513,20 @@ def review_plan_cmd(
default=False,
help='Print per-operation dry-run logs at INFO level'
)
@click.option(
'--preserve-directories',
is_flag=True,
default=False,
help='Preserve empty source directories instead of allowing them to be destroyed'
)
@click.option(
'--safe-mode',
is_flag=True,
default=False,
help='Enable safe mode: prevent any operations that would destroy directories'
)
@pass_context
def execute(ctx: CLIContext, plan: Path, confirm: bool, yes: bool, verbose_ops: bool):
def execute(ctx: CLIContext, plan: Path, confirm: bool, yes: bool, verbose_ops: bool, preserve_directories: bool, safe_mode: bool):
"""Execute plan (defaults to dry-run, requires --confirm).
Executes file operations from a plan. Defaults to dry-run mode which
@@ -806,155 +540,20 @@ def execute(ctx: CLIContext, plan: Path, confirm: bool, yes: bool, verbose_ops:
vlm execute --confirm # Actually execute operations (with prompt)
vlm execute --confirm --yes # Execute without confirmation prompt
"""
from vlm.planner import load_plan
from vlm.executor import ExecutionEngine
config = ctx.config
logger = ctx.logger
try:
# Determine execution mode
mode = "execute" if confirm else "dry-run"
# Display execution start message
click.echo(f"Loading execution plan from: {plan}")
click.echo()
# Load execution plan
execution_plan = load_plan(plan)
# Display plan summary
click.echo(f"Execution plan loaded: {execution_plan.plan_id}")
click.echo(f"Created at: {execution_plan.created_at}")
click.echo(f"Total operations: {len(execution_plan.operations)}")
if execution_plan.human_summary:
click.echo()
click.echo(execution_plan.human_summary)
elif execution_plan.summary or execution_plan.summary_by_reason:
s = execution_plan.summary or {}
by_r = execution_plan.summary_by_reason or {}
parts = [f"操作统计:共 {s.get('total', len(execution_plan.operations))} 条(move {s.get('move', 0)},rename {s.get('rename', 0)},quarantine {s.get('quarantine', 0)},no-op {s.get('no-op', 0)})"]
if by_r:
parts.append("原因分布:" + ";".join(f"{r}: {c}" for r, c in list(by_r.items())[:5]))
click.echo()
click.echo("\n".join(parts))
click.echo()
# Display mode warning
if mode == "dry-run":
click.echo("⚠️ DRY-RUN MODE - No files will be modified")
click.echo(" Use --confirm to actually execute operations")
else:
click.echo("⚠️ EXECUTE MODE - Files will be modified!")
click.echo(" This operation cannot be undone without rollback")
click.echo()
if not yes:
if not click.confirm("Are you sure you want to proceed?"):
click.echo("Execution cancelled.")
return
else:
click.echo("Auto-approved via --yes flag")
click.echo()
click.echo(f"Executing {len(execution_plan.operations)} operations...")
click.echo()
# Load state manager to update file statuses during execution
from vlm.state import StateManager
state_path = Path.home() / ".vlm" / "state.json"
state_manager = StateManager(state_path)
# Create execution engine and execute plan
engine = ExecutionEngine(
logger=logger,
config=config,
verbose_operations=verbose_ops,
state_manager=state_manager
)
results, summary, rollback_log = engine.execute_plan(
execution_plan,
mode=mode,
confirmed=confirm
)
# Display execution progress (show some operations)
if mode == "dry-run":
click.echo("Sample operations (dry-run):")
# Show first 5 operations as examples
for i, result in enumerate(results[:5]):
op = result.operation
if op.operation_type != "no-op":
click.echo(f" [{i+1}] {op.operation_type}: {op.source_path.name}")
if op.destination_path:
click.echo(f" -> {op.destination_path}")
if len(results) > 5:
click.echo(f" ... and {len(results) - 5} more operations")
else:
# In execute mode, show progress for all operations
for i, result in enumerate(results):
op = result.operation
if op.operation_type != "no-op" and not op.has_conflict:
status = "✓" if result.success else "✗"
click.echo(f" [{i+1}/{len(results)}] {status} {op.operation_type}: {op.source_path.name}")
if result.error_message:
click.echo(f" Error: {result.error_message}")
# Display execution summary
click.echo()
click.echo("=" * 60)
click.echo(f"Execution Summary ({mode} mode)")
click.echo("=" * 60)
click.echo(f" Total operations: {summary['total']}")
click.echo(f" Successful: {summary['successful']}")
click.echo(f" Failed: {summary['failed']}")
click.echo(f" Skipped: {summary['skipped']}")
click.echo()
# Save rollback log if in execute mode
if mode == "execute" and rollback_log:
# Save to ~/.vlm/rollback/ directory
rollback_dir = Path.home() / ".vlm" / "rollback"
rollback_dir.mkdir(parents=True, exist_ok=True)
rollback_path = rollback_dir / f"rollback_{rollback_log.log_id}.json"
engine.save_rollback_log(rollback_log, rollback_path)
click.echo(f"Rollback log saved to: {rollback_path}")
click.echo()
click.echo("To undo these operations, run:")
click.echo(f" vlm rollback --log {rollback_path}")
click.echo()
# Log completion
if mode == "dry-run":
click.echo("Dry-run complete! No files were modified.")
click.echo("Review the operations above and use --confirm to execute.")
else:
if summary['failed'] > 0:
click.echo(f"⚠️ Execution completed with {summary['failed']} failures.")
click.echo(" Check the log file for details.")
else:
click.echo("✓ Execution completed successfully!")
logger.info(
f"Execution completed in {mode} mode: "
f"{summary['successful']} successful, {summary['failed']} failed, {summary['skipped']} skipped"
)
from vlm.commands.execute import execute_cmd
execute_cmd(ctx, plan, confirm, yes, verbose_ops, preserve_directories, safe_mode)
except FileNotFoundError:
click.echo(f"Error: Plan file not found: {plan}", err=True)
logger.error(f"Plan file not found: {plan}")
click.echo(f"Error: File not found: {plan}", err=True)
ctx.logger.error(f"Execution file not found: {plan}")
sys.exit(1)
except ValueError as e:
click.echo(f"Error: {e}", err=True)
logger.error(f"Execution failed: {e}")
ctx.logger.error(f"Execution validation failed: {e}")
sys.exit(1)
except Exception as e:
except OSError as e:
click.echo(f"Error during execution: {e}", err=True)
logger.error(f"Execution failed: {e}", exc_info=True)
ctx.logger.error(f"Execution I/O failed: {e}", exc_info=True)
sys.exit(1)
@@ -1177,114 +776,20 @@ def rollback(ctx: CLIContext, log: Optional[Path]):
vlm rollback --log rollback_<uuid>.json # Use specific log
vlm rollback --log ~/.vlm/rollback/rollback_*.json
"""
from vlm.executor import ExecutionEngine
config = ctx.config
logger = ctx.logger
try:
# If no log specified, find the most recent rollback log
if log is None:
rollback_dir = Path.home() / ".vlm" / "rollback"
if not rollback_dir.exists():
click.echo("Error: No rollback logs found.", err=True)
click.echo(f"Rollback directory does not exist: {rollback_dir}", err=True)
sys.exit(1)
# Find all rollback log files
rollback_logs = sorted(rollback_dir.glob("rollback_*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
if not rollback_logs:
click.echo("Error: No rollback logs found.", err=True)
click.echo(f"No rollback_*.json files in: {rollback_dir}", err=True)
sys.exit(1)
# Use the most recent log
log = rollback_logs[0]
click.echo(f"Using most recent rollback log: {log}")
click.echo()
# Display rollback start message
click.echo(f"Loading rollback log from: {log}")
click.echo()
# Create execution engine
engine = ExecutionEngine(logger=logger, config=config)
# Load rollback log
rollback_log = engine.load_rollback_log(log)
# Display rollback log info
click.echo(f"Rollback log loaded: {rollback_log.log_id}")
click.echo(f"Original execution: {rollback_log.execution_plan_id}")
click.echo(f"Executed at: {rollback_log.executed_at.strftime('%Y-%m-%d %H:%M:%S')}")
click.echo(f"Operations to rollback: {len(rollback_log.operations)}")
click.echo()
# Display warning
click.echo("⚠️ ROLLBACK OPERATION - Best-effort restoration")
click.echo(" This will attempt to move files back to their original locations.")
click.echo(" Some operations may fail if files have been modified or moved.")
click.echo()
if not click.confirm("Are you sure you want to proceed with rollback?"):
click.echo("Rollback cancelled.")
return
click.echo()
click.echo(f"Rolling back {len(rollback_log.operations)} operations...")
click.echo()
# Perform rollback
results, summary = engine.rollback(rollback_log)
# Display rollback progress
for i, result in enumerate(results):
op = result.operation
if op.operation_type != "no-op":
status = "✓" if result.success else "✗"
click.echo(f" [{i+1}/{len(results)}] {status} Rollback: {op.destination_path.name if op.destination_path else op.source_path.name}")
if result.error_message:
click.echo(f" Error: {result.error_message}")
# Display rollback summary
click.echo()
click.echo("=" * 60)
click.echo("Rollback Summary")
click.echo("=" * 60)
click.echo(f" Total operations: {summary['total']}")
click.echo(f" Successful: {summary['successful']}")
click.echo(f" Failed: {summary['failed']}")
click.echo(f" Skipped: {summary['skipped']}")
click.echo()
# Display completion message
if summary['failed'] > 0:
click.echo(f"⚠️ Rollback completed with {summary['failed']} failures.")
click.echo(" Check the log file for details.")
click.echo(" Some files may not have been restored to their original locations.")
else:
click.echo("✓ Rollback completed successfully!")
click.echo(" All files have been restored to their original locations.")
logger.info(
f"Rollback completed: "
f"{summary['successful']} successful, {summary['failed']} failed, {summary['skipped']} skipped"
)
from vlm.commands.execute import rollback_cmd
rollback_cmd(ctx, log)
except FileNotFoundError as e:
click.echo(f"Error: {e}", err=True)
logger.error(f"Rollback log not found: {e}")
ctx.logger.error(f"Rollback log not found: {e}")
sys.exit(1)
except ValueError as e:
click.echo(f"Error: Invalid rollback log format: {e}", err=True)
logger.error(f"Invalid rollback log: {e}")
click.echo(f"Error: {e}", err=True)
ctx.logger.error(f"Rollback failed: {e}")
sys.exit(1)
except Exception as e:
except OSError as e:
click.echo(f"Error during rollback: {e}", err=True)
logger.error(f"Rollback failed: {e}", exc_info=True)
ctx.logger.error(f"Rollback failed: {e}", exc_info=True)
sys.exit(1)