add incremental enrich controls with progress and retry limits

src/vlm/cli.py

@@ -116,8 +116,31 @@ def main(ctx, config: Path, log_level: Optional[str]):
     default=Path('inventory.csv'),
     help='Output file for inventory (default: inventory.csv)'
 )
+@click.option(
+    '--metadata/--no-metadata',
+    default=True,
+    help='Extract video metadata via ffprobe (default: enabled)'
+)
+@click.option(
+    '--reuse-from',
+    type=click.Path(exists=True, path_type=Path),
+    default=None,
+    help='Reuse metadata cache from an existing inventory CSV (default: output file if it exists)'
+)
+@click.option(
+    '--force-refresh-metadata',
+    is_flag=True,
+    default=False,
+    help='Ignore metadata cache and re-run ffprobe for all files'
+)
 @pass_context
-def scan(ctx: CLIContext, output: Path):
+def scan(
+    ctx: CLIContext,
+    output: Path,
+    metadata: bool,
+    reuse_from: Optional[Path],
+    force_refresh_metadata: bool
+):
     """Scan library and generate inventory.
     
     Discovers all video files in the library and records their metadata.
@@ -127,8 +150,11 @@ def scan(ctx: CLIContext, output: Path):
     
         vlm scan                          # Save to inventory.csv
         vlm scan --output my_library.csv  # Save to custom file
+        vlm scan --no-metadata            # Faster scan without ffprobe
+        vlm scan --reuse-from old.csv     # Reuse prior metadata cache
+        vlm scan --force-refresh-metadata # Re-run ffprobe for all files
     """
-    from vlm.scanner import scan_library, save_inventory_csv
+    from vlm.scanner import scan_library, save_inventory_csv, load_inventory_csv
     
     config = ctx.config
     logger = ctx.logger
@@ -139,8 +165,59 @@ def scan(ctx: CLIContext, output: Path):
         click.echo("This may take a while for large libraries...")
         click.echo()
         
+        progress_state: dict[str, Optional[click.ProgressBar]] = {"bar": None}
+        progress_position = {"current": 0}
+
+        def _scan_progress(processed: int, total: int) -> None:
+            if total <= 0:
+                return
+            if progress_state["bar"] is None:
+                bar = click.progressbar(
+                    length=total,
+                    label="Scanning files",
+                    show_pos=True,
+                )
+                progress_state["bar"] = bar.__enter__()
+
+            step = processed - progress_position["current"]
+            if step > 0 and progress_state["bar"] is not None:
+                progress_state["bar"].update(step)
+                progress_position["current"] = processed
+
+        metadata_cache = None
+        cache_source = None
+        if not force_refresh_metadata:
+            cache_source = reuse_from if reuse_from is not None else (output if output.exists() else None)
+
+        if metadata and force_refresh_metadata:
+            click.echo("Forcing metadata refresh for all files (cache disabled).")
+            click.echo()
+
+        if metadata and cache_source is not None:
+            click.echo(f"Loading metadata cache from: {cache_source}")
+            try:
+                cached_files = load_inventory_csv(cache_source)
+                metadata_cache = {str(vf.path): vf for vf in cached_files}
+                click.echo(f"Loaded metadata cache entries: {len(metadata_cache)}")
+                click.echo()
+            except Exception as e:
+                click.echo(f"Warning: could not load metadata cache: {e}")
+                click.echo("Continuing without cache.")
+                click.echo()
+
         # Perform the scan
-        video_files = scan_library(config.library_root, config)
+        try:
+            video_files = scan_library(
+                config.library_root,
+                config,
+                progress_callback=_scan_progress,
+                include_video_metadata=metadata,
+                metadata_cache=metadata_cache
+            )
+        finally:
+            if progress_state["bar"] is not None:
+                progress_state["bar"].__exit__(None, None, None)
+                click.echo()
         
         # Display summary
         click.echo(f"Scan complete!")
@@ -369,6 +446,187 @@ def parse(ctx: CLIContext, input: Path, output: Path):
         sys.exit(1)
 
 
+@main.command()
+@click.option(
+    '--input',
+    type=click.Path(exists=True, path_type=Path),
+    default=Path('identities.json'),
+    help='Path to identities JSON file (default: identities.json)'
+)
+@click.option(
+    '--output',
+    type=click.Path(path_type=Path),
+    default=None,
+    help='Output path (default: overwrite input file)'
+)
+@click.option(
+    '--refresh-changed-only',
+    is_flag=True,
+    default=False,
+    help='Refresh only records whose identity fingerprint changed (incremental behavior)'
+)
+@click.option(
+    '--refresh-all',
+    is_flag=True,
+    default=False,
+    help='Bypass cache and re-enrich all records'
+)
+@click.option(
+    '--timeout',
+    type=int,
+    default=6,
+    show_default=True,
+    help='Per-request timeout in seconds'
+)
+@click.option(
+    '--retries',
+    type=int,
+    default=2,
+    show_default=True,
+    help='Retry attempts for provider/API requests'
+)
+@pass_context
+def enrich(
+    ctx: CLIContext,
+    input: Path,
+    output: Optional[Path],
+    refresh_changed_only: bool,
+    refresh_all: bool,
+    timeout: int,
+    retries: int,
+):
+    """Enrich identities with translation and reputation metadata.
+
+    Applies incremental cache-backed enrichment to parsed identities and writes
+    results back into identities JSON.
+    """
+    import json
+    from vlm.enrichment import enrich_identities_data
+
+    config = ctx.config
+    logger = ctx.logger
+
+    if output is None:
+        output = input
+
+    try:
+        click.echo(f"Enriching identities from: {input}")
+        click.echo(f"Output file: {output}")
+        click.echo()
+
+        with open(input, 'r', encoding='utf-8') as jsonfile:
+            identities_data = json.load(jsonfile)
+
+        if refresh_all and refresh_changed_only:
+            click.echo("Error: --refresh-all and --refresh-changed-only are mutually exclusive.", err=True)
+            sys.exit(1)
+
+        if timeout < 1:
+            click.echo("Error: --timeout must be >= 1", err=True)
+            sys.exit(1)
+
+        if retries < 0:
+            click.echo("Error: --retries must be >= 0", err=True)
+            sys.exit(1)
+
+        refresh_mode = "incremental"
+        if refresh_all:
+            refresh_mode = "refresh_all"
+        elif refresh_changed_only:
+            refresh_mode = "refresh_changed_only"
+
+        total = (
+            len(identities_data.get('movies', []))
+            + len(identities_data.get('series', []))
+            + len(identities_data.get('anime', []))
+        )
+
+        progress_state: dict[str, Optional[click.ProgressBar]] = {"bar": None}
+        progress_position = {"current": 0}
+
+        def _enrich_progress(processed: int, total_count: int, _metrics: dict[str, int]) -> None:
+            if total_count <= 0:
+                return
+
+            if progress_state["bar"] is None:
+                bar = click.progressbar(
+                    length=total_count,
+                    label="Enriching records",
+                    show_pos=True,
+                )
+                progress_state["bar"] = bar.__enter__()
+
+            step = processed - progress_position["current"]
+            if step > 0 and progress_state["bar"] is not None:
+                progress_state["bar"].update(step)
+                progress_position["current"] = processed
+
+        try:
+            if total == 0:
+                click.echo("No movie/series/anime records found to enrich.")
+            enriched_data, stats = enrich_identities_data(
+                identities_data,
+                config,
+                refresh_mode=refresh_mode,
+                request_timeout=timeout,
+                retries=retries,
+                logger=logger,
+                progress_callback=_enrich_progress,
+            )
+        finally:
+            if progress_state["bar"] is not None:
+                progress_state["bar"].__exit__(None, None, None)
+                click.echo()
+
+        output.parent.mkdir(parents=True, exist_ok=True)
+        with open(output, 'w', encoding='utf-8') as jsonfile:
+            json.dump(enriched_data, jsonfile, indent=2, ensure_ascii=False)
+
+        total_records = int(stats['total']) if stats['total'] else 0
+        cache_hits = int(stats['cache_hits'])
+        hit_rate = (cache_hits / total_records * 100.0) if total_records else 0.0
+
+        click.echo("Enrichment complete!")
+        click.echo(f"  Total records: {total_records}")
+        click.echo(f"  Refresh mode: {refresh_mode}")
+        click.echo(f"  Enriched now: {stats['enriched']}")
+        click.echo(f"  Cache hits: {stats['cache_hits']}")
+        click.echo(f"  Cache hit rate: {hit_rate:.1f}%")
+        click.echo(f"  API calls: {stats['api_calls']}")
+        click.echo(f"  Failed requests: {stats['failed']}")
+        click.echo(f"  Skipped: {stats['skipped']}")
+        click.echo(f"  Needs review: {stats['needs_review']}")
+        failed_items = stats.get('failed_items', [])
+        if isinstance(failed_items, list) and failed_items:
+            click.echo("  Failure sample:")
+            for item in failed_items[:3]:
+                click.echo(
+                    f"    - [{item.get('provider', 'unknown')}] {item.get('title', '')}: {item.get('reason', '')}"
+                )
+
+        logger.info(
+            "Enrich completed: total=%s enriched=%s cache_hits=%s failed=%s skipped=%s mode=%s",
+            stats['total'],
+            stats['enriched'],
+            stats['cache_hits'],
+            stats['failed'],
+            stats['skipped'],
+            refresh_mode,
+        )
+    except FileNotFoundError:
+        click.echo(f"Error: Input file not found: {input}", err=True)
+        logger.error(f"Input file not found: {input}")
+        sys.exit(1)
+    except json.JSONDecodeError as e:
+        click.echo(f"Error: Failed to parse JSON file: {e}", err=True)
+        logger.error(f"JSON parsing failed during enrich: {e}", exc_info=True)
+        sys.exit(1)
+    except Exception as e:
+        click.echo(f"Error during enrichment: {e}", err=True)
+        logger.error(f"Enrich failed: {e}", exc_info=True)
+        sys.exit(1)
+
+
 @main.command()
 @click.option(
     '--input',
@@ -632,6 +890,7 @@ def plan(ctx: CLIContext, input: Path, output: Path):
         
         # Process movies
         for m in movies_data:
+            is_approved = m.get('review_status') == 'approved'
             video_file = VideoFile(
                 path=Path(m['path']),
                 filename=m['filename'],
@@ -645,17 +904,28 @@ def plan(ctx: CLIContext, input: Path, output: Path):
             )
             
             movie_identity = MovieIdentity(
-                title=m['title'],
+                title=m.get('display_title', m['title']),
                 year=m.get('year'),
                 confidence=m['confidence'],
-                needs_review=m['needs_review'],
-                original_filename=m['filename']
+                needs_review=(m['needs_review'] and not is_approved),
+                original_filename=m['filename'],
+                canonical_id=m.get('canonical_id'),
+                title_zh=m.get('title_zh'),
+                title_en=m.get('title_en'),
+                translation_source=m.get('translation_source'),
+                reputation_score=m.get('reputation_score'),
+                reputation_votes=m.get('reputation_votes'),
+                reputation_source=m.get('reputation_source'),
+                review_status=m.get('review_status', 'pending'),
+                enrichment_confidence=m.get('enrichment_confidence'),
+                provider_metadata=m.get('provider_metadata', {})
             )
             
             identities_list.append((video_file, movie_identity))
         
         # Process series
         for s in series_data:
+            is_approved = s.get('review_status') == 'approved'
             video_file = VideoFile(
                 path=Path(s['path']),
                 filename=s['filename'],
@@ -669,12 +939,22 @@ def plan(ctx: CLIContext, input: Path, output: Path):
             )
             
             series_identity = SeriesIdentity(
-                title=s['title'],
+                title=s.get('display_title', s['title']),
                 season=s.get('season'),
                 episodes=s.get('episodes', []),
                 confidence=s['confidence'],
-                needs_review=s['needs_review'],
-                original_filename=s['filename']
+                needs_review=(s['needs_review'] and not is_approved),
+                original_filename=s['filename'],
+                canonical_id=s.get('canonical_id'),
+                title_zh=s.get('title_zh'),
+                title_en=s.get('title_en'),
+                translation_source=s.get('translation_source'),
+                reputation_score=s.get('reputation_score'),
+                reputation_votes=s.get('reputation_votes'),
+                reputation_source=s.get('reputation_source'),
+                review_status=s.get('review_status', 'pending'),
+                enrichment_confidence=s.get('enrichment_confidence'),
+                provider_metadata=s.get('provider_metadata', {})
             )
             
             identities_list.append((video_file, series_identity))