windy/dl-organizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: snapshot current project updates

Browse Source
This commit is contained in:
windyboy
2026-02-13 13:36:39 +08:00
parent 5931f16a71
commit 0be802eac5
74 changed files with 594899 additions and 66082 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vlm/analysis.py
+1
View File
@@ -142,6 +142,7 @@ def compare_quality(files: list[VideoFile]) -> list[dict]:
'filename': file.filename,
'path': str(file.path),
'size_bytes': file.size_bytes,
'modified_timestamp': file.modified_timestamp.isoformat(),
}
# Add optional metadata if available
src/vlm/cli.py
+163 -133
View File
@@ -4,6 +4,7 @@ This module provides the main CLI entry point using Click framework.
It implements global options (--config, --log-level) and error handling.
"""
import json
import sys
import traceback
from pathlib import Path
@@ -192,11 +193,9 @@ def parse(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Path])
vlm parse --input my_inventory.csv # Custom input
vlm parse --output parsed_identities.json # Custom output
"""
import csv
import json
from datetime import datetime, timezone
from vlm.parser import parse_movie, parse_series
from vlm.io import load_inventory_csv
from vlm.io import load_inventory_csv, save_identities_json
config = ctx.config
logger = ctx.logger
@@ -215,23 +214,16 @@ def parse(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Path])
click.echo()
# Load inventory from CSV
video_files = []
with open(input, 'r', encoding='utf-8') as csvfile:
# Skip comment lines
lines = []
for line in csvfile:
if not line.startswith('#'):
lines.append(line)
# Parse CSV
reader = csv.DictReader(lines)
for row in reader:
video_files.append({
'path': row['path'],
'filename': row['filename'],
'category': row['category']
})
# Load inventory via unified I/O layer
inventory_files = load_inventory_csv(input)
video_files = [
{
'path': str(vf.path),
'filename': vf.filename,
'category': vf.category,
}
for vf in inventory_files
]
click.echo(f"Loaded {len(video_files)} files from inventory")
click.echo()
@@ -360,9 +352,8 @@ def parse(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Path])
'other': other_files
}
# Write JSON file with pretty formatting
with open(output, 'w', encoding='utf-8') as jsonfile:
json.dump(identities_data, jsonfile, indent=2, ensure_ascii=False)
# Write JSON via unified I/O layer
save_identities_json(identities_data, output)
click.echo(f"Parsed identities saved successfully!")
@@ -373,11 +364,6 @@ def parse(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Path])
logger.error(f"Input file not found: {input}")
sys.exit(1)
except csv.Error as e:
click.echo(f"Error: Failed to parse CSV file: {e}", err=True)
logger.error(f"CSV parsing failed: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error during parsing: {e}", err=True)
logger.error(f"Parse failed: {e}", exc_info=True)
@@ -685,6 +671,101 @@ def plan(ctx: CLIContext, input: Path, output: Path, analysis: Path | None):
sys.exit(1)
@main.command(name="review-plan")
@click.option(
'--input',
type=click.Path(exists=True, path_type=Path),
default=Path('plan.json'),
help='Path to execution plan JSON file (default: plan.json)'
)
@click.option(
'--output',
type=click.Path(path_type=Path),
default=Path('plan_manual_review.csv'),
help='Path to save manual review CSV (default: plan_manual_review.csv)'
)
@click.option(
'--season-threshold',
type=int,
default=20,
show_default=True,
help='Flag operations with season >= this value as high risk'
)
@click.option(
'--episode-threshold',
type=int,
default=40,
show_default=True,
help='Flag operations with episode >= this value as high risk'
)
@pass_context
def review_plan_cmd(
ctx: CLIContext,
input: Path,
output: Path,
season_threshold: int,
episode_threshold: int,
):
"""Review a plan and export high-risk operations for manual confirmation."""
from vlm.planner import load_plan
from vlm.plan_review import review_plan, save_review_csv
logger = ctx.logger
try:
if season_threshold < 1 or episode_threshold < 1:
click.echo("Error: thresholds must be >= 1", err=True)
sys.exit(1)
click.echo(f"Loading plan: {input}")
execution_plan = load_plan(input)
rows, counters = review_plan(
execution_plan,
season_threshold=season_threshold,
episode_threshold=episode_threshold,
)
save_review_csv(rows, output)
click.echo()
click.echo("Plan review summary:")
click.echo(f" Total operations: {counters['total_operations']}")
click.echo(f" High-risk operations: {counters['high_risk_operations']}")
click.echo(f" manual_review: {counters['manual_review']}")
click.echo(f" sample_source: {counters['sample_source']}")
click.echo(f" high_season: {counters['high_season']}")
click.echo(f" high_episode: {counters['high_episode']}")
click.echo(f" conflicts: {counters['conflicts']}")
click.echo()
click.echo(f"Saved manual review CSV to: {output}")
if rows:
click.echo("Top review samples:")
for row in rows[:5]:
click.echo(
f" - [{row['index']}] {row['operation_type']} {Path(row['source_path']).name} ({row['risk_flags']})"
)
logger.info(
"Plan review completed: total=%s high_risk=%s output=%s",
counters["total_operations"],
counters["high_risk_operations"],
output,
)
except FileNotFoundError:
click.echo(f"Error: Plan file not found: {input}", err=True)
logger.error(f"Plan file not found: {input}")
sys.exit(1)
except json.JSONDecodeError as e:
click.echo(f"Error: Failed to parse plan JSON: {e}", err=True)
logger.error(f"Plan review JSON parsing failed: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error during plan review: {e}", err=True)
logger.error(f"Plan review failed: {e}", exc_info=True)
sys.exit(1)
@main.command()
@click.option(
'--plan',
@@ -698,19 +779,32 @@ def plan(ctx: CLIContext, input: Path, output: Path, analysis: Path | None):
default=False,
help='Actually execute operations (default is dry-run)'
)
@click.option(
'--yes',
is_flag=True,
default=False,
help='Skip confirmation prompt (auto-approve)'
)
@click.option(
'--verbose-ops',
is_flag=True,
default=False,
help='Print per-operation dry-run logs at INFO level'
)
@pass_context
def execute(ctx: CLIContext, plan: Path, confirm: bool):
def execute(ctx: CLIContext, plan: Path, confirm: bool, yes: bool, verbose_ops: bool):
"""Execute plan (defaults to dry-run, requires --confirm).
Executes file operations from a plan. Defaults to dry-run mode which
simulates operations without making changes. Use --confirm to actually
execute operations.
Example:
vlm execute # Dry-run with plan.json
vlm execute --plan my_plan.json # Dry-run with custom plan
vlm execute --confirm # Actually execute operations
vlm execute --confirm # Actually execute operations (with prompt)
vlm execute --confirm --yes # Execute without confirmation prompt
"""
from vlm.planner import load_plan
from vlm.executor import ExecutionEngine
@@ -754,16 +848,29 @@ def execute(ctx: CLIContext, plan: Path, confirm: bool):
click.echo("⚠️ EXECUTE MODE - Files will be modified!")
click.echo(" This operation cannot be undone without rollback")
click.echo()
if not click.confirm("Are you sure you want to proceed?"):
click.echo("Execution cancelled.")
return
if not yes:
if not click.confirm("Are you sure you want to proceed?"):
click.echo("Execution cancelled.")
return
else:
click.echo("Auto-approved via --yes flag")
click.echo()
click.echo(f"Executing {len(execution_plan.operations)} operations...")
click.echo()
# Load state manager to update file statuses during execution
from vlm.state import StateManager
state_path = Path.home() / ".vlm" / "state.json"
state_manager = StateManager(state_path)
# Create execution engine and execute plan
engine = ExecutionEngine(logger=logger, config=config)
engine = ExecutionEngine(
logger=logger,
config=config,
verbose_operations=verbose_ops,
state_manager=state_manager
)
results, summary, rollback_log = engine.execute_plan(
execution_plan,
mode=mode,
@@ -1237,10 +1344,8 @@ def report_inventory(ctx: CLIContext, format: str, input: Path, output: Optional
vlm report inventory --format csv # CSV format to console
vlm report inventory --format json --output inventory_report.json
"""
import csv
from datetime import datetime, timezone
from vlm.reports import generate_inventory_report
from vlm.models import VideoFile
from vlm.io import load_inventory_csv
config = ctx.config
logger = ctx.logger
@@ -1249,40 +1354,7 @@ def report_inventory(ctx: CLIContext, format: str, input: Path, output: Optional
# Load inventory from CSV
click.echo(f"Loading inventory from: {input}")
video_files = []
with open(input, 'r', encoding='utf-8') as csvfile:
# Skip comment lines
lines = []
for line in csvfile:
if not line.startswith('#'):
lines.append(line)
# Parse CSV
reader = csv.DictReader(lines)
for row in reader:
# Parse timestamp
modified_timestamp = datetime.fromisoformat(row['modified_timestamp'])
if modified_timestamp.tzinfo is None:
modified_timestamp = modified_timestamp.replace(tzinfo=timezone.utc)
# Parse optional fields
resolution = row.get('resolution') if row.get('resolution') else None
codec = row.get('codec') if row.get('codec') else None
duration_seconds = float(row['duration_seconds']) if row.get('duration_seconds') else None
bitrate_kbps = int(row['bitrate_kbps']) if row.get('bitrate_kbps') else None
video_file = VideoFile(
path=Path(row['path']),
filename=row['filename'],
size_bytes=int(row['size_bytes']),
modified_timestamp=modified_timestamp,
category=row['category'],
resolution=resolution,
codec=codec,
duration_seconds=duration_seconds,
bitrate_kbps=bitrate_kbps
)
video_files.append(video_file)
video_files = load_inventory_csv(input)
click.echo(f"Loaded {len(video_files)} files")
click.echo()
@@ -1313,11 +1385,6 @@ def report_inventory(ctx: CLIContext, format: str, input: Path, output: Optional
logger.error(f"Input file not found: {input}")
sys.exit(1)
except csv.Error as e:
click.echo(f"Error: Failed to parse CSV file: {e}", err=True)
logger.error(f"CSV parsing failed: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error generating inventory report: {e}", err=True)
logger.error(f"Inventory report generation failed: {e}", exc_info=True)
@@ -1362,10 +1429,10 @@ def report_completeness(ctx: CLIContext, format: str, input: Path, output: Optio
vlm report completeness --plan plan.json # Include plan content summary
vlm report completeness --format text --output completeness.txt
"""
import json
from vlm.reports import generate_completeness_report
from vlm.models import SeasonCompleteness
from vlm.planner import load_plan
from vlm.io import load_analysis_json
config = ctx.config
logger = ctx.logger
@@ -1382,8 +1449,7 @@ def report_completeness(ctx: CLIContext, format: str, input: Path, output: Optio
# Load analysis from JSON
click.echo(f"Loading analysis from: {input}")
with open(input, 'r', encoding='utf-8') as jsonfile:
analysis_data = json.load(jsonfile)
analysis_data = load_analysis_json(input)
# Extract completeness data
completeness_list = analysis_data.get('completeness', [])
@@ -1476,11 +1542,11 @@ def report_duplicates(ctx: CLIContext, format: str, input: Path, output: Optiona
vlm report duplicates --plan plan.json # Include plan content summary
vlm report duplicates --format text --output duplicates.txt
"""
import json
from vlm.reports import generate_duplicate_report
from vlm.models import DuplicateGroup, MovieIdentity, SeriesIdentity, VideoFile
from vlm.planner import load_plan
from datetime import datetime, timezone
from vlm.io import load_analysis_json
config = ctx.config
logger = ctx.logger
@@ -1497,8 +1563,7 @@ def report_duplicates(ctx: CLIContext, format: str, input: Path, output: Optiona
# Load analysis from JSON
click.echo(f"Loading analysis from: {input}")
with open(input, 'r', encoding='utf-8') as jsonfile:
analysis_data = json.load(jsonfile)
analysis_data = load_analysis_json(input)
# Extract duplicates data
duplicates_list = analysis_data.get('duplicates', [])
@@ -1527,19 +1592,24 @@ def report_duplicates(ctx: CLIContext, format: str, input: Path, output: Optiona
original_filename=""
)
# Reconstruct VideoFile objects from file paths
quality_by_path = {
str(item.get("path", "")): item for item in d.get("quality_comparison", [])
}
# Reconstruct VideoFile objects from file paths and preserve size metadata
files = []
for file_path in d['files']:
quality = quality_by_path.get(str(file_path), {})
files.append(VideoFile(
path=Path(file_path),
filename=Path(file_path).name,
size_bytes=0,
size_bytes=int(quality.get("size_bytes", 0) or 0),
modified_timestamp=datetime.now(timezone.utc),
category="",
resolution=None,
codec=None,
duration_seconds=None,
bitrate_kbps=None
resolution=quality.get("resolution"),
codec=quality.get("codec"),
duration_seconds=quality.get("duration_seconds"),
bitrate_kbps=quality.get("bitrate_kbps"),
))
duplicate_groups.append(DuplicateGroup(
@@ -1612,10 +1682,8 @@ def report_summary(ctx: CLIContext, input: Path, output: Optional[Path]):
vlm report summary # Print to console
vlm report summary --output summary.txt # Save to file
"""
import csv
from datetime import datetime, timezone
from vlm.reports import generate_summary_report
from vlm.models import VideoFile
from vlm.io import load_inventory_csv
config = ctx.config
logger = ctx.logger
@@ -1624,40 +1692,7 @@ def report_summary(ctx: CLIContext, input: Path, output: Optional[Path]):
# Load inventory from CSV
click.echo(f"Loading inventory from: {input}")
video_files = []
with open(input, 'r', encoding='utf-8') as csvfile:
# Skip comment lines
lines = []
for line in csvfile:
if not line.startswith('#'):
lines.append(line)
# Parse CSV
reader = csv.DictReader(lines)
for row in reader:
# Parse timestamp
modified_timestamp = datetime.fromisoformat(row['modified_timestamp'])
if modified_timestamp.tzinfo is None:
modified_timestamp = modified_timestamp.replace(tzinfo=timezone.utc)
# Parse optional fields
resolution = row.get('resolution') if row.get('resolution') else None
codec = row.get('codec') if row.get('codec') else None
duration_seconds = float(row['duration_seconds']) if row.get('duration_seconds') else None
bitrate_kbps = int(row['bitrate_kbps']) if row.get('bitrate_kbps') else None
video_file = VideoFile(
path=Path(row['path']),
filename=row['filename'],
size_bytes=int(row['size_bytes']),
modified_timestamp=modified_timestamp,
category=row['category'],
resolution=resolution,
codec=codec,
duration_seconds=duration_seconds,
bitrate_kbps=bitrate_kbps
)
video_files.append(video_file)
video_files = load_inventory_csv(input)
click.echo(f"Loaded {len(video_files)} files")
click.echo()
@@ -1685,11 +1720,6 @@ def report_summary(ctx: CLIContext, input: Path, output: Optional[Path]):
logger.error(f"Input file not found: {input}")
sys.exit(1)
except csv.Error as e:
click.echo(f"Error: Failed to parse CSV file: {e}", err=True)
logger.error(f"CSV parsing failed: {e}", exc_info=True)
sys.exit(1)
except Exception as e:
click.echo(f"Error generating summary report: {e}", err=True)
logger.error(f"Summary report generation failed: {e}", exc_info=True)
src/vlm/commands/plan.py
+15 -1
View File
@@ -49,7 +49,21 @@ def plan_cmd(ctx: CLIContext, input: Path, output: Path, analysis: Optional[Path
raise
click.echo("Generating execution plan...")
execution_plan = generate_plan(identities_list, config, analysis_data=analysis_data)
# Load state to filter ignored files
from vlm.state import StateManager
state_path = Path.home() / ".vlm" / "state.json"
state_manager = StateManager(state_path)
ignored_paths = {str(s.file_path) for s in state_manager.query_by_status("ignored")}
if ignored_paths:
click.echo(f" Filtering {len(ignored_paths)} files marked as 'ignored' in state.")
execution_plan = generate_plan(
identities_list,
config,
analysis_data=analysis_data,
ignored_paths=ignored_paths
)
click.echo()
click.echo("Plan generation complete!")
src/vlm/config.py
+30 -2
View File
@@ -49,6 +49,9 @@ class Config:
# Plan settings (e.g. duplicate handling when consuming analysis)
duplicate_keep: str = "by_reputation"
plan_max_season: int = 15
plan_max_episode: int = 100
plan_include_sample_files: bool = False
def load_config(path: Path) -> Config:
@@ -91,6 +94,9 @@ def load_config(path: Path) -> Config:
plan = data.get("plan", {})
duplicate_keep = plan.get("duplicate_keep", "by_reputation")
plan_max_season = int(plan.get("max_season", 15))
plan_max_episode = int(plan.get("max_episode", 100))
plan_include_sample_files = bool(plan.get("include_sample_files", False))
enrichment = data.get("enrichment")
if enrichment is None:
@@ -133,6 +139,9 @@ def load_config(path: Path) -> Config:
reputation_policy=reputation.get("policy", "flag_for_review"),
naming_title_format=naming.get("title_format", "{title_zh} {title_en}"),
duplicate_keep=duplicate_keep,
plan_max_season=plan_max_season,
plan_max_episode=plan_max_episode,
plan_include_sample_files=plan_include_sample_files,
)
@@ -175,7 +184,12 @@ def create_default_config(path: Path) -> Config:
},
}
plan_content = {"duplicate_keep": default_config.duplicate_keep}
plan_content = {
"duplicate_keep": default_config.duplicate_keep,
"max_season": default_config.plan_max_season,
"max_episode": default_config.plan_max_episode,
"include_sample_files": default_config.plan_include_sample_files,
}
yaml_content = {
"library_root": str(default_config.library_root),
@@ -244,6 +258,11 @@ def validate_config(config: Config) -> list[str]:
elif not isinstance(config.series_filename_template, str):
errors.append("series_filename_template must be a string")
if not isinstance(config.enrichment_max_concurrency, int):
errors.append("enrichment_max_concurrency must be an integer")
elif config.enrichment_max_concurrency < 1:
errors.append("enrichment_max_concurrency must be >= 1")
valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
if not config.log_level:
errors.append("log_level cannot be empty")
@@ -326,12 +345,21 @@ def validate_config(config: Config) -> list[str]:
errors.append("tmdb_include_adult must be a boolean")
if config.duplicate_keep not in (
"by_reputation",
"by_reputation_quality_time",
"first_seen",
"manual",
"by_quality",
):
errors.append(
f"duplicate_keep must be one of 'by_reputation', 'first_seen', 'manual', 'by_quality', got: {config.duplicate_keep!r}"
"duplicate_keep must be one of "
"'by_reputation', 'by_reputation_quality_time', 'first_seen', 'manual', 'by_quality', "
f"got: {config.duplicate_keep!r}"
)
if not isinstance(config.plan_max_season, int) or config.plan_max_season < 1:
errors.append("plan_max_season must be an integer >= 1")
if not isinstance(config.plan_max_episode, int) or config.plan_max_episode < 1:
errors.append("plan_max_episode must be an integer >= 1")
if not isinstance(config.plan_include_sample_files, bool):
errors.append("plan_include_sample_files must be a boolean")
return errors
src/vlm/duplicate_resolve.py
+90 -9
View File
@@ -1,5 +1,6 @@
"""Duplicate group resolution: choose which file to keep when consuming analysis."""
from datetime import datetime
import re
from pathlib import Path
from typing import Optional, Union
@@ -7,6 +8,14 @@ from typing import Optional, Union
from vlm.models import MovieIdentity, SeriesIdentity
def _is_sample_path(path: Path) -> bool:
"""Identify likely sample clips by path component or filename token."""
parts = [part.casefold() for part in path.parts]
if "sample" in parts:
return True
return bool(re.search(r"(^|[\s._-])sample($|[\s._-])", path.stem.casefold()))
def choose_keep_index(
items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]],
strategy: str,
@@ -17,13 +26,14 @@ def choose_keep_index(
Strategies:
- by_quality: Prefer higher quality (resolution > source > codec > size). Requires quality_comparison.
- by_reputation: Prefer items with reputation_score; then sort by score desc,
then reputation_votes desc, then first_seen (input order). Keep index 0 after sort.
then reputation_votes desc, then quality, then first_seen (input order).
- by_reputation_quality_time: Prefer reputation first, then quality, then newer modified time.
- first_seen: Keep the first item (index 0).
- manual: Return None; caller should not generate quarantine ops, only record in metadata.
Args:
items: List of (path, identity) for the duplicate group.
strategy: One of "by_quality", "by_reputation", "first_seen", "manual".
strategy: One of "by_quality", "by_reputation", "by_reputation_quality_time", "first_seen", "manual".
quality_comparison: List of quality dicts (filename, path, size_bytes, resolution?, codec?)
aligned with items. Required when strategy is "by_quality".
@@ -39,7 +49,9 @@ def choose_keep_index(
return 0 # Fallback to first if quality data missing/mismatched
return _by_quality_index(items, quality_comparison)
if strategy == "by_reputation":
return _by_reputation_index(items)
return _by_reputation_index(items, quality_comparison=quality_comparison)
if strategy == "by_reputation_quality_time":
return _by_reputation_quality_time_index(items, quality_comparison=quality_comparison)
return 0
@@ -101,9 +113,7 @@ def _by_quality_index(
quality_comparison: list[dict],
) -> int:
"""Sort by: resolution desc, source desc, codec desc, size desc, index asc. Return best index."""
def key(idx_item: tuple[int, tuple[Path, Union[MovieIdentity, SeriesIdentity]]]) -> tuple:
idx, (path, _) = idx_item
qc = quality_comparison[idx] if idx < len(quality_comparison) else {}
def quality_key(path: Path, qc: dict, idx: int) -> tuple:
resolution = qc.get("resolution")
codec = qc.get("codec")
size = qc.get("size_bytes", 0) or 0
@@ -112,6 +122,12 @@ def _by_quality_index(
codec_tier = _parse_codec_tier(codec, path)
return (-res_tier, -src_tier, -codec_tier, -size, idx)
def key(idx_item: tuple[int, tuple[Path, Union[MovieIdentity, SeriesIdentity]]]) -> tuple:
idx, (path, _) = idx_item
qc = quality_comparison[idx] if idx < len(quality_comparison) else {}
is_sample = _is_sample_path(path)
return (is_sample, *quality_key(path, qc, idx))
indexed = list(enumerate(items))
indexed.sort(key=key)
return indexed[0][0]
@@ -119,15 +135,80 @@ def _by_quality_index(
def _by_reputation_index(
items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]],
quality_comparison: Optional[list[dict]] = None,
) -> int:
"""Sort by: has reputation > no reputation; then score desc; then votes desc; then order. Return 0."""
"""Sort by: sample, reputation, quality, then input order."""
quality_list = quality_comparison or []
def quality_key(path: Path, qc: dict, idx: int) -> tuple:
resolution = qc.get("resolution")
codec = qc.get("codec")
size = qc.get("size_bytes", 0) or 0
res_tier = _parse_resolution_tier(resolution, path)
src_tier = _parse_source_tier(path)
codec_tier = _parse_codec_tier(codec, path)
# Reputation fallback: prioritize source first (e.g. BluRay over WEB-DL).
return (-src_tier, -res_tier, -codec_tier, -size, idx)
def key(idx_reason: tuple[int, tuple[Path, Union[MovieIdentity, SeriesIdentity]]]) -> tuple:
idx, (path, identity) = idx_reason
qc = quality_list[idx] if idx < len(quality_list) else {}
has_rep = identity.reputation_score is not None
score = identity.reputation_score if identity.reputation_score is not None else -1.0
votes = identity.reputation_votes if identity.reputation_votes is not None else -1
# Prefer has reputation (True > False), then higher score, then higher votes, then lower index
return (not has_rep, -score, -votes, idx)
is_sample = _is_sample_path(path)
# Prefer non-sample, then reputation, then better quality, then lower index.
return (is_sample, not has_rep, -score, -votes, *quality_key(path, qc, idx))
indexed = list(enumerate(items))
indexed.sort(key=key)
return indexed[0][0]
def _parse_modified_timestamp(value: Optional[str]) -> float:
"""Parse ISO modified timestamp to unix epoch seconds; unknown returns 0."""
if not value:
return 0.0
try:
normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
return datetime.fromisoformat(normalized).timestamp()
except (ValueError, TypeError):
return 0.0
def _by_reputation_quality_time_index(
items: list[tuple[Path, Union[MovieIdentity, SeriesIdentity]]],
quality_comparison: Optional[list[dict]] = None,
) -> int:
"""Sort by: sample, reputation, quality, modified time (newer first), then order."""
quality_list = quality_comparison or []
def quality_key(path: Path, qc: dict) -> tuple:
resolution = qc.get("resolution")
codec = qc.get("codec")
size = qc.get("size_bytes", 0) or 0
res_tier = _parse_resolution_tier(resolution, path)
src_tier = _parse_source_tier(path)
codec_tier = _parse_codec_tier(codec, path)
return (-res_tier, -src_tier, -codec_tier, -size)
def key(idx_reason: tuple[int, tuple[Path, Union[MovieIdentity, SeriesIdentity]]]) -> tuple:
idx, (path, identity) = idx_reason
qc = quality_list[idx] if idx < len(quality_list) else {}
has_rep = identity.reputation_score is not None
score = identity.reputation_score if identity.reputation_score is not None else -1.0
votes = identity.reputation_votes if identity.reputation_votes is not None else -1
modified_ts = _parse_modified_timestamp(qc.get("modified_timestamp"))
is_sample = _is_sample_path(path)
return (
is_sample,
not has_rep,
-score,
-votes,
*quality_key(path, qc),
-modified_ts,
idx,
)
indexed = list(enumerate(items))
indexed.sort(key=key)
src/vlm/enrichment.py
+69 -19
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import hashlib
import json
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Callable, Optional
from urllib.request import urlopen, Request
@@ -14,6 +15,7 @@ from vlm.cache import EnrichmentCache
from vlm.config import Config
from vlm.providers import ProviderResult, TMDBAuthError, TMDBProvider, TMDBProviderError
from vlm.parser import normalize_title
from vlm.utils import sanitize_path_component
RefreshMode = str
ProgressCallback = Callable[[int, int, dict[str, int]], None]
@@ -37,11 +39,6 @@ def enrich_identities_data(
- refresh_all: bypasses cache and re-fetches all records
"""
cache = EnrichmentCache(config.enrichment_cache_db)
providers = _build_providers(
config,
request_timeout=request_timeout,
retries=retries,
)
total_records = (
len(identities_data.get("movies", []))
@@ -64,8 +61,12 @@ def enrich_identities_data(
refresh_all = refresh_mode == "refresh_all"
max_workers = max(1, int(config.enrichment_max_concurrency))
for section, media_type in (("movies", "movie"), ("series", "series"), ("anime", "anime")):
records = identities_data.get(section, [])
pending_jobs: list[tuple[dict, str, str, str]] = []
for record in records:
title = record.get("title") or _fallback_title_from_filename(record.get("filename"))
if not title:
@@ -94,20 +95,47 @@ def enrich_identities_data(
_emit_progress(stats, progress_callback)
continue
payload, api_calls, failures, skip_reason = _enrich_record(
record,
media_type,
providers,
config,
request_timeout=request_timeout,
retries=retries,
)
pending_jobs.append((record, identity_key, fingerprint, media_type))
_apply_payload(record, payload)
cache.put_identity(identity_key, fingerprint, payload)
_update_stats_after_enrich(
stats, payload, failures, api_calls, record, progress_callback, skip_reason
)
if not pending_jobs:
continue
if max_workers <= 1:
for record, identity_key, fingerprint, pending_media_type in pending_jobs:
payload, api_calls, failures, skip_reason = _enrich_record_with_fresh_providers(
record,
pending_media_type,
config,
request_timeout=request_timeout,
retries=retries,
)
_apply_payload(record, payload)
cache.put_identity(identity_key, fingerprint, payload)
_update_stats_after_enrich(
stats, payload, failures, api_calls, record, progress_callback, skip_reason
)
else:
future_map = {}
with ThreadPoolExecutor(max_workers=max_workers) as pool:
for record, identity_key, fingerprint, pending_media_type in pending_jobs:
future = pool.submit(
_enrich_record_with_fresh_providers,
record,
pending_media_type,
config,
request_timeout,
retries,
)
future_map[future] = (record, identity_key, fingerprint)
for future in as_completed(future_map):
record, identity_key, fingerprint = future_map[future]
payload, api_calls, failures, skip_reason = future.result()
_apply_payload(record, payload)
cache.put_identity(identity_key, fingerprint, payload)
_update_stats_after_enrich(
stats, payload, failures, api_calls, record, progress_callback, skip_reason
)
metadata = identities_data.setdefault("metadata", {})
metadata["enriched"] = True
@@ -198,6 +226,28 @@ def _build_providers(config: Config, *, request_timeout: int, retries: int) -> l
return providers
def _enrich_record_with_fresh_providers(
record: dict,
media_type: str,
config: Config,
request_timeout: int,
retries: int,
) -> tuple[dict, int, list[dict[str, str]], str]:
providers = _build_providers(
config,
request_timeout=request_timeout,
retries=retries,
)
return _enrich_record(
record,
media_type,
providers,
config,
request_timeout=request_timeout,
retries=retries,
)
def _enrich_record(
record: dict,
media_type: str,
@@ -519,4 +569,4 @@ def _build_display_title(record: dict, payload: dict, config: Config) -> str:
).strip()
except Exception:
formatted = f"{title_zh or ''} {title_en or ''}".strip()
return " ".join(formatted.split())
return sanitize_path_component(" ".join(formatted.split()), fallback=fallback_title or "untitled")
src/vlm/executor.py
+72 -5
View File
@@ -18,7 +18,9 @@ from .config import Config
from .logging_config import get_logger, log_operation
from .models import ExecutionPlan, FileOperation, OperationResult, RollbackLog
from .quarantine import QuarantineManager
from .utils import ensure_utc, utc_now
from .state import StateManager
from .transaction import TransactionLog
from .utils import ensure_utc, is_within_root, utc_now
class ExecutionEngine:
@@ -28,15 +30,21 @@ class ExecutionEngine:
self,
logger: Optional[logging.Logger] = None,
config: Optional[Config] = None,
verbose_operations: bool = False,
state_manager: Optional[StateManager] = None,
):
"""Initialize the execution engine.
Args:
logger: Optional logger instance (uses default if not provided)
config: Optional config (required for quarantine operations)
verbose_operations: Emit per-operation dry-run logs at INFO when True
state_manager: Optional state manager for updating file statuses
"""
self.logger = logger or get_logger()
self.config = config
self.verbose_operations = verbose_operations
self.state_manager = state_manager
self._quarantine_manager: Optional[QuarantineManager] = (
QuarantineManager(config, self.logger) if config else None
)
@@ -80,12 +88,47 @@ class ExecutionEngine:
f"Starting execution in {mode} mode with {len(plan.operations)} operations",
operation_type="execute"
)
# Initialize transaction log for execute mode
transaction_log = None
if mode == "execute":
log_path = Path.home() / ".vlm" / "transaction.json"
transaction_log = TransactionLog(log_path)
transaction_log.start_transaction(plan)
# Execute all operations
results = []
for operation in plan.operations:
for i, operation in enumerate(plan.operations):
result = self.execute_operation(operation, mode)
results.append(result)
# Update transaction and state logs in execute mode
if mode == "execute":
if transaction_log:
transaction_log.mark_operation_complete(
i, result.success, result.error_message
)
# Update file state if successful and not a no-op
if result.success and operation.operation_type != "no-op" and self.state_manager:
new_status = "quarantined" if operation.operation_type == "quarantine" else "executed"
self.state_manager.set_file_state(
operation.source_path,
status=new_status,
reason=operation.reason
)
# We could save state incrementally, but saving at the end is more efficient.
# For extra safety, we'll save every 10 operations.
if (i + 1) % 10 == 0:
self.state_manager.save()
# Finalize transaction and state
if mode == "execute":
if transaction_log:
status = "completed" if all(r.success for r in results) else "failed"
transaction_log.complete_transaction(status=status)
if self.state_manager:
self.state_manager.save()
# Generate execution summary
summary = self._generate_execution_summary(results)
@@ -156,9 +199,10 @@ class ExecutionEngine:
executed_at=executed_at
)
if mode == "dry-run":
level = logging.INFO if self.verbose_operations else logging.DEBUG
log_operation(
self.logger,
logging.INFO,
level,
f"[DRY-RUN] Would quarantine: {operation.source_path} ({operation.reason})",
operation_type="execute",
file_path=operation.source_path
@@ -219,7 +263,7 @@ class ExecutionEngine:
)
log_operation(
self.logger,
logging.INFO,
logging.INFO if self.verbose_operations else logging.DEBUG,
msg,
operation_type="execute",
file_path=operation.source_path
@@ -266,6 +310,25 @@ class ExecutionEngine:
# Create destination directory if needed
if operation.destination_path:
if self.config and not is_within_root(
operation.destination_path, self.config.library_root
):
error_msg = (
f"Unsafe destination outside library root: {operation.destination_path}"
)
log_operation(
self.logger,
logging.ERROR,
error_msg,
operation_type="execute",
file_path=operation.source_path
)
return OperationResult(
operation=operation,
success=False,
error_message=error_msg,
executed_at=executed_at
)
operation.destination_path.parent.mkdir(parents=True, exist_ok=True)
# Perform the move/rename operation
@@ -316,7 +379,11 @@ class ExecutionEngine:
Dictionary with counts of successful, failed, and skipped operations
"""
successful = sum(1 for r in results if r.success)
failed = sum(1 for r in results if not r.success)
failed = sum(
1
for r in results
if (not r.success) and (not r.operation.has_conflict)
)
skipped = sum(
1 for r in results
if r.operation.operation_type == "no-op" or r.operation.has_conflict
src/vlm/parser.py
+4 -7
View File
@@ -65,13 +65,13 @@ def remove_release_groups(text: str) -> str:
def normalize_title(title: str) -> str:
"""Normalize a title by cleaning whitespace and standardizing capitalization.
"""Normalize a title by cleaning separators and whitespace.
Args:
title: Raw title string
Returns:
Normalized title with proper capitalization and spacing
Normalized title with cleaned separators and spacing
"""
# Replace dots and underscores with spaces
title = title.replace('.', ' ').replace('_', ' ')
@@ -79,9 +79,6 @@ def normalize_title(title: str) -> str:
# Remove extra whitespace
title = ' '.join(title.split())
# Apply title case
title = title.title()
return title.strip()
@@ -193,8 +190,8 @@ def parse_series(
# Pattern: SXXEYY or SXXeYY - High confidence
# Also handles multi-episode: S01E01-E02, S01E01E02E03, etc.
(r'[Ss](\d{1,2})[Ee](\d{1,2})', 0.9),
# Pattern: XXxYY - High confidence
(r'(\d{1,2})x(\d{1,2})', 0.9),
# Pattern: XXxYY - High confidence (boundary-sensitive, avoid resolution substrings)
(r'(?<!\d)(\d{1,2})x(\d{1,2})(?!\d)', 0.9),
# Pattern: Season X Episode Y - Medium confidence
(r'[Ss]eason\s*(\d{1,2})\s*[Ee]pisode\s*(\d{1,2})', 0.7),
]
src/vlm/plan_review.py
+102
View File
@@ -0,0 +1,102 @@
"""Plan review helpers for flagging high-risk operations before execution."""
from __future__ import annotations
import csv
import re
from pathlib import Path
from vlm.models import ExecutionPlan, FileOperation
def _is_sample_path(path: Path) -> bool:
parts = [part.casefold() for part in path.parts]
if "sample" in parts:
return True
return bool(re.search(r"(^|[\s._-])sample($|[\s._-])", path.stem.casefold()))
def _extract_season_episode(operation: FileOperation) -> tuple[int | None, int | None]:
season: int | None = None
episode: int | None = None
for text in [
str(operation.destination_path) if operation.destination_path else "",
operation.reason,
operation.source_path.name,
]:
if season is None:
m = re.search(r"S(\d{1,3})E(\d{1,3})", text, re.IGNORECASE)
if m:
season = int(m.group(1))
episode = int(m.group(2))
else:
m = re.search(r"Season\s+(\d{1,3})", text, re.IGNORECASE)
if m:
season = int(m.group(1))
if episode is None:
m = re.search(r"E(\d{1,3})", text, re.IGNORECASE)
if m:
episode = int(m.group(1))
return season, episode
def review_plan(
plan: ExecutionPlan,
season_threshold: int = 20,
episode_threshold: int = 40,
) -> tuple[list[dict[str, str]], dict[str, int]]:
"""Build review rows and aggregate risk counters."""
rows: list[dict[str, str]] = []
counters = {
"total_operations": len(plan.operations),
"high_risk_operations": 0,
"manual_review": 0,
"sample_source": 0,
"high_season": 0,
"high_episode": 0,
"conflicts": 0,
}
for idx, op in enumerate(plan.operations, start=1):
flags: list[str] = []
reason_l = op.reason.casefold()
if "manual review" in reason_l:
flags.append("manual_review")
counters["manual_review"] += 1
if _is_sample_path(op.source_path):
flags.append("sample_source")
counters["sample_source"] += 1
season, episode = _extract_season_episode(op)
if season is not None and season >= season_threshold:
flags.append("high_season")
counters["high_season"] += 1
if episode is not None and episode >= episode_threshold:
flags.append("high_episode")
counters["high_episode"] += 1
if op.has_conflict:
flags.append("conflict")
counters["conflicts"] += 1
if flags:
counters["high_risk_operations"] += 1
rows.append(
{
"index": str(idx),
"operation_type": op.operation_type,
"risk_flags": "|".join(flags),
"source_path": str(op.source_path),
"destination_path": str(op.destination_path) if op.destination_path else "",
"reason": op.reason,
}
)
return rows, counters
def save_review_csv(rows: list[dict[str, str]], output: Path) -> None:
"""Write review rows to CSV."""
output.parent.mkdir(parents=True, exist_ok=True)
fields = ["index", "operation_type", "risk_flags", "source_path", "destination_path", "reason"]
with open(output, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fields)
writer.writeheader()
writer.writerows(rows)
src/vlm/planner.py
+133 -10
View File
@@ -5,6 +5,7 @@ should be organized based on their parsed identities and configuration templates
"""
import json
import re
import uuid
from datetime import datetime
from pathlib import Path
@@ -12,7 +13,7 @@ from typing import Optional, Union
from vlm.config import Config
from vlm.duplicate_resolve import choose_keep_index
from vlm.utils import ensure_utc, utc_now
from vlm.utils import ensure_utc, is_within_root, sanitize_path_component, utc_now
from vlm.models import (
ExecutionPlan,
FileOperation,
@@ -22,22 +23,50 @@ from vlm.models import (
)
QUARANTINE_REASON_DUPLICATE = "重复项（已按外部评分保留一条）"
QUARANTINE_REASON_DUPLICATE_FALLBACK_QUALITY = "重复项（外部评分缺失/并列，按画质回退保留一条）"
QUARANTINE_REASON_DUPLICATE_BY_QUALITY = "重复项（已按画质保留最优）"
QUARANTINE_REASON_DUPLICATE_BY_REPUTATION_QUALITY_TIME = "重复项（按外部评分>画质>时间保留一条）"
NO_OP_REASON_SAMPLE_EXCLUDED = "Sample file excluded by plan include_sample_files=false"
NO_OP_REASON_SEASON_OUT_OF_RANGE = "Series needs manual review (season exceeds configured threshold)"
NO_OP_REASON_EPISODE_OUT_OF_RANGE = "Series needs manual review (episode exceeds configured threshold)"
def _is_sample_path(path: Path) -> bool:
"""Return True if path appears to be a sample clip."""
parts = [part.casefold() for part in path.parts]
if "sample" in parts:
return True
stem = path.stem.casefold()
return bool(re.search(r"(^|[\s._-])sample($|[\s._-])", stem))
def generate_plan(
identities: list[tuple[VideoFile, Union[MovieIdentity, SeriesIdentity, None]]],
config: Config,
analysis_data: Optional[dict] = None,
ignored_paths: Optional[set[str]] = None,
) -> ExecutionPlan:
"""Generate an execution plan from parsed identities; optionally apply analysis duplicates.
When analysis_data is provided and duplicate_keep is not "manual", duplicate groups
are resolved (one kept, rest quarantined) according to config.duplicate_keep.
If ignored_paths is provided, files in that set will generate no-op operations.
"""
operations = []
for video_file, identity in identities:
operation = _create_operation(video_file, identity, config)
# Check if file is ignored in state
if ignored_paths and str(video_file.path) in ignored_paths:
operation = FileOperation(
operation_type="no-op",
source_path=video_file.path,
destination_path=None,
reason="User marked as ignored in state",
has_conflict=False,
conflict_reason=None
)
else:
operation = _create_operation(video_file, identity, config)
operations.append(operation)
metadata: dict = {}
@@ -59,6 +88,9 @@ def generate_plan(
if identity is not None and isinstance(
identity, (MovieIdentity, SeriesIdentity)
):
if not config.plan_include_sample_files and _is_sample_path(identities[i][0].path):
# Keep sample files out of duplicate keep/quarantine competition by default.
continue
items.append((identities[i][0].path, identity))
valid_indices.append(i)
if not items:
@@ -73,10 +105,9 @@ def generate_plan(
continue
keep_identity_index = valid_indices[keep_idx]
quarantine_indices = set(valid_indices) - {keep_identity_index}
reason = (
QUARANTINE_REASON_DUPLICATE_BY_QUALITY
if config.duplicate_keep == "by_quality"
else QUARANTINE_REASON_DUPLICATE
reason = _select_duplicate_quarantine_reason(
config.duplicate_keep,
[identity for _, identity in items],
)
for i in quarantine_indices:
vf = identities[i][0]
@@ -119,6 +150,16 @@ def _create_operation(
Returns:
FileOperation specifying what to do with the file
"""
if not config.plan_include_sample_files and _is_sample_path(video_file.path):
return FileOperation(
operation_type="no-op",
source_path=video_file.path,
destination_path=None,
reason=NO_OP_REASON_SAMPLE_EXCLUDED,
has_conflict=False,
conflict_reason=None
)
# Handle anime category - generate no-op (v1 constraint)
if video_file.category == "anime":
return FileOperation(
@@ -208,9 +249,11 @@ def _create_movie_operation(
conflict_reason=None
)
safe_title = sanitize_path_component(identity.title, fallback="untitled")
# Apply movie directory template
target_dir = config.movie_template.format(
title=identity.title,
title=safe_title,
year=identity.year
)
@@ -219,13 +262,22 @@ def _create_movie_operation(
# Apply movie filename template
target_filename = config.movie_filename_template.format(
title=identity.title,
title=safe_title,
year=identity.year,
ext=ext
)
# Construct full destination path
destination = config.library_root / target_dir / target_filename
if not is_within_root(destination, config.library_root):
return FileOperation(
operation_type="no-op",
source_path=video_file.path,
destination_path=None,
reason=f"Unsafe destination outside library root: {destination}",
has_conflict=False,
conflict_reason=None
)
# Check if source and destination are the same
if video_file.path.resolve() == destination.resolve():
@@ -296,10 +348,30 @@ def _create_series_operation(
has_conflict=False,
conflict_reason=None
)
if identity.season > config.plan_max_season:
return FileOperation(
operation_type="no-op",
source_path=video_file.path,
destination_path=None,
reason=NO_OP_REASON_SEASON_OUT_OF_RANGE,
has_conflict=False,
conflict_reason=None
)
if any(ep > config.plan_max_episode for ep in identity.episodes):
return FileOperation(
operation_type="no-op",
source_path=video_file.path,
destination_path=None,
reason=NO_OP_REASON_EPISODE_OUT_OF_RANGE,
has_conflict=False,
conflict_reason=None
)
safe_title = sanitize_path_component(identity.title, fallback="untitled")
# Apply series directory template
target_dir = config.series_template.format(
title=identity.title,
title=safe_title,
season=identity.season
)
@@ -316,6 +388,15 @@ def _create_series_operation(
# Construct full destination path
destination = config.library_root / target_dir / target_filename
if not is_within_root(destination, config.library_root):
return FileOperation(
operation_type="no-op",
source_path=video_file.path,
destination_path=None,
reason=f"Unsafe destination outside library root: {destination}",
has_conflict=False,
conflict_reason=None
)
# Check if source and destination are the same
if video_file.path.resolve() == destination.resolve():
@@ -412,9 +493,51 @@ def _generate_human_summary(
gaps = metadata.get("completeness_seasons_with_gaps", 0)
if dup or gaps:
parts.append(f"依据 analysis：重复组 {dup} 个；剧集缺口 {gaps} 季。")
quarantine_lines = _build_quarantine_recommendation_lines(operations)
if quarantine_lines:
parts.append("删除建议（仅隔离建议，执行删除前请人工复核）：")
parts.extend(quarantine_lines)
return "\n".join(parts)
def _select_duplicate_quarantine_reason(
strategy: str,
identities: list[Union[MovieIdentity, SeriesIdentity]],
) -> str:
"""Choose a user-facing reason string for duplicate quarantine."""
if strategy == "by_quality":
return QUARANTINE_REASON_DUPLICATE_BY_QUALITY
if strategy == "by_reputation_quality_time":
return QUARANTINE_REASON_DUPLICATE_BY_REPUTATION_QUALITY_TIME
if strategy == "by_reputation":
rep_values = [i.reputation_score for i in identities if i.reputation_score is not None]
if len(rep_values) <= 1:
return QUARANTINE_REASON_DUPLICATE_FALLBACK_QUALITY
return QUARANTINE_REASON_DUPLICATE
return QUARANTINE_REASON_DUPLICATE
def _build_quarantine_recommendation_lines(operations: list[FileOperation], limit: int = 20) -> list[str]:
"""Build human-readable quarantine recommendations with reasons."""
quarantines = [op for op in operations if op.operation_type == "quarantine"]
if not quarantines:
return []
lines: list[str] = []
for op in quarantines[:limit]:
risk_tags = []
name_l = op.source_path.name.casefold()
if any(tok in name_l for tok in ("disc1", "disc2", "part.", " part ", "cd1", "cd2")):
risk_tags.append("疑似多碟/分段文件，建议勿直接删除")
if "评分缺失/并列" in op.reason:
risk_tags.append("评分依据不足，已回退画质规则")
risk_text = f"（{'; '.join(risk_tags)}）" if risk_tags else ""
lines.append(f" - {op.source_path.name} -> {op.reason}{risk_text}")
if len(quarantines) > limit:
lines.append(f" - 其余 {len(quarantines) - limit} 条请查看 plan.json 的 quarantine 操作。")
return lines
def save_plan(plan: ExecutionPlan, output_path: Path) -> None:
"""Save execution plan to JSON file.
@@ -495,4 +618,4 @@ def load_plan(input_path: Path) -> ExecutionPlan:
summary_by_reason=plan_dict.get("summary_by_reason", {}),
human_summary=plan_dict.get("human_summary", ""),
metadata=plan_dict.get("metadata", {}),
)
)
src/vlm/quarantine.py
+79 -26
View File
@@ -19,6 +19,7 @@ from .config import Config
from .utils import utc_now
from .logging_config import get_logger, log_operation
from .models import QuarantineEntry, QuarantineManifest, OperationResult, FileOperation
from .scanner import categorize_file
class QuarantineManager:
@@ -127,9 +128,39 @@ class QuarantineManager:
)
raise ValueError(error_msg)
# Get category root directory
category_root = self.config.library_root / category
# Get the actual category directory name from the file path
# (not the category name, which may differ due to category mappings)
try:
relative_from_lib = file_path.relative_to(self.config.library_root)
actual_category_dir = relative_from_lib.parts[0] if relative_from_lib.parts else None
except ValueError:
actual_category_dir = None
if not actual_category_dir:
error_msg = f"File is not within library root {self.config.library_root}: {file_path}"
log_operation(
self.logger,
logging.ERROR,
error_msg,
operation_type="quarantine",
file_path=file_path
)
return OperationResult(
operation=FileOperation(
operation_type="quarantine",
source_path=file_path,
destination_path=None,
reason=reason or "Invalid path",
has_conflict=False
),
success=False,
error_message=error_msg,
executed_at=executed_at
)
# Get category root directory using actual directory name
category_root = self.config.library_root / actual_category_dir
# Determine relative path from category root
try:
relative_path = file_path.relative_to(category_root)
@@ -319,31 +350,51 @@ class QuarantineManager:
def _determine_category(self, file_path: Path) -> str:
"""Determine the category of a file based on its path.
Uses the configured category mappings to support custom directory names.
Args:
file_path: Path to the file
Returns:
Category name ("movie", "series", "anime", "other")
"""
# Get path relative to library root
try:
relative_path = file_path.relative_to(self.config.library_root)
except ValueError:
return "other"
# First component of relative path is the category
parts = relative_path.parts
if not parts:
return "other"
category = parts[0].lower()
# Validate category
if category in ("movie", "series", "anime", "other"):
return category
else:
return "other"
# Use the same categorization logic as the scanner
categories_config = self.config.categories or {
"movie": ["movie"],
"series": ["series"],
"anime": ["anime"]
}
return categorize_file(file_path, self.config.library_root, categories_config)
def _find_category_dir(self, category: str) -> Optional[str]:
"""Find the actual directory name for a given category.
Scans the library root for directories that match the category mapping.
Args:
category: Category name ("movie" or "series")
Returns:
Actual directory name if found, or category name as fallback
"""
categories_config = self.config.categories or {
"movie": ["movie"],
"series": ["series"],
"anime": ["anime"]
}
# Get the list of possible directory names for this category
dir_names = categories_config.get(category, [category])
# Check which one actually exists in the library root
for dir_name in dir_names:
candidate = self.config.library_root / dir_name
if candidate.exists() and candidate.is_dir():
return dir_name
# Fallback to category name itself
return category
def _resolve_conflict(self, quarantine_path: Path) -> Path:
"""Resolve destination conflicts by appending numeric suffix.
@@ -387,14 +438,16 @@ class QuarantineManager:
def _get_manifest_path(self, category: str) -> Path:
"""Get the path to the manifest file for a category.
Args:
category: Category name ("movie" or "series")
Returns:
Path to the manifest.json file
"""
category_root = self.config.library_root / category
# Find the actual directory name for this category
actual_dir = self._find_category_dir(category)
category_root = self.config.library_root / actual_dir
quarantine_root = category_root / self.config.quarantine_dir
return quarantine_root / "manifest.json"
src/vlm/reports.py
+13 -1
View File
@@ -333,10 +333,22 @@ def _generate_duplicate_text(
lines.append("No duplicate files detected.")
return "\n".join(lines)
def _group_max_size(group: DuplicateGroup) -> int:
quality_sizes = [
int(item.get("size_bytes", 0) or 0)
for item in group.quality_comparison
if isinstance(item, dict)
]
if quality_sizes:
return max(quality_sizes)
if group.files:
return max(f.size_bytes for f in group.files)
return 0
# Sort by largest file size first
sorted_duplicates = sorted(
duplicates,
key=lambda g: max(f.size_bytes for f in g.files),
key=_group_max_size,
reverse=True
)
src/vlm/scanner.py
+53 -17
View File
@@ -6,6 +6,7 @@ directory structure.
"""
import csv
from concurrent.futures import ThreadPoolExecutor, as_completed
import json
import logging
import os
@@ -72,26 +73,61 @@ def scan_library(
if progress_callback is not None:
progress_callback(0, total_paths)
for index, file_path in enumerate(discovered_paths, start=1):
video_file = _create_video_file(
file_path,
root,
config.categories,
include_video_metadata=include_video_metadata,
metadata_cache=metadata_cache
)
if video_file is None:
max_workers = max(1, min(config.enrichment_max_concurrency, total_paths or 1))
use_parallel = include_video_metadata and total_paths > 1 and max_workers > 1
if use_parallel:
indexed_results: list[tuple[int, VideoFile]] = []
with ThreadPoolExecutor(max_workers=max_workers) as pool:
future_to_index = {
pool.submit(
_create_video_file,
file_path,
root,
config.categories,
include_video_metadata,
metadata_cache,
): index
for index, file_path in enumerate(discovered_paths, start=1)
}
for completed_count, future in enumerate(as_completed(future_to_index), start=1):
index = future_to_index[future]
try:
video_file = future.result()
except Exception as exc:
logger.error(f"Failed to scan file #{index}: {exc}")
video_file = None
if video_file is not None:
indexed_results.append((index, video_file))
file_count += 1
if progress_callback is not None:
progress_callback(completed_count, total_paths)
if file_count % 100 == 0 and file_count > 0:
logger.debug(f"Scanned {file_count} files so far...")
indexed_results.sort(key=lambda item: item[0])
video_files = [item[1] for item in indexed_results]
else:
for index, file_path in enumerate(discovered_paths, start=1):
video_file = _create_video_file(
file_path,
root,
config.categories,
include_video_metadata=include_video_metadata,
metadata_cache=metadata_cache
)
if video_file is None:
if progress_callback is not None:
progress_callback(index, total_paths)
continue
video_files.append(video_file)
file_count += 1
if progress_callback is not None:
progress_callback(index, total_paths)
continue
video_files.append(video_file)
file_count += 1
if progress_callback is not None:
progress_callback(index, total_paths)
if file_count % 100 == 0:
logger.debug(f"Scanned {file_count} files so far...")
if file_count % 100 == 0:
logger.debug(f"Scanned {file_count} files so far...")
logger.info(f"Scan complete. Found {file_count} video files")
return video_files
src/vlm/state.py
+37 -10
View File
@@ -5,12 +5,14 @@ and user decisions throughout the workflow.
"""
import json
import os
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Optional
from vlm.models import FileState, StateStore
from vlm.utils import ensure_utc, utc_now
from vlm.utils import canonical_path, canonical_path_str, ensure_utc, utc_now
# Valid status values
@@ -35,9 +37,10 @@ def load_state(path: Path) -> StateStore:
# Parse states dictionary (normalize naive datetime to UTC)
states = {}
for file_path_str, state_data in data.get('states', {}).items():
states[file_path_str] = FileState(
file_path=Path(state_data['file_path']),
for _file_path_str, state_data in data.get('states', {}).items():
file_path = canonical_path(Path(state_data['file_path']))
states[canonical_path_str(file_path)] = FileState(
file_path=file_path,
status=state_data['status'],
reason=state_data.get('reason'),
updated_at=ensure_utc(datetime.fromisoformat(state_data['updated_at']))
@@ -76,8 +79,27 @@ def save_state(store: StateStore, path: Path) -> None:
'last_updated': store.last_updated.isoformat()
}
with open(path, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
temp_file = None
try:
with tempfile.NamedTemporaryFile(
mode='w',
encoding='utf-8',
dir=str(path.parent),
prefix=f".{path.name}.",
suffix=".tmp",
delete=False,
) as f:
temp_file = Path(f.name)
json.dump(data, f, indent=2, ensure_ascii=False)
f.flush()
os.fsync(f.fileno())
os.replace(temp_file, path)
finally:
if temp_file and temp_file.exists():
try:
temp_file.unlink()
except OSError:
pass
class StateManager:
@@ -105,6 +127,10 @@ class StateManager:
last_updated=utc_now()
)
@staticmethod
def _canonical_key(file_path: Path) -> str:
return canonical_path_str(file_path)
def get_file_state(self, file_path: Path) -> Optional[FileState]:
"""Get state for a specific file.
@@ -114,7 +140,7 @@ class StateManager:
Returns:
FileState object if found, None otherwise
"""
file_path_str = str(file_path)
file_path_str = self._canonical_key(file_path)
return self.store.states.get(file_path_str)
def set_file_state(self, file_path: Path, status: str, reason: Optional[str] = None) -> None:
@@ -136,11 +162,12 @@ class StateManager:
f"Invalid status '{status}'. Must be one of: {', '.join(sorted(VALID_STATUSES))}"
)
file_path_str = str(file_path)
file_path_str = self._canonical_key(file_path)
canonical_file_path = canonical_path(file_path)
now = utc_now()
self.store.states[file_path_str] = FileState(
file_path=file_path,
file_path=canonical_file_path,
status=status,
reason=reason,
updated_at=now
@@ -168,7 +195,7 @@ class StateManager:
Args:
file_path: Path to the file
"""
file_path_str = str(file_path)
file_path_str = self._canonical_key(file_path)
if file_path_str in self.store.states:
del self.store.states[file_path_str]
self.store.last_updated = utc_now()
src/vlm/transaction.py
+103
View File
@@ -0,0 +1,103 @@
"""Transaction logging for Video Library Manager.
This module provides a simple mechanism for logging intended file operations
before they are executed, allowing for crash recovery and audit trails.
"""
import json
from datetime import datetime
from pathlib import Path
from typing import Any, Optional, Dict, List
from uuid import uuid4
from vlm.models import ExecutionPlan, FileOperation
from vlm.utils import utc_now
class TransactionLog:
"""Manages a transaction log for file operations."""
def __init__(self, log_path: Path):
"""Initialize the transaction log.
Args:
log_path: Path to the transaction log JSON file
"""
self.log_path = log_path
self.data: Dict[str, Any] = {
"vlm_schema_version": "1.0",
"transaction_id": str(uuid4()),
"started_at": None,
"status": "pending", # pending, in_progress, completed, failed
"operations": []
}
def start_transaction(self, plan: ExecutionPlan) -> None:
"""Start a new transaction based on an execution plan.
Args:
plan: The execution plan being executed
"""
self.data["started_at"] = utc_now().isoformat()
self.data["status"] = "in_progress"
self.data["execution_plan_id"] = plan.plan_id
self.data["operations"] = [
{
"id": i,
"operation_type": op.operation_type,
"source_path": str(op.source_path),
"destination_path": str(op.destination_path) if op.destination_path else None,
"status": "pending", # pending, completed, failed
"error": None
}
for i, op in enumerate(plan.operations)
]
self.save()
def mark_operation_complete(self, op_index: int, success: bool, error: Optional[str] = None) -> None:
"""Mark a single operation as complete in the log.
Args:
op_index: Index of the operation in the log
success: Whether the operation succeeded
error: Error message if failed
"""
if 0 <= op_index < len(self.data["operations"]):
op = self.data["operations"][op_index]
op["status"] = "completed" if success else "failed"
op["error"] = error
op["completed_at"] = utc_now().isoformat()
self.save()
def complete_transaction(self, status: str = "completed") -> None:
"""Finalize the transaction.
Args:
status: Final status (completed or failed)
"""
self.data["status"] = status
self.data["finished_at"] = utc_now().isoformat()
self.save()
def save(self) -> None:
"""Save the log to disk."""
self.log_path.parent.mkdir(parents=True, exist_ok=True)
with open(self.log_path, 'w', encoding='utf-8') as f:
json.dump(self.data, f, indent=2, ensure_ascii=False)
@classmethod
def load(cls, log_path: Path) -> 'TransactionLog':
"""Load an existing transaction log.
Args:
log_path: Path to the log file
Returns:
TransactionLog instance with loaded data
"""
log = cls(log_path)
if log_path.exists():
with open(log_path, 'r', encoding='utf-8') as f:
log.data = json.load(f)
return log
src/vlm/utils.py
+24
View File
@@ -2,6 +2,7 @@
from datetime import datetime, timezone
from pathlib import Path
import re
def utc_now() -> datetime:
@@ -47,3 +48,26 @@ def canonical_path_str(path: Path) -> str:
String representation of canonical path
"""
return str(canonical_path(path))
_PATH_SEPARATORS_PATTERN = re.compile(r"[\\/]+")
_CONTROL_CHARS_PATTERN = re.compile(r"[\x00-\x1f\x7f]")
def sanitize_path_component(value: str, fallback: str = "unknown") -> str:
"""Sanitize untrusted text for safe use as a single path component."""
text = (value or "").strip()
text = _CONTROL_CHARS_PATTERN.sub("", text)
text = _PATH_SEPARATORS_PATTERN.sub(" ", text)
text = text.replace("..", " ")
text = " ".join(text.split()).strip(" .")
return text or fallback
def is_within_root(path: Path, root: Path) -> bool:
"""Return True when path is located inside root after canonicalization."""
try:
canonical_path(path).relative_to(canonical_path(root))
return True
except ValueError:
return False