commit remaining modified project files
This commit is contained in:
windyboy
@@ -84,6 +84,11 @@ def _command_error(ctx: CLIContext, user_message: str, logger_message: str, *, e
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def _review_plan_tui_streams_ok() -> bool:
|
||||
"""Return True if stdin/stdout appear to be an interactive terminal."""
|
||||
return sys.stdin.isatty() and sys.stdout.isatty()
|
||||
|
||||
|
||||
def _load_or_create_config(config: Path) -> Config:
|
||||
"""Load configuration from disk or create a default config file."""
|
||||
if config.exists():
|
||||
@@ -527,6 +532,12 @@ def plan(ctx: CLIContext, input: Path, output: Path, analysis: Path | None):
|
||||
default=False,
|
||||
help='Show all high-risk operations in console preview'
|
||||
)
|
||||
@click.option(
|
||||
'--tui',
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help='Interactive Textual UI (requires: uv pip install -e ".[tui]")'
|
||||
)
|
||||
@pass_context
|
||||
def review_plan_cmd(
|
||||
ctx: CLIContext,
|
||||
@@ -536,6 +547,7 @@ def review_plan_cmd(
|
||||
episode_threshold: int,
|
||||
preview_limit: int,
|
||||
show_all: bool,
|
||||
tui: bool,
|
||||
):
|
||||
"""Review a plan and export high-risk operations for manual confirmation."""
|
||||
from vlm.planner import load_plan
|
||||
@@ -552,6 +564,19 @@ def review_plan_cmd(
|
||||
click.echo("Error: --preview-limit must be >= 1", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
if tui:
|
||||
if not _review_plan_tui_streams_ok():
|
||||
click.echo("Error: --tui requires an interactive terminal (TTY)", err=True)
|
||||
sys.exit(1)
|
||||
try:
|
||||
from textual.app import App as _TextualApp # noqa: F401
|
||||
except ImportError:
|
||||
click.echo(
|
||||
'Error: Textual is not installed. Install with: uv pip install -e ".[tui]"',
|
||||
err=True,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
click.echo(f"Loading plan: {input}")
|
||||
execution_plan = load_plan(input)
|
||||
rows, counters = review_plan(
|
||||
@@ -560,6 +585,30 @@ def review_plan_cmd(
|
||||
episode_threshold=episode_threshold,
|
||||
)
|
||||
|
||||
if tui:
|
||||
from vlm.review_tui import ReviewTUIContext, run_plan_review_tui
|
||||
|
||||
tui_ctx = ReviewTUIContext(
|
||||
rows=rows,
|
||||
counters=counters,
|
||||
library_root=ctx.config.library_root,
|
||||
output_csv=output,
|
||||
plan_input=input,
|
||||
summary_text=preferred_plan_summary(execution_plan),
|
||||
)
|
||||
rc = run_plan_review_tui(tui_ctx)
|
||||
if rc != 0:
|
||||
click.echo("Plan review aborted (no CSV written).", err=True)
|
||||
sys.exit(rc)
|
||||
click.echo(f"Saved manual review CSV to: {output}")
|
||||
logger.info(
|
||||
"Plan review TUI completed: total=%s high_risk=%s output=%s",
|
||||
counters["total_operations"],
|
||||
counters["high_risk_operations"],
|
||||
output,
|
||||
)
|
||||
return
|
||||
|
||||
save_review_csv(rows, output)
|
||||
|
||||
click.echo()
|
||||
|
||||
+20
-10
@@ -13,7 +13,13 @@ from vlm.io import (
|
||||
load_inventory_csv,
|
||||
save_analysis_json,
|
||||
)
|
||||
from vlm.models import MovieIdentity, SeriesIdentity
|
||||
from vlm.models import (
|
||||
AnalysisCompletenessRecord,
|
||||
AnalysisDuplicateIdentityRecord,
|
||||
AnalysisDuplicateRecord,
|
||||
MovieIdentity,
|
||||
SeriesIdentity,
|
||||
)
|
||||
|
||||
|
||||
def analyze_cmd(
|
||||
@@ -39,19 +45,16 @@ def analyze_cmd(
|
||||
click.echo()
|
||||
|
||||
inventory_files = load_inventory_csv(inventory) if inventory else None
|
||||
movie_identities, series_identities, video_files = identities_to_analysis_input(
|
||||
movie_pairs, series_pairs = identities_to_analysis_input(
|
||||
identities_data, inventory_files=inventory_files
|
||||
)
|
||||
series_identities = [identity for identity, _ in series_pairs]
|
||||
|
||||
click.echo("Analyzing series completeness...")
|
||||
completeness_results = analyze_series_completeness(series_identities)
|
||||
|
||||
click.echo("Detecting duplicates...")
|
||||
n_movies = len(movie_identities)
|
||||
identity_file_pairs = (
|
||||
list(zip(movie_identities, video_files[:n_movies]))
|
||||
+ list(zip(series_identities, video_files[n_movies:]))
|
||||
)
|
||||
identity_file_pairs = movie_pairs + series_pairs
|
||||
duplicate_groups = detect_duplicates(identity_file_pairs)
|
||||
|
||||
click.echo()
|
||||
@@ -71,7 +74,7 @@ def analyze_cmd(
|
||||
click.echo(f"Saving analysis results to: {output}")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
completeness_list = [
|
||||
completeness_list: list[AnalysisCompletenessRecord] = [
|
||||
{
|
||||
"series_title": c.series_title,
|
||||
"season": c.season,
|
||||
@@ -80,14 +83,21 @@ def analyze_cmd(
|
||||
}
|
||||
for c in completeness_results
|
||||
]
|
||||
duplicates_list = []
|
||||
duplicates_list: list[AnalysisDuplicateRecord] = []
|
||||
for d in duplicate_groups:
|
||||
if isinstance(d.identity, MovieIdentity):
|
||||
identity_info = {"type": "movie", "title": d.identity.title, "year": d.identity.year}
|
||||
identity_info: AnalysisDuplicateIdentityRecord = {
|
||||
"type": "movie",
|
||||
"title": d.identity.title,
|
||||
"year": d.identity.year,
|
||||
"season": None,
|
||||
"episodes": [],
|
||||
}
|
||||
else:
|
||||
identity_info = {
|
||||
"type": "series",
|
||||
"title": d.identity.title,
|
||||
"year": None,
|
||||
"season": d.identity.season,
|
||||
"episodes": d.identity.episodes,
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import click
|
||||
|
||||
from vlm.context import CLIContext
|
||||
from vlm.io import load_inventory_csv, save_identities_json
|
||||
from vlm.models import IdentityRecord, MovieIdentityRecord, ParsedIdentitiesJSON, SeriesIdentityRecord, VideoFile
|
||||
from vlm.parser import parse_movie, parse_series
|
||||
from vlm.utils import utc_now
|
||||
|
||||
@@ -20,7 +21,7 @@ def parse_cmd(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Pa
|
||||
|
||||
click.echo(f"Parsing identities from: {input}")
|
||||
|
||||
path_to_metadata: dict[str, object] = {}
|
||||
path_to_metadata: dict[str, VideoFile] = {}
|
||||
if inventory:
|
||||
click.echo(f"Loading video metadata from: {inventory}")
|
||||
inventory_files = load_inventory_csv(inventory)
|
||||
@@ -30,7 +31,7 @@ def parse_cmd(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Pa
|
||||
click.echo()
|
||||
|
||||
inventory_files = load_inventory_csv(input)
|
||||
video_files = [
|
||||
video_files: list[dict[str, str]] = [
|
||||
{
|
||||
"path": str(vf.path),
|
||||
"filename": vf.filename,
|
||||
@@ -42,10 +43,10 @@ def parse_cmd(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Pa
|
||||
click.echo(f"Loaded {len(video_files)} files from inventory")
|
||||
click.echo()
|
||||
|
||||
movie_identities: list[dict] = []
|
||||
series_identities: list[dict] = []
|
||||
anime_files: list[dict] = []
|
||||
other_files: list[dict] = []
|
||||
movie_identities: list[MovieIdentityRecord] = []
|
||||
series_identities: list[SeriesIdentityRecord] = []
|
||||
anime_files: list[IdentityRecord] = []
|
||||
other_files: list[IdentityRecord] = []
|
||||
|
||||
def get_video_metadata(file_path: str) -> dict:
|
||||
"""Extract video metadata from inventory if available."""
|
||||
@@ -141,7 +142,7 @@ def parse_cmd(ctx: CLIContext, input: Path, output: Path, inventory: Optional[Pa
|
||||
generation_timestamp = utc_now().strftime("%Y-%m-%dT%H:%M:%S")
|
||||
schema_version = "2.0" if path_to_metadata else "1.0"
|
||||
|
||||
identities_data = {
|
||||
identities_data: ParsedIdentitiesJSON = {
|
||||
"vlm_schema_version": schema_version,
|
||||
"metadata": {
|
||||
"generated": generation_timestamp,
|
||||
|
||||
@@ -82,9 +82,12 @@ def plan_cmd(ctx: CLIContext, input: Path, output: Path, analysis: Optional[Path
|
||||
click.echo(" Review the plan file for details on conflicting operations.")
|
||||
|
||||
# Check for directory warnings
|
||||
directory_warning = execution_plan.metadata.get("directory_warning", False)
|
||||
validation_snapshot = execution_plan.metadata.get("validation_snapshot", {})
|
||||
if not isinstance(validation_snapshot, dict):
|
||||
validation_snapshot = {}
|
||||
directory_warning = bool(validation_snapshot.get("directory_warning", False))
|
||||
if directory_warning:
|
||||
emptied_dirs = execution_plan.metadata.get("emptied_directories", [])
|
||||
emptied_dirs = validation_snapshot.get("emptied_directories", [])
|
||||
click.echo()
|
||||
click.echo(f" ⚠️ Directory preservation warning: {len(emptied_dirs)} directories will be emptied")
|
||||
click.echo(" These directories will be preserved but may be empty after execution.")
|
||||
|
||||
@@ -6,14 +6,7 @@ from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
from vlm.models import MovieIdentity, SeriesIdentity
|
||||
|
||||
|
||||
def _is_sample_path(path: Path) -> bool:
|
||||
"""Identify likely sample clips by path component or filename token."""
|
||||
parts = [part.casefold() for part in path.parts]
|
||||
if "sample" in parts:
|
||||
return True
|
||||
return bool(re.search(r"(^|[\s._-])sample($|[\s._-])", path.stem.casefold()))
|
||||
from vlm.utils import is_sample_path
|
||||
|
||||
|
||||
def choose_keep_index(
|
||||
@@ -125,7 +118,7 @@ def _by_quality_index(
|
||||
def key(idx_item: tuple[int, tuple[Path, Union[MovieIdentity, SeriesIdentity]]]) -> tuple:
|
||||
idx, (path, _) = idx_item
|
||||
qc = quality_comparison[idx] if idx < len(quality_comparison) else {}
|
||||
is_sample = _is_sample_path(path)
|
||||
is_sample = is_sample_path(path)
|
||||
return (is_sample, *quality_key(path, qc, idx))
|
||||
|
||||
indexed = list(enumerate(items))
|
||||
@@ -156,7 +149,7 @@ def _by_reputation_index(
|
||||
has_rep = identity.reputation_score is not None
|
||||
score = identity.reputation_score if identity.reputation_score is not None else -1.0
|
||||
votes = identity.reputation_votes if identity.reputation_votes is not None else -1
|
||||
is_sample = _is_sample_path(path)
|
||||
is_sample = is_sample_path(path)
|
||||
# Prefer non-sample, then reputation, then better quality, then lower index.
|
||||
return (is_sample, not has_rep, -score, -votes, *quality_key(path, qc, idx))
|
||||
|
||||
@@ -199,7 +192,7 @@ def _by_reputation_quality_time_index(
|
||||
score = identity.reputation_score if identity.reputation_score is not None else -1.0
|
||||
votes = identity.reputation_votes if identity.reputation_votes is not None else -1
|
||||
modified_ts = _parse_modified_timestamp(qc.get("modified_timestamp"))
|
||||
is_sample = _is_sample_path(path)
|
||||
is_sample = is_sample_path(path)
|
||||
return (
|
||||
is_sample,
|
||||
not has_rep,
|
||||
|
||||
+248
-75
@@ -6,7 +6,17 @@ import json
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
from vlm.models import MovieIdentity, SeriesIdentity, VideoFile
|
||||
from vlm.models import (
|
||||
AnalysisJSON,
|
||||
AnalysisCompletenessRecord,
|
||||
AnalysisDuplicateRecord,
|
||||
MovieIdentity,
|
||||
MovieIdentityRecord,
|
||||
ParsedIdentitiesJSON,
|
||||
SeriesIdentity,
|
||||
SeriesIdentityRecord,
|
||||
VideoFile,
|
||||
)
|
||||
from vlm.utils import utc_now
|
||||
|
||||
# Re-export scanner CSV functions so CLI and others use a single I/O entry point
|
||||
@@ -39,28 +49,230 @@ def save_json_file(data: dict, path: Path) -> None:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def load_analysis_json(path: Path) -> dict:
|
||||
def _ensure_dict(value: object, label: str) -> dict:
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _ensure_list(value: object, label: str) -> list:
|
||||
if not isinstance(value, list):
|
||||
raise ValueError(f"{label} must be a list")
|
||||
return value
|
||||
|
||||
|
||||
def _ensure_str(value: object, label: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f"{label} must be a string")
|
||||
return value
|
||||
|
||||
|
||||
def _ensure_bool(value: object, label: str) -> bool:
|
||||
if not isinstance(value, bool):
|
||||
raise ValueError(f"{label} must be a boolean")
|
||||
return value
|
||||
|
||||
|
||||
def _ensure_int(value: object, label: str, *, allow_none: bool = False) -> int | None:
|
||||
if value is None and allow_none:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ValueError(f"{label} must be an integer")
|
||||
return value
|
||||
|
||||
|
||||
def _ensure_float(value: object, label: str, *, allow_none: bool = False) -> float | None:
|
||||
if value is None and allow_none:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{label} must be a number")
|
||||
return float(value)
|
||||
|
||||
|
||||
def _ensure_string_dict(value: object, label: str) -> dict[str, str]:
|
||||
mapping = _ensure_dict(value, label)
|
||||
for key, item in mapping.items():
|
||||
if not isinstance(key, str) or not isinstance(item, str):
|
||||
raise ValueError(f"{label} must map strings to strings")
|
||||
return mapping # type: ignore[return-value]
|
||||
|
||||
|
||||
def _validate_video_metadata(record: dict, label: str) -> None:
|
||||
if "video_metadata" not in record:
|
||||
return
|
||||
metadata = _ensure_dict(record["video_metadata"], f"{label}.video_metadata")
|
||||
if "size_bytes" in metadata:
|
||||
_ensure_int(metadata["size_bytes"], f"{label}.video_metadata.size_bytes")
|
||||
if "modified_timestamp" in metadata:
|
||||
_ensure_str(metadata["modified_timestamp"], f"{label}.video_metadata.modified_timestamp")
|
||||
if "resolution" in metadata and metadata["resolution"] is not None:
|
||||
_ensure_str(metadata["resolution"], f"{label}.video_metadata.resolution")
|
||||
if "codec" in metadata and metadata["codec"] is not None:
|
||||
_ensure_str(metadata["codec"], f"{label}.video_metadata.codec")
|
||||
if "duration_seconds" in metadata:
|
||||
_ensure_float(
|
||||
metadata["duration_seconds"],
|
||||
f"{label}.video_metadata.duration_seconds",
|
||||
allow_none=True,
|
||||
)
|
||||
if "bitrate_kbps" in metadata:
|
||||
_ensure_int(
|
||||
metadata["bitrate_kbps"],
|
||||
f"{label}.video_metadata.bitrate_kbps",
|
||||
allow_none=True,
|
||||
)
|
||||
|
||||
|
||||
def _validate_identity_record(record: object, *, label: str, expect_kind: str | None = None) -> dict:
|
||||
mapping = _ensure_dict(record, label)
|
||||
_ensure_str(mapping.get("path"), f"{label}.path")
|
||||
_ensure_str(mapping.get("filename"), f"{label}.filename")
|
||||
_ensure_str(mapping.get("category"), f"{label}.category")
|
||||
|
||||
if "review_status" in mapping:
|
||||
review_status = _ensure_str(mapping["review_status"], f"{label}.review_status")
|
||||
if review_status not in {"pending", "approved", "rejected"}:
|
||||
raise ValueError(f"{label}.review_status must be one of pending, approved, rejected")
|
||||
|
||||
if expect_kind in {"movie", "series"}:
|
||||
_ensure_str(mapping.get("title"), f"{label}.title")
|
||||
_ensure_float(mapping.get("confidence"), f"{label}.confidence")
|
||||
_ensure_bool(mapping.get("needs_review"), f"{label}.needs_review")
|
||||
|
||||
if expect_kind == "movie":
|
||||
_ensure_int(mapping.get("year"), f"{label}.year", allow_none=True)
|
||||
elif expect_kind == "series":
|
||||
_ensure_int(mapping.get("season"), f"{label}.season", allow_none=True)
|
||||
episodes = _ensure_list(mapping.get("episodes"), f"{label}.episodes")
|
||||
for idx, episode in enumerate(episodes):
|
||||
_ensure_int(episode, f"{label}.episodes[{idx}]")
|
||||
|
||||
if "provider_metadata" in mapping:
|
||||
_ensure_string_dict(mapping["provider_metadata"], f"{label}.provider_metadata")
|
||||
|
||||
_validate_video_metadata(mapping, label)
|
||||
return mapping
|
||||
|
||||
|
||||
def _validate_parsed_identities_json(data: object) -> ParsedIdentitiesJSON:
|
||||
mapping = _ensure_dict(data, "identities JSON")
|
||||
if "vlm_schema_version" in mapping and mapping["vlm_schema_version"] is not None:
|
||||
_ensure_str(mapping["vlm_schema_version"], "identities JSON.vlm_schema_version")
|
||||
_ensure_dict(mapping.get("metadata"), "identities JSON.metadata")
|
||||
|
||||
movies = _ensure_list(mapping.get("movies", []), "identities JSON.movies")
|
||||
series = _ensure_list(mapping.get("series", []), "identities JSON.series")
|
||||
anime = _ensure_list(mapping.get("anime", []), "identities JSON.anime")
|
||||
other = _ensure_list(mapping.get("other", []), "identities JSON.other")
|
||||
|
||||
mapping["movies"] = [
|
||||
_validate_identity_record(movie, label=f"identities JSON.movies[{idx}]", expect_kind="movie")
|
||||
for idx, movie in enumerate(movies)
|
||||
]
|
||||
mapping["series"] = [
|
||||
_validate_identity_record(item, label=f"identities JSON.series[{idx}]", expect_kind="series")
|
||||
for idx, item in enumerate(series)
|
||||
]
|
||||
mapping["anime"] = [
|
||||
_validate_identity_record(item, label=f"identities JSON.anime[{idx}]", expect_kind=None)
|
||||
for idx, item in enumerate(anime)
|
||||
]
|
||||
mapping["other"] = [
|
||||
_validate_identity_record(item, label=f"identities JSON.other[{idx}]", expect_kind=None)
|
||||
for idx, item in enumerate(other)
|
||||
]
|
||||
return mapping # type: ignore[return-value]
|
||||
|
||||
|
||||
def _validate_analysis_json(data: object) -> AnalysisJSON:
|
||||
mapping = _ensure_dict(data, "analysis JSON")
|
||||
if not isinstance(mapping.get("vlm_schema_version"), str):
|
||||
mapping["vlm_schema_version"] = "1.0"
|
||||
_ensure_dict(mapping.get("metadata"), "analysis JSON.metadata")
|
||||
|
||||
completeness = _ensure_list(mapping.get("completeness", []), "analysis JSON.completeness")
|
||||
duplicates = _ensure_list(mapping.get("duplicates", []), "analysis JSON.duplicates")
|
||||
|
||||
for idx, item in enumerate(completeness):
|
||||
completeness_item = _ensure_dict(item, f"analysis JSON.completeness[{idx}]")
|
||||
_ensure_str(completeness_item.get("series_title"), f"analysis JSON.completeness[{idx}].series_title")
|
||||
_ensure_int(completeness_item.get("season"), f"analysis JSON.completeness[{idx}].season")
|
||||
for field_name in ("episodes_found", "episodes_missing"):
|
||||
field_value = _ensure_list(completeness_item.get(field_name), f"analysis JSON.completeness[{idx}].{field_name}")
|
||||
for episode_idx, episode in enumerate(field_value):
|
||||
_ensure_int(episode, f"analysis JSON.completeness[{idx}].{field_name}[{episode_idx}]")
|
||||
|
||||
for idx, item in enumerate(duplicates):
|
||||
duplicate_item = _ensure_dict(item, f"analysis JSON.duplicates[{idx}]")
|
||||
identity = _ensure_dict(duplicate_item.get("identity"), f"analysis JSON.duplicates[{idx}].identity")
|
||||
_ensure_str(identity.get("type"), f"analysis JSON.duplicates[{idx}].identity.type")
|
||||
_ensure_str(identity.get("title"), f"analysis JSON.duplicates[{idx}].identity.title")
|
||||
_ensure_int(identity.get("year"), f"analysis JSON.duplicates[{idx}].identity.year", allow_none=True)
|
||||
_ensure_int(identity.get("season"), f"analysis JSON.duplicates[{idx}].identity.season", allow_none=True)
|
||||
episodes = _ensure_list(identity.get("episodes", []), f"analysis JSON.duplicates[{idx}].identity.episodes")
|
||||
for episode_idx, episode in enumerate(episodes):
|
||||
_ensure_int(episode, f"analysis JSON.duplicates[{idx}].identity.episodes[{episode_idx}]")
|
||||
files = _ensure_list(duplicate_item.get("files", []), f"analysis JSON.duplicates[{idx}].files")
|
||||
for file_idx, file_path in enumerate(files):
|
||||
_ensure_str(file_path, f"analysis JSON.duplicates[{idx}].files[{file_idx}]")
|
||||
quality = _ensure_list(duplicate_item.get("quality_comparison", []), f"analysis JSON.duplicates[{idx}].quality_comparison")
|
||||
for quality_idx, q_item in enumerate(quality):
|
||||
_ensure_dict(q_item, f"analysis JSON.duplicates[{idx}].quality_comparison[{quality_idx}]")
|
||||
|
||||
return mapping # type: ignore[return-value]
|
||||
|
||||
|
||||
def validate_plan_json(data: object) -> dict:
|
||||
"""Validate the on-disk execution plan schema."""
|
||||
mapping = _ensure_dict(data, "plan JSON")
|
||||
_ensure_str(mapping.get("vlm_schema_version"), "plan JSON.vlm_schema_version")
|
||||
_ensure_str(mapping.get("plan_id"), "plan JSON.plan_id")
|
||||
_ensure_str(mapping.get("created_at"), "plan JSON.created_at")
|
||||
_ensure_dict(mapping.get("summary"), "plan JSON.summary")
|
||||
if "summary_by_reason" in mapping:
|
||||
_ensure_dict(mapping["summary_by_reason"], "plan JSON.summary_by_reason")
|
||||
if "human_summary" in mapping:
|
||||
_ensure_str(mapping["human_summary"], "plan JSON.human_summary")
|
||||
if "metadata" in mapping:
|
||||
_ensure_dict(mapping["metadata"], "plan JSON.metadata")
|
||||
|
||||
operations = _ensure_list(mapping.get("operations", []), "plan JSON.operations")
|
||||
for idx, item in enumerate(operations):
|
||||
operation = _ensure_dict(item, f"plan JSON.operations[{idx}]")
|
||||
_ensure_str(operation.get("operation_type"), f"plan JSON.operations[{idx}].operation_type")
|
||||
_ensure_str(operation.get("source_path"), f"plan JSON.operations[{idx}].source_path")
|
||||
if operation.get("destination_path") is not None:
|
||||
_ensure_str(operation.get("destination_path"), f"plan JSON.operations[{idx}].destination_path")
|
||||
_ensure_str(operation.get("reason"), f"plan JSON.operations[{idx}].reason")
|
||||
_ensure_bool(operation.get("has_conflict"), f"plan JSON.operations[{idx}].has_conflict")
|
||||
if "conflict_reason" in operation and operation["conflict_reason"] is not None:
|
||||
_ensure_str(operation["conflict_reason"], f"plan JSON.operations[{idx}].conflict_reason")
|
||||
|
||||
return mapping
|
||||
|
||||
|
||||
def load_analysis_json(path: Path) -> AnalysisJSON:
|
||||
"""Load analysis result from JSON file (metadata, completeness, duplicates).
|
||||
|
||||
Caller should check file existence and handle missing/invalid keys.
|
||||
"""
|
||||
return load_json_file(path)
|
||||
return _validate_analysis_json(load_json_file(path))
|
||||
|
||||
|
||||
def load_identities_json(path: Path) -> dict:
|
||||
def load_identities_json(path: Path) -> ParsedIdentitiesJSON:
|
||||
"""Load identities from JSON file."""
|
||||
return load_json_file(path)
|
||||
return _validate_parsed_identities_json(load_json_file(path))
|
||||
|
||||
|
||||
def save_identities_json(data: dict, path: Path) -> None:
|
||||
def save_identities_json(data: ParsedIdentitiesJSON, path: Path) -> None:
|
||||
"""Save identities dict to JSON file."""
|
||||
save_json_file(data, path)
|
||||
save_json_file(_validate_parsed_identities_json(data), path)
|
||||
|
||||
|
||||
def save_analysis_json(
|
||||
*,
|
||||
completeness: list[dict],
|
||||
duplicates: list[dict],
|
||||
completeness: list[AnalysisCompletenessRecord],
|
||||
duplicates: list[AnalysisDuplicateRecord],
|
||||
source_identities: Path,
|
||||
total_movies: int,
|
||||
total_series: int,
|
||||
@@ -68,7 +280,7 @@ def save_analysis_json(
|
||||
) -> None:
|
||||
"""Save analysis result JSON using the canonical schema."""
|
||||
generation_timestamp = utc_now().strftime("%Y-%m-%dT%H:%M:%S")
|
||||
analysis_data = {
|
||||
analysis_data: AnalysisJSON = {
|
||||
"vlm_schema_version": "1.0",
|
||||
"metadata": {
|
||||
"generated": generation_timestamp,
|
||||
@@ -79,10 +291,10 @@ def save_analysis_json(
|
||||
"completeness": completeness,
|
||||
"duplicates": duplicates,
|
||||
}
|
||||
save_json_file(analysis_data, output)
|
||||
save_json_file(_validate_analysis_json(analysis_data), output)
|
||||
|
||||
|
||||
def _video_file_from_record(record: dict) -> VideoFile:
|
||||
def _video_file_from_record(record: MovieIdentityRecord | SeriesIdentityRecord) -> VideoFile:
|
||||
"""Build a VideoFile from an identities record.
|
||||
|
||||
If the record contains embedded video_metadata (v2 schema), use it.
|
||||
@@ -111,7 +323,16 @@ def _video_file_from_record(record: dict) -> VideoFile:
|
||||
)
|
||||
|
||||
|
||||
def _movie_identity_from_record(m: dict) -> MovieIdentity:
|
||||
def _video_file_from_inventory(
|
||||
record: MovieIdentityRecord | SeriesIdentityRecord,
|
||||
path_to_inventory: dict[str, VideoFile],
|
||||
) -> VideoFile:
|
||||
"""Build a VideoFile and prefer matching inventory metadata when available."""
|
||||
video_file = _video_file_from_record(record)
|
||||
return path_to_inventory.get(str(video_file.path), video_file)
|
||||
|
||||
|
||||
def _movie_identity_from_record(m: MovieIdentityRecord) -> MovieIdentity:
|
||||
"""Build MovieIdentity from identities JSON record."""
|
||||
is_approved = m.get("review_status") == "approved"
|
||||
return MovieIdentity(
|
||||
@@ -133,7 +354,7 @@ def _movie_identity_from_record(m: dict) -> MovieIdentity:
|
||||
)
|
||||
|
||||
|
||||
def _series_identity_from_record(s: dict) -> SeriesIdentity:
|
||||
def _series_identity_from_record(s: SeriesIdentityRecord) -> SeriesIdentity:
|
||||
"""Build SeriesIdentity from identities JSON record."""
|
||||
is_approved = s.get("review_status") == "approved"
|
||||
return SeriesIdentity(
|
||||
@@ -157,7 +378,7 @@ def _series_identity_from_record(s: dict) -> SeriesIdentity:
|
||||
|
||||
|
||||
def identities_to_plan_input(
|
||||
data: dict,
|
||||
data: ParsedIdentitiesJSON,
|
||||
) -> list[tuple[VideoFile, Union[MovieIdentity, SeriesIdentity, None]]]:
|
||||
"""Convert identities JSON dict to list of (VideoFile, Identity) for plan generator."""
|
||||
result: list[tuple[VideoFile, Union[MovieIdentity, SeriesIdentity, None]]] = []
|
||||
@@ -179,71 +400,23 @@ def identities_to_plan_input(
|
||||
|
||||
|
||||
def identities_to_analysis_input(
|
||||
data: dict,
|
||||
data: ParsedIdentitiesJSON,
|
||||
inventory_files: list[VideoFile] | None = None,
|
||||
) -> tuple[list[MovieIdentity], list[SeriesIdentity], list[VideoFile]]:
|
||||
"""Convert identities JSON dict to analysis inputs; optionally merge inventory metadata by path."""
|
||||
) -> tuple[list[tuple[MovieIdentity, VideoFile]], list[tuple[SeriesIdentity, VideoFile]]]:
|
||||
"""Convert identities JSON dict to explicit identity-file pairs for analysis."""
|
||||
movies_data = data.get("movies", [])
|
||||
series_data = data.get("series", [])
|
||||
movie_identities = []
|
||||
for m in movies_data:
|
||||
movie_identities.append(
|
||||
MovieIdentity(
|
||||
title=m["title"],
|
||||
year=m.get("year"),
|
||||
confidence=m["confidence"],
|
||||
needs_review=m["needs_review"],
|
||||
original_filename=m["filename"],
|
||||
)
|
||||
)
|
||||
series_identities = []
|
||||
for s in series_data:
|
||||
series_identities.append(
|
||||
SeriesIdentity(
|
||||
title=s["title"],
|
||||
season=s.get("season"),
|
||||
episodes=s.get("episodes", []),
|
||||
confidence=s["confidence"],
|
||||
needs_review=s["needs_review"],
|
||||
original_filename=s["filename"],
|
||||
)
|
||||
)
|
||||
video_files: list[VideoFile] = []
|
||||
path_to_inventory: dict[str, VideoFile] = {}
|
||||
if inventory_files:
|
||||
path_to_inventory = {str(vf.path): vf for vf in inventory_files}
|
||||
|
||||
movie_pairs: list[tuple[MovieIdentity, VideoFile]] = []
|
||||
for m in movies_data:
|
||||
vf = _video_file_from_record(m)
|
||||
if path_to_inventory:
|
||||
inv = path_to_inventory.get(str(vf.path))
|
||||
if inv:
|
||||
vf = VideoFile(
|
||||
path=inv.path,
|
||||
filename=inv.filename,
|
||||
size_bytes=inv.size_bytes,
|
||||
modified_timestamp=inv.modified_timestamp,
|
||||
category=inv.category,
|
||||
resolution=inv.resolution,
|
||||
codec=inv.codec,
|
||||
duration_seconds=inv.duration_seconds,
|
||||
bitrate_kbps=inv.bitrate_kbps,
|
||||
)
|
||||
video_files.append(vf)
|
||||
movie_pairs.append((_movie_identity_from_record(m), _video_file_from_inventory(m, path_to_inventory)))
|
||||
|
||||
series_pairs: list[tuple[SeriesIdentity, VideoFile]] = []
|
||||
for s in series_data:
|
||||
vf = _video_file_from_record(s)
|
||||
if path_to_inventory:
|
||||
inv = path_to_inventory.get(str(vf.path))
|
||||
if inv:
|
||||
vf = VideoFile(
|
||||
path=inv.path,
|
||||
filename=inv.filename,
|
||||
size_bytes=inv.size_bytes,
|
||||
modified_timestamp=inv.modified_timestamp,
|
||||
category=inv.category,
|
||||
resolution=inv.resolution,
|
||||
codec=inv.codec,
|
||||
duration_seconds=inv.duration_seconds,
|
||||
bitrate_kbps=inv.bitrate_kbps,
|
||||
)
|
||||
video_files.append(vf)
|
||||
return movie_identities, series_identities, video_files
|
||||
series_pairs.append((_series_identity_from_record(s), _video_file_from_inventory(s, path_to_inventory)))
|
||||
|
||||
return movie_pairs, series_pairs
|
||||
|
||||
|
||||
+89
-1
@@ -7,7 +7,7 @@ for representing video files and their parsed identities.
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from typing import Optional, TypedDict
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -275,3 +275,91 @@ class DuplicateGroup:
|
||||
identity: MovieIdentity | SeriesIdentity
|
||||
files: list[VideoFile]
|
||||
quality_comparison: list[dict]
|
||||
|
||||
|
||||
class VideoMetadataRecord(TypedDict, total=False):
|
||||
"""Embedded video metadata stored in identities records."""
|
||||
|
||||
size_bytes: int
|
||||
modified_timestamp: str
|
||||
resolution: str
|
||||
codec: str
|
||||
duration_seconds: float
|
||||
bitrate_kbps: int
|
||||
|
||||
|
||||
class IdentityRecord(TypedDict, total=False):
|
||||
"""Common record fields used in parsed/enriched JSON payloads."""
|
||||
|
||||
path: str
|
||||
filename: str
|
||||
category: str
|
||||
title: str
|
||||
display_title: str
|
||||
confidence: float
|
||||
needs_review: bool
|
||||
canonical_id: str
|
||||
title_zh: str
|
||||
title_en: str
|
||||
translation_source: str
|
||||
reputation_score: float
|
||||
reputation_votes: int
|
||||
reputation_source: str
|
||||
review_status: str
|
||||
enrichment_confidence: float
|
||||
provider_metadata: dict[str, str]
|
||||
video_metadata: VideoMetadataRecord
|
||||
note: str
|
||||
|
||||
|
||||
class MovieIdentityRecord(IdentityRecord, total=False):
|
||||
"""TypedDict for movie entries in identities.json."""
|
||||
|
||||
year: int | None
|
||||
|
||||
|
||||
class SeriesIdentityRecord(IdentityRecord, total=False):
|
||||
"""TypedDict for series entries in identities.json."""
|
||||
|
||||
season: int | None
|
||||
episodes: list[int]
|
||||
|
||||
|
||||
class ParsedIdentitiesJSON(TypedDict, total=False):
|
||||
"""Canonical parsed identities payload stored between pipeline stages."""
|
||||
|
||||
vlm_schema_version: str
|
||||
metadata: dict[str, object]
|
||||
movies: list[MovieIdentityRecord]
|
||||
series: list[SeriesIdentityRecord]
|
||||
anime: list[IdentityRecord]
|
||||
other: list[IdentityRecord]
|
||||
|
||||
|
||||
class AnalysisCompletenessRecord(TypedDict):
|
||||
series_title: str
|
||||
season: int
|
||||
episodes_found: list[int]
|
||||
episodes_missing: list[int]
|
||||
|
||||
|
||||
class AnalysisDuplicateIdentityRecord(TypedDict):
|
||||
type: str
|
||||
title: str
|
||||
year: int | None
|
||||
season: int | None
|
||||
episodes: list[int]
|
||||
|
||||
|
||||
class AnalysisDuplicateRecord(TypedDict):
|
||||
identity: AnalysisDuplicateIdentityRecord
|
||||
files: list[str]
|
||||
quality_comparison: list[dict]
|
||||
|
||||
|
||||
class AnalysisJSON(TypedDict, total=False):
|
||||
vlm_schema_version: str
|
||||
metadata: dict[str, object]
|
||||
completeness: list[AnalysisCompletenessRecord]
|
||||
duplicates: list[AnalysisDuplicateRecord]
|
||||
|
||||
|
||||
+41
-8
@@ -23,9 +23,11 @@ QUALITY_TAGS = [
|
||||
r'\b10bit\b', r'\b8bit\b',
|
||||
]
|
||||
|
||||
# Release group patterns (in brackets, but NOT years in parentheses)
|
||||
# Release group patterns (in brackets or parentheses at start/end)
|
||||
RELEASE_GROUP_PATTERNS = [
|
||||
r'\[[\w\s\-\.]+\]', # [RARBG], [YTS], etc.
|
||||
r'^\[[\w\s\-\.]+\]', # [Group] at start
|
||||
r'\[[\w\s\-\.]+\]$', # [Group] at end
|
||||
r'\b[\w\s\-\.]+[-_]Subs\b', # Group_Subs
|
||||
]
|
||||
|
||||
|
||||
@@ -56,6 +58,12 @@ def remove_release_groups(text: str) -> str:
|
||||
result = text
|
||||
for pattern in RELEASE_GROUP_PATTERNS:
|
||||
result = re.sub(pattern, '', result)
|
||||
|
||||
# Remove trailing parenthetical groups, but keep years like (2020)
|
||||
match = re.search(r'\s*(\([^)]+\))$', result)
|
||||
if match and not re.fullmatch(r'\(\d{4}\)', match.group(1)):
|
||||
result = result[:match.start()].rstrip()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -77,6 +85,22 @@ def normalize_title(title: str) -> str:
|
||||
return title.strip()
|
||||
|
||||
|
||||
def humanize_parsed_title(title: str) -> str:
|
||||
"""Make parsed titles less likely to contain accidental all-caps tags.
|
||||
|
||||
This keeps short acronyms like "IV" intact while softening long all-caps
|
||||
words that are likely part of the filename rather than intentional styling.
|
||||
"""
|
||||
normalized = normalize_title(title)
|
||||
words = []
|
||||
for word in normalized.split():
|
||||
if word.isalpha() and word.isupper() and len(word) > 3:
|
||||
words.append(word.capitalize())
|
||||
else:
|
||||
words.append(word)
|
||||
return ' '.join(words).strip()
|
||||
|
||||
|
||||
def parse_movie(
|
||||
filename: str,
|
||||
extensions: Optional[list[str]] = None,
|
||||
@@ -127,7 +151,7 @@ def parse_movie(
|
||||
# Now clean the title
|
||||
title = remove_quality_tags(title)
|
||||
title = remove_release_groups(title)
|
||||
title = normalize_title(title)
|
||||
title = humanize_parsed_title(title or match.group(1))
|
||||
|
||||
return MovieIdentity(
|
||||
title=title,
|
||||
@@ -140,7 +164,7 @@ def parse_movie(
|
||||
# No year found - clean and extract title, flag for review
|
||||
cleaned = remove_quality_tags(name_without_ext)
|
||||
cleaned = remove_release_groups(cleaned)
|
||||
title = normalize_title(cleaned)
|
||||
title = humanize_parsed_title(cleaned or name_without_ext)
|
||||
|
||||
return MovieIdentity(
|
||||
title=title,
|
||||
@@ -189,6 +213,8 @@ def parse_series(
|
||||
(r'(?<!\d)(\d{1,2})x(\d{1,2})(?!\d)', 0.9),
|
||||
# Pattern: Season X Episode Y - Medium confidence
|
||||
(r'[Ss]eason\s*(\d{1,2})\s*[Ee]pisode\s*(\d{1,2})', 0.7),
|
||||
# Pattern: Hyphen Episode (Anime style: Name - 01) - Medium confidence, assume Season 1
|
||||
(r'\s+-\s+(\d{1,3})(?!\d)', 0.6),
|
||||
]
|
||||
|
||||
season = None
|
||||
@@ -199,8 +225,14 @@ def parse_series(
|
||||
for pattern, conf in patterns:
|
||||
match = re.search(pattern, name_without_ext, re.IGNORECASE)
|
||||
if match:
|
||||
season = int(match.group(1))
|
||||
episodes = [int(match.group(2))]
|
||||
if len(match.groups()) == 2:
|
||||
season = int(match.group(1))
|
||||
episodes = [int(match.group(2))]
|
||||
else:
|
||||
# Hyphen episode only
|
||||
season = 1
|
||||
episodes = [int(match.group(1))]
|
||||
|
||||
confidence = conf
|
||||
|
||||
# Extract title (everything before the match)
|
||||
@@ -222,12 +254,12 @@ def parse_series(
|
||||
if title_part:
|
||||
title_part = remove_quality_tags(title_part)
|
||||
title_part = remove_release_groups(title_part)
|
||||
title_part = normalize_title(title_part)
|
||||
title_part = humanize_parsed_title(title_part or name_without_ext)
|
||||
else:
|
||||
# If no title part found, use the whole filename cleaned
|
||||
title_part = remove_quality_tags(name_without_ext)
|
||||
title_part = remove_release_groups(title_part)
|
||||
title_part = normalize_title(title_part)
|
||||
title_part = humanize_parsed_title(title_part or name_without_ext)
|
||||
|
||||
# Determine if review is needed
|
||||
needs_review = season is None or len(episodes) == 0
|
||||
@@ -247,6 +279,7 @@ def parse_series(
|
||||
|
||||
|
||||
def group_episodes(episodes: list[SeriesIdentity]) -> dict[tuple[str, int], list[SeriesIdentity]]:
|
||||
|
||||
"""Group parsed episodes by normalized series title and season number.
|
||||
|
||||
Episodes are grouped by (normalized_title, season) tuple. Episodes with
|
||||
|
||||
@@ -7,13 +7,7 @@ import re
|
||||
from pathlib import Path
|
||||
|
||||
from vlm.models import ExecutionPlan, FileOperation
|
||||
|
||||
|
||||
def _is_sample_path(path: Path) -> bool:
|
||||
parts = [part.casefold() for part in path.parts]
|
||||
if "sample" in parts:
|
||||
return True
|
||||
return bool(re.search(r"(^|[\s._-])sample($|[\s._-])", path.stem.casefold()))
|
||||
from vlm.utils import is_sample_path
|
||||
|
||||
|
||||
def _extract_season_episode(operation: FileOperation) -> tuple[int | None, int | None]:
|
||||
@@ -63,7 +57,7 @@ def review_plan(
|
||||
if "manual review" in reason_l:
|
||||
flags.append("manual_review")
|
||||
counters["manual_review"] += 1
|
||||
if _is_sample_path(op.source_path):
|
||||
if is_sample_path(op.source_path):
|
||||
flags.append("sample_source")
|
||||
counters["sample_source"] += 1
|
||||
season, episode = _extract_season_episode(op)
|
||||
@@ -100,3 +94,4 @@ def save_review_csv(rows: list[dict[str, str]], output: Path) -> None:
|
||||
writer = csv.DictWriter(f, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
+69
-22
@@ -13,7 +13,8 @@ from typing import Optional, Union
|
||||
|
||||
from vlm.config import Config
|
||||
from vlm.duplicate_resolve import choose_keep_index
|
||||
from vlm.utils import ensure_utc, is_within_root, sanitize_path_component, utc_now
|
||||
from vlm.io import validate_plan_json
|
||||
from vlm.utils import ensure_utc, is_sample_path, is_within_root, sanitize_path_component, utc_now
|
||||
from vlm.models import (
|
||||
ExecutionPlan,
|
||||
FileOperation,
|
||||
@@ -31,15 +32,6 @@ NO_OP_REASON_SEASON_OUT_OF_RANGE = "Series needs manual review (season exceeds c
|
||||
NO_OP_REASON_EPISODE_OUT_OF_RANGE = "Series needs manual review (episode exceeds configured threshold)"
|
||||
|
||||
|
||||
def _is_sample_path(path: Path) -> bool:
|
||||
"""Return True if path appears to be a sample clip."""
|
||||
parts = [part.casefold() for part in path.parts]
|
||||
if "sample" in parts:
|
||||
return True
|
||||
stem = path.stem.casefold()
|
||||
return bool(re.search(r"(^|[\s._-])sample($|[\s._-])", stem))
|
||||
|
||||
|
||||
def _analyze_directory_impact(operations: list[FileOperation]) -> dict:
|
||||
"""Analyze which directories will be emptied by the plan."""
|
||||
# Get all source directories that have files being moved/renamed
|
||||
@@ -72,7 +64,7 @@ def _analyze_directory_impact(operations: list[FileOperation]) -> dict:
|
||||
|
||||
return {
|
||||
"emptied_directories": emptied_dirs,
|
||||
"warning_required": len(emptied_dirs) > 0
|
||||
"warning_required": len(emptied_dirs) > 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -106,10 +98,11 @@ def generate_plan(
|
||||
operations.append(operation)
|
||||
|
||||
metadata: dict = {}
|
||||
validation_snapshot: dict[str, object] = {}
|
||||
if analysis_data is not None:
|
||||
metadata["analysis_source"] = analysis_data.get("metadata", {}).get("source_identities", "")
|
||||
metadata["duplicate_groups_considered"] = len(analysis_data.get("duplicates", []))
|
||||
metadata["completeness_seasons_with_gaps"] = len(analysis_data.get("completeness", []))
|
||||
validation_snapshot["analysis_source"] = analysis_data.get("metadata", {}).get("source_identities", "")
|
||||
validation_snapshot["duplicate_groups_considered"] = len(analysis_data.get("duplicates", []))
|
||||
validation_snapshot["completeness_seasons_with_gaps"] = len(analysis_data.get("completeness", []))
|
||||
|
||||
if config.duplicate_keep != "manual":
|
||||
path_to_index = {str(vf.path): i for i, (vf, _) in enumerate(identities)}
|
||||
@@ -124,7 +117,7 @@ def generate_plan(
|
||||
if identity is not None and isinstance(
|
||||
identity, (MovieIdentity, SeriesIdentity)
|
||||
):
|
||||
if not config.plan_include_sample_files and _is_sample_path(identities[i][0].path):
|
||||
if not config.plan_include_sample_files and is_sample_path(identities[i][0].path):
|
||||
# Keep sample files out of duplicate keep/quarantine competition by default.
|
||||
continue
|
||||
items.append((identities[i][0].path, identity))
|
||||
@@ -169,15 +162,16 @@ def generate_plan(
|
||||
has_conflict=False,
|
||||
conflict_reason=None
|
||||
))
|
||||
validation_snapshot["emptied_directories"] = [str(d) for d in directory_analysis["emptied_directories"]]
|
||||
validation_snapshot["directory_warning"] = True
|
||||
|
||||
summary = _generate_summary(operations)
|
||||
summary_by_reason = _generate_summary_by_reason(operations)
|
||||
human_summary = _generate_human_summary(operations, summary, summary_by_reason, metadata)
|
||||
|
||||
# Add directory warnings to metadata
|
||||
if directory_analysis["warning_required"]:
|
||||
metadata["emptied_directories"] = [str(d) for d in directory_analysis["emptied_directories"]]
|
||||
metadata["directory_warning"] = True
|
||||
if validation_snapshot:
|
||||
validation_snapshot["captured_at"] = utc_now().isoformat()
|
||||
metadata["validation_snapshot"] = validation_snapshot
|
||||
|
||||
return ExecutionPlan(
|
||||
plan_id=str(uuid.uuid4()),
|
||||
@@ -205,7 +199,7 @@ def _create_operation(
|
||||
Returns:
|
||||
FileOperation specifying what to do with the file
|
||||
"""
|
||||
if not config.plan_include_sample_files and _is_sample_path(video_file.path):
|
||||
if not config.plan_include_sample_files and is_sample_path(video_file.path):
|
||||
return FileOperation(
|
||||
operation_type="no-op",
|
||||
source_path=video_file.path,
|
||||
@@ -628,7 +622,7 @@ def save_plan(plan: ExecutionPlan, output_path: Path) -> None:
|
||||
|
||||
# Write to JSON file with indentation for human readability
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(plan_dict, f, indent=2, ensure_ascii=False)
|
||||
json.dump(validate_plan_json(plan_dict), f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def load_plan(input_path: Path) -> ExecutionPlan:
|
||||
@@ -649,7 +643,7 @@ def load_plan(input_path: Path) -> ExecutionPlan:
|
||||
KeyError: If required fields are missing from the JSON
|
||||
"""
|
||||
with open(input_path, 'r', encoding='utf-8') as f:
|
||||
plan_dict = json.load(f)
|
||||
plan_dict = validate_plan_json(json.load(f))
|
||||
|
||||
# Reconstruct FileOperation objects
|
||||
operations = [
|
||||
@@ -675,3 +669,56 @@ def load_plan(input_path: Path) -> ExecutionPlan:
|
||||
human_summary=plan_dict.get("human_summary", ""),
|
||||
metadata=plan_dict.get("metadata", {}),
|
||||
)
|
||||
|
||||
|
||||
def apply_review_to_plan(plan: ExecutionPlan, csv_path: Path) -> ExecutionPlan:
|
||||
"""Update a plan's operations based on a modified review CSV.
|
||||
|
||||
Args:
|
||||
plan: The original ExecutionPlan
|
||||
csv_path: Path to the modified manual review CSV
|
||||
|
||||
Returns:
|
||||
Updated ExecutionPlan with modified operation types
|
||||
"""
|
||||
import csv
|
||||
|
||||
# Create a copy of operations to modify
|
||||
updated_ops = list(plan.operations)
|
||||
modified_count = 0
|
||||
|
||||
with open(csv_path, "r", encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
try:
|
||||
# CSV index is 1-based
|
||||
idx = int(row["index"]) - 1
|
||||
if 0 <= idx < len(updated_ops):
|
||||
new_op_type = row["operation_type"]
|
||||
old_op_type = updated_ops[idx].operation_type
|
||||
|
||||
if new_op_type != old_op_type:
|
||||
# Update the operation type (usually to 'no-op')
|
||||
updated_ops[idx].operation_type = new_op_type
|
||||
updated_ops[idx].reason = f"Modified via manual review: {updated_ops[idx].reason}"
|
||||
modified_count += 1
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
|
||||
if modified_count > 0:
|
||||
# Re-generate summary and human summary for the updated plan
|
||||
summary = _generate_summary(updated_ops)
|
||||
summary_by_reason = _generate_summary_by_reason(updated_ops)
|
||||
human_summary = _generate_human_summary(updated_ops, summary, summary_by_reason, plan.metadata)
|
||||
|
||||
return ExecutionPlan(
|
||||
plan_id=plan.plan_id,
|
||||
created_at=plan.created_at,
|
||||
operations=updated_ops,
|
||||
summary=summary,
|
||||
summary_by_reason=summary_by_reason,
|
||||
human_summary=human_summary,
|
||||
metadata=plan.metadata,
|
||||
)
|
||||
|
||||
return plan
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Shared helpers for human-readable plan review (CLI preview and TUI)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
# Internal risk flag keys from plan_review.review_plan (pipe-separated in CSV).
|
||||
RISK_FLAG_LABELS: dict[str, str] = {
|
||||
"manual_review": "需人工判断",
|
||||
"sample_source": "样片路径",
|
||||
"high_season": "季号偏高",
|
||||
"high_episode": "集号偏高",
|
||||
"conflict": "目标冲突",
|
||||
}
|
||||
|
||||
|
||||
def risk_flags_to_labels(flags: str, *, max_len: int = 24) -> str:
|
||||
"""Map pipe-separated risk flags to short Chinese labels."""
|
||||
if not flags or flags.strip().lower() == "none":
|
||||
return ""
|
||||
parts = [p.strip() for p in flags.split("|") if p.strip()]
|
||||
labels = [RISK_FLAG_LABELS.get(p, p) for p in parts]
|
||||
text = " ".join(labels)
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
return text[: max_len - 1] + "…"
|
||||
|
||||
|
||||
def _display_path(p: Path, library_root: Path) -> str:
|
||||
try:
|
||||
resolved = p.resolve()
|
||||
root = library_root.resolve()
|
||||
rel = resolved.relative_to(root)
|
||||
return str(rel)
|
||||
except (ValueError, OSError):
|
||||
return str(p)
|
||||
|
||||
|
||||
def format_paths_for_detail(
|
||||
source_s: str,
|
||||
dest_s: str,
|
||||
library_root: Path,
|
||||
) -> str:
|
||||
"""Build multi-line before/after path text for review detail panes."""
|
||||
source = Path(source_s)
|
||||
dest = Path(dest_s) if dest_s.strip() else None
|
||||
src_line = _display_path(source, library_root)
|
||||
lines = [f"来源: {src_line}", f"完整: {source}"]
|
||||
if dest is not None:
|
||||
dst_line = _display_path(dest, library_root)
|
||||
lines.append(f"目标: {dst_line}")
|
||||
lines.append(f"完整: {dest}")
|
||||
else:
|
||||
lines.append("目标: (无)")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_csv_rows(
|
||||
base_rows: list[dict[str, str]],
|
||||
op_by_index: dict[int, str],
|
||||
) -> list[dict[str, str]]:
|
||||
"""Return CSV row dicts with operation_type taken from op_by_index per 1-based index."""
|
||||
result: list[dict[str, str]] = []
|
||||
for r in base_rows:
|
||||
idx = int(r["index"])
|
||||
new_r = dict(r)
|
||||
if idx in op_by_index:
|
||||
new_r["operation_type"] = op_by_index[idx]
|
||||
result.append(new_r)
|
||||
return result
|
||||
|
||||
|
||||
def review_row_status_symbol(
|
||||
op_by_index: dict[int, str],
|
||||
initial_op_by_index: dict[int, str],
|
||||
index: int,
|
||||
) -> str:
|
||||
"""Return a single-character status marker for the review table."""
|
||||
cur = op_by_index.get(index, initial_op_by_index[index])
|
||||
init = initial_op_by_index[index]
|
||||
if cur != init and cur == "no-op":
|
||||
return "✗"
|
||||
if cur != init and cur != "no-op":
|
||||
return "✓"
|
||||
return "·"
|
||||
@@ -0,0 +1,388 @@
|
||||
"""Textual TUI for reviewing high-risk plan operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from textual import on
|
||||
from textual.app import App, ComposeResult
|
||||
from textual.binding import Binding
|
||||
from textual.containers import Container, Horizontal, ScrollableContainer, Vertical
|
||||
from textual.geometry import Size
|
||||
from textual.screen import ModalScreen, Screen
|
||||
from textual.widgets import DataTable, Footer, Static
|
||||
|
||||
from vlm.plan_review import save_review_csv
|
||||
from vlm.review_display import (
|
||||
build_csv_rows,
|
||||
format_paths_for_detail,
|
||||
review_row_status_symbol,
|
||||
risk_flags_to_labels,
|
||||
)
|
||||
|
||||
|
||||
def _index_from_row_key(row_key) -> int: # noqa: ANN001 - RowKey | str
|
||||
if isinstance(row_key, str):
|
||||
return int(row_key)
|
||||
val = getattr(row_key, "value", None)
|
||||
if val is None:
|
||||
return int(row_key)
|
||||
return int(val)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReviewTUIContext:
|
||||
"""Inputs for the plan review TUI."""
|
||||
|
||||
rows: list[dict[str, str]]
|
||||
counters: dict[str, int]
|
||||
library_root: Path
|
||||
output_csv: Path
|
||||
plan_input: Path
|
||||
summary_text: str
|
||||
|
||||
|
||||
class SummaryScreen(Screen):
|
||||
"""Migration summary; Enter continues, q aborts."""
|
||||
|
||||
BINDINGS = [
|
||||
Binding("enter", "continue_", "继续", show=True),
|
||||
Binding("q", "quit", "退出", show=True),
|
||||
]
|
||||
|
||||
def __init__(self, ctx: ReviewTUIContext) -> None:
|
||||
super().__init__()
|
||||
self._ctx = ctx
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
stats_lines = [
|
||||
"---",
|
||||
"计划统计",
|
||||
f" 总操作: {self._ctx.counters['total_operations']}",
|
||||
f" 高危: {self._ctx.counters['high_risk_operations']}",
|
||||
f" manual_review: {self._ctx.counters['manual_review']}",
|
||||
f" sample_source: {self._ctx.counters['sample_source']}",
|
||||
f" high_season: {self._ctx.counters['high_season']}",
|
||||
f" high_episode: {self._ctx.counters['high_episode']}",
|
||||
f" conflicts: {self._ctx.counters['conflicts']}",
|
||||
"",
|
||||
f"将要写入: {self._ctx.output_csv}",
|
||||
"",
|
||||
"默认仅审核标记为高危的操作（与 CSV 行一致）。",
|
||||
"",
|
||||
"Enter 进入审核 · q 退出",
|
||||
]
|
||||
body = self._ctx.summary_text + "\n\n" + "\n".join(stats_lines)
|
||||
yield ScrollableContainer(Static(body, id="summary_body"))
|
||||
yield Footer()
|
||||
|
||||
def action_continue_(self) -> None:
|
||||
self.dismiss(True)
|
||||
|
||||
def action_quit(self) -> None:
|
||||
self.dismiss(False)
|
||||
|
||||
|
||||
class ConfirmDiscardScreen(ModalScreen[bool]):
|
||||
"""Confirm discarding unsaved edits."""
|
||||
|
||||
BINDINGS = [
|
||||
Binding("y", "yes", show=False),
|
||||
Binding("n", "no", show=False),
|
||||
]
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
yield Container(
|
||||
Static("未保存的修改将丢失。放弃？ (y / n)", id="confirm_text"),
|
||||
id="confirm_box",
|
||||
)
|
||||
|
||||
def action_yes(self) -> None:
|
||||
self.dismiss(True)
|
||||
|
||||
def action_no(self) -> None:
|
||||
self.dismiss(False)
|
||||
|
||||
DEFAULT_CSS = """
|
||||
ConfirmDiscardScreen {
|
||||
align: center middle;
|
||||
}
|
||||
#confirm_box {
|
||||
width: auto;
|
||||
height: auto;
|
||||
padding: 1 2;
|
||||
border: thick $primary;
|
||||
background: $surface;
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
class ReviewMainScreen(Screen):
|
||||
"""High-risk table + detail pane."""
|
||||
|
||||
BINDINGS = [
|
||||
Binding("up", "cursor_up", show=False),
|
||||
Binding("down", "cursor_down", show=False),
|
||||
Binding("k", "cursor_up", show=False),
|
||||
Binding("j", "cursor_down", show=False),
|
||||
Binding("a", "keep_row", "保留", show=True),
|
||||
Binding("r", "reject_row", "驳回", show=True),
|
||||
Binding("u", "undo_row", "撤销", show=True),
|
||||
Binding("s", "save", "保存", show=True),
|
||||
Binding("q", "request_quit", "退出", show=True),
|
||||
]
|
||||
|
||||
def __init__(self, ctx: ReviewTUIContext) -> None:
|
||||
super().__init__()
|
||||
self.ctx = ctx
|
||||
self._by_index: dict[int, dict[str, str]] = {
|
||||
int(r["index"]): r for r in ctx.rows
|
||||
}
|
||||
self.initial_op_by_index: dict[int, str] = {
|
||||
int(r["index"]): r["operation_type"] for r in ctx.rows
|
||||
}
|
||||
self.op_by_index: dict[int, str] = dict(self.initial_op_by_index)
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
plan_s = str(self.ctx.plan_input)
|
||||
if len(plan_s) > 72:
|
||||
plan_s = plan_s[:35] + "…" + plan_s[-34:]
|
||||
hdr = (
|
||||
f"{plan_s} · 高危 {self.ctx.counters['high_risk_operations']}"
|
||||
f" / {self.ctx.counters['total_operations']}"
|
||||
)
|
||||
yield Static(hdr, id="header_line")
|
||||
with Horizontal(id="body"):
|
||||
yield DataTable(id="review_table", cursor_type="row", zebra_stripes=True)
|
||||
with ScrollableContainer(id="detail_scroll"):
|
||||
yield Static("", id="detail_text")
|
||||
yield Static(
|
||||
"↑↓ j/k 移动 · a 保留 · r 驳回(no-op) · u 撤销本条 · s 保存退出 · q 退出",
|
||||
id="footer_line",
|
||||
)
|
||||
yield Footer()
|
||||
|
||||
DEFAULT_CSS = """
|
||||
#header_line {
|
||||
dock: top;
|
||||
padding: 0 1;
|
||||
background: $primary-darken-2;
|
||||
color: $text;
|
||||
}
|
||||
#footer_line {
|
||||
dock: bottom;
|
||||
padding: 0 1;
|
||||
background: $panel;
|
||||
color: $text-muted;
|
||||
}
|
||||
#body {
|
||||
layout: horizontal;
|
||||
height: 1fr;
|
||||
}
|
||||
#body.vertical-split {
|
||||
layout: vertical;
|
||||
height: 1fr;
|
||||
}
|
||||
#review_table {
|
||||
width: 1fr;
|
||||
min-height: 5;
|
||||
}
|
||||
#body.vertical-split #review_table {
|
||||
height: 40%;
|
||||
}
|
||||
#detail_scroll {
|
||||
width: 1fr;
|
||||
min-height: 5;
|
||||
border-left: solid $primary-darken-3;
|
||||
padding: 0 1;
|
||||
}
|
||||
#body.vertical-split #detail_scroll {
|
||||
border-left: none;
|
||||
border-top: solid $primary-darken-3;
|
||||
height: 1fr;
|
||||
}
|
||||
"""
|
||||
|
||||
def on_mount(self) -> None:
|
||||
table = self.query_one("#review_table", DataTable)
|
||||
table.cursor_type = "row"
|
||||
table.add_column(" ", key="sym", width=3)
|
||||
table.add_column("#", key="idx", width=4)
|
||||
table.add_column("类型", key="op", width=11)
|
||||
table.add_column("风险", key="risk", width=18)
|
||||
table.add_column("文件", key="file")
|
||||
|
||||
for r in self.ctx.rows:
|
||||
idx = int(r["index"])
|
||||
sym = self._symbol_for(idx)
|
||||
table.add_row(
|
||||
sym,
|
||||
str(idx),
|
||||
self.op_by_index[idx],
|
||||
risk_flags_to_labels(r["risk_flags"]),
|
||||
Path(r["source_path"]).name,
|
||||
key=str(idx),
|
||||
)
|
||||
self._apply_body_layout(self.app.size)
|
||||
if table.row_count > 0:
|
||||
table.focus()
|
||||
self._refresh_detail(_index_from_row_key(table.ordered_rows[0].key))
|
||||
else:
|
||||
self.query_one("#detail_text", Static).update(
|
||||
"无高危项。按 s 保存仅含表头的 CSV（与无 --tui 行为一致）。"
|
||||
)
|
||||
|
||||
def _table(self) -> DataTable:
|
||||
return self.query_one("#review_table", DataTable)
|
||||
|
||||
def _symbol_for(self, index: int) -> str:
|
||||
return review_row_status_symbol(
|
||||
self.op_by_index,
|
||||
self.initial_op_by_index,
|
||||
index,
|
||||
)
|
||||
|
||||
def _apply_body_layout(self, app_size: Size) -> None:
|
||||
body = self.query_one("#body", Horizontal)
|
||||
if app_size.width < 100:
|
||||
body.add_class("vertical-split")
|
||||
else:
|
||||
body.remove_class("vertical-split")
|
||||
|
||||
def on_resize(self, event) -> None: # noqa: ANN001 - textual Resize
|
||||
self._apply_body_layout(self.app.size)
|
||||
|
||||
def _current_index(self) -> int | None:
|
||||
table = self._table()
|
||||
if table.row_count == 0:
|
||||
return None
|
||||
row_index = table.cursor_coordinate.row
|
||||
row = table.ordered_rows[row_index]
|
||||
return _index_from_row_key(row.key)
|
||||
|
||||
@on(DataTable.RowHighlighted) # type: ignore[misc]
|
||||
def on_row_highlighted(self, event: DataTable.RowHighlighted) -> None:
|
||||
if event.data_table.id != "review_table":
|
||||
return
|
||||
idx = _index_from_row_key(event.row_key)
|
||||
self._refresh_detail(idx)
|
||||
|
||||
def _refresh_detail(self, index: int) -> None:
|
||||
r = self._by_index[index]
|
||||
op = self.op_by_index[index]
|
||||
paths = format_paths_for_detail(
|
||||
r["source_path"],
|
||||
r.get("destination_path", ""),
|
||||
self.ctx.library_root,
|
||||
)
|
||||
risk_cn = risk_flags_to_labels(r["risk_flags"], max_len=120)
|
||||
summary = (
|
||||
f"#{index} · {op} · {Path(r['source_path']).name}"
|
||||
+ (f" · {risk_cn}" if risk_cn else "")
|
||||
)
|
||||
text = (
|
||||
f"{summary}\n\n"
|
||||
f"变更\n{paths}\n\n"
|
||||
f"依据\n{r['reason']}\n\n"
|
||||
f"标记\n{r['risk_flags']}"
|
||||
)
|
||||
self.query_one("#detail_text", Static).update(text)
|
||||
|
||||
def _refresh_row_cells(self, index: int) -> None:
|
||||
table = self._table()
|
||||
key = str(index)
|
||||
sym = self._symbol_for(index)
|
||||
table.update_cell(key, "sym", sym)
|
||||
table.update_cell(key, "op", self.op_by_index[index])
|
||||
|
||||
def _update_dirty_header(self) -> None:
|
||||
dirty = self._is_dirty()
|
||||
hdr = self.query_one("#header_line", Static)
|
||||
plan_s = str(self.ctx.plan_input)
|
||||
if len(plan_s) > 72:
|
||||
plan_s = plan_s[:35] + "…" + plan_s[-34:]
|
||||
star = " *" if dirty else ""
|
||||
hdr.update(
|
||||
f"{plan_s}{star} · 高危 {self.ctx.counters['high_risk_operations']}"
|
||||
f" / {self.ctx.counters['total_operations']}"
|
||||
)
|
||||
|
||||
def _is_dirty(self) -> bool:
|
||||
return self.op_by_index != self.initial_op_by_index
|
||||
|
||||
def action_cursor_up(self) -> None:
|
||||
if self._table().row_count:
|
||||
self._table().action_cursor_up()
|
||||
|
||||
def action_cursor_down(self) -> None:
|
||||
if self._table().row_count:
|
||||
self._table().action_cursor_down()
|
||||
|
||||
def action_keep_row(self) -> None:
|
||||
idx = self._current_index()
|
||||
if idx is None:
|
||||
return
|
||||
self.op_by_index[idx] = self.initial_op_by_index[idx]
|
||||
self._refresh_row_cells(idx)
|
||||
self._refresh_detail(idx)
|
||||
self._update_dirty_header()
|
||||
|
||||
def action_reject_row(self) -> None:
|
||||
idx = self._current_index()
|
||||
if idx is None:
|
||||
return
|
||||
self.op_by_index[idx] = "no-op"
|
||||
self._refresh_row_cells(idx)
|
||||
self._refresh_detail(idx)
|
||||
self._update_dirty_header()
|
||||
|
||||
def action_undo_row(self) -> None:
|
||||
self.action_keep_row()
|
||||
|
||||
def action_save(self) -> None:
|
||||
out_rows = build_csv_rows(self.ctx.rows, self.op_by_index)
|
||||
save_review_csv(out_rows, self.ctx.output_csv)
|
||||
self.dismiss("saved")
|
||||
|
||||
def action_request_quit(self) -> None:
|
||||
if not self._is_dirty():
|
||||
self.dismiss("aborted")
|
||||
return
|
||||
|
||||
def after_confirm(confirmed: bool | None) -> None:
|
||||
if confirmed:
|
||||
self.dismiss("aborted")
|
||||
|
||||
self.app.push_screen(ConfirmDiscardScreen(), callback=after_confirm)
|
||||
|
||||
|
||||
class PlanReviewApp(App):
|
||||
"""Application shell: summary screen then review screen."""
|
||||
|
||||
def __init__(self, ctx: ReviewTUIContext) -> None:
|
||||
super().__init__()
|
||||
self.ctx = ctx
|
||||
|
||||
def on_mount(self) -> None:
|
||||
self.push_screen(SummaryScreen(self.ctx), self._after_summary)
|
||||
|
||||
def _after_summary(self, result: bool | None) -> None:
|
||||
if not result:
|
||||
self.exit(return_code=1)
|
||||
return
|
||||
self.push_screen(ReviewMainScreen(self.ctx), self._after_main)
|
||||
|
||||
def _after_main(self, result: str | None) -> None:
|
||||
if result == "saved":
|
||||
self.exit(return_code=0)
|
||||
else:
|
||||
self.exit(return_code=1)
|
||||
|
||||
|
||||
def run_plan_review_tui(ctx: ReviewTUIContext) -> int:
|
||||
"""Block until the user finishes the TUI. Returns process exit code."""
|
||||
app = PlanReviewApp(ctx)
|
||||
app.run()
|
||||
code = app.return_code
|
||||
return 0 if code is None else code
|
||||
@@ -65,6 +65,15 @@ def scan_library(
|
||||
logger.error(f"Library root is not a directory: {root}")
|
||||
return []
|
||||
|
||||
if include_video_metadata:
|
||||
import shutil
|
||||
if not shutil.which("ffprobe"):
|
||||
logger.warning(
|
||||
"ffprobe command not found in PATH. Video metadata extraction will be skipped. "
|
||||
"Only file-level information (size, mtime) will be recorded."
|
||||
)
|
||||
include_video_metadata = False
|
||||
|
||||
video_files = []
|
||||
file_count = 0
|
||||
discovered_paths = _discover_video_paths(root, config.video_extensions)
|
||||
|
||||
@@ -51,6 +51,17 @@ def canonical_path_str(path: Path) -> str:
|
||||
|
||||
|
||||
_PATH_SEPARATORS_PATTERN = re.compile(r"[\\/]+")
|
||||
_SAMPLE_TOKEN_PATTERN = re.compile(r"(^|[\s._-])sample($|[\s._-])")
|
||||
|
||||
|
||||
def is_sample_path(path: Path) -> bool:
|
||||
"""Return True if path appears to be a sample clip."""
|
||||
parts = [part.casefold() for part in path.parts]
|
||||
if "sample" in parts:
|
||||
return True
|
||||
return bool(_SAMPLE_TOKEN_PATTERN.search(path.stem.casefold()))
|
||||
|
||||
|
||||
_CONTROL_CHARS_PATTERN = re.compile(r"[\x00-\x1f\x7f]")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user