From 259e7506d7c9f33a1ff2c1cb5f7372add7795a08 Mon Sep 17 00:00:00 2001 From: windyboy Date: Mon, 9 Feb 2026 22:40:48 +0800 Subject: [PATCH] feat: add configurable category mappings for directory recognition Allow users to configure multiple directory names per category (movie/series/anime) to support variations like "movies", "tv", "films". This enables proper categorization of files in directories that don't match the hardcoded singular forms, solving the issue where 635 files in "/mnt/Downloads/movies/" were incorrectly categorized as "other". Configuration example: categories: movie: [movie, movies, films] series: [series, tv, shows] anime: [anime] Changes include comprehensive validation, backward-compatible defaults, case-insensitive matching, and full test coverage (395 tests passing). Co-Authored-By: Claude Sonnet 4.5 --- CLAUDE.md | 199 ++++++++++++++ README.md | 610 +++++++++++++++++++++++++++++++++++++++++- src/vlm/config.py | 72 ++++- src/vlm/scanner.py | 174 +++++++++--- tests/test_config.py | 167 +++++++++++- tests/test_scanner.py | 243 +++++++++++++++-- 6 files changed, 1380 insertions(+), 85 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..e617b45 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,199 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Video Library Manager (VLM) is a Python CLI tool for managing personal video collections with a safety-first, human-in-the-loop approach. All file operations are reversible, require explicit confirmation, and generate reviewable execution plans before making changes. + +## Development Commands + +### Installation +```bash +# Install package in editable mode +uv pip install -e . + +# Install with dev dependencies (pytest, hypothesis) +uv pip install -e ".[dev]" +``` + +### Testing +```bash +# Run all tests +pytest + +# Run specific test file +pytest tests/test_scanner.py + +# Run with verbose output +pytest -v +``` + +### Running the CLI +```bash +# Verify CLI works +vlm --help + +# Initialize config +vlm config init + +# Common workflow +vlm scan # Discover files +vlm parse # Extract identities +vlm analyze # Detect gaps/duplicates +vlm plan # Generate execution plan +vlm execute # Dry-run (default) +vlm execute --confirm # Actually execute +``` + +## Architecture + +### Core Workflow +VLM follows a read-first, multi-stage pipeline: +1. **Scan** → discovers video files, extracts metadata via ffprobe (optional), saves to inventory.csv +2. **Parse** → extracts titles/years/seasons/episodes from filenames, saves to identities.json +3. **Analyze** → detects episode gaps and duplicates, saves to analysis.json +4. **Plan** → generates reviewable execution plan (plan.json) with file operations +5. **Execute** → performs file operations (dry-run by default, --confirm to execute) +6. **Rollback** → reverses executed operations (best-effort) + +### Module Organization +- `cli.py` - Click-based CLI interface, command definitions, all user-facing commands +- `scanner.py` - File discovery using system `find` command, metadata extraction via ffprobe +- `parser.py` - Filename parsing using regex patterns (movies: title + year, series: SxxExx) +- `analysis.py` - Completeness checking (episode gaps) and duplicate detection +- `planner.py` - Execution plan generation with conflict detection +- `executor.py` - File operations (move/rename/quarantine) with rollback logging +- `quarantine.py` - Quarantine management with manifest tracking +- `state.py` - File state tracking across workflow stages +- `reports.py` - Report generation (inventory, completeness, duplicates, summary) +- `config.py` - YAML configuration loading and validation +- `models.py` - Dataclass definitions for all data structures +- `logging_config.py` - Logging setup with fallback to console if file logging fails + +### Key Data Structures +All defined in `models.py`: +- `VideoFile` - represents discovered video file with metadata +- `MovieIdentity` / `SeriesIdentity` - parsed identity with confidence score +- `ExecutionPlan` - collection of file operations with summary +- `FileOperation` - single operation (move/rename/quarantine/no-op) with conflict detection +- `RollbackLog` - log of executed operations for reversal +- `QuarantineEntry` - quarantined file with original location +- `FileState` - workflow state (reviewed/ignored/planned/executed/quarantined) + +### File Categorization +Based on top-level directory within library root, matched against configured category mappings (case-insensitive). + +Files in unmapped directories are categorized as "other" and skipped by planner/quarantine operations. + +### Parsing Patterns (Hardcoded) +**Movies** (high confidence): +- `{title} ({year})` +- `{title}.{year}` + +**Series** (high confidence): +- `S{season:02d}E{episode:02d}` +- `{season}x{episode}` + +Patterns are hardcoded in parser.py, not user-configurable. + +### Configuration +Default location: `~/.vlm/config.yaml` + +Key settings: +- `library_root` - root directory to scan (required) +- `video_extensions` - list of extensions to recognize +- `templates.movie_dir` / `templates.series_dir` - directory structure templates +- `templates.movie_filename` / `templates.series_filename` - filename templates +- `quarantine_dir` - name of quarantine directory (default: `.quarantine`) +- `log_level` - logging verbosity +- `categories` - mapping of category names to directory name lists + +### Category Mappings + +Categories are determined by matching the top-level directory name against configured mappings: + +**Default mappings:** +```yaml +categories: + movie: [movie] + series: [series] + anime: [anime] +``` + +**Custom mappings** support multiple directory names per category: +```yaml +categories: + movie: [movie, movies, films] + series: [series, tv, shows, television] + anime: [anime] +``` + +This allows files in `/library/movies/` or `/library/films/` to be recognized as the "movie" category. Directory matching is case-insensitive. + +**Migration Note:** If you have existing directories with non-standard names (like "movies" or "tv"), update your config.yaml and re-run `vlm scan` to fix categorization. No files will be moved. + +### File Discovery +Uses system `find` command for speed, falls back to Python recursion if unavailable. Hidden paths (starting with `.`) are skipped automatically. + +### Timestamp Handling +All timestamps stored in UTC using ISO 8601 format (`YYYY-MM-DDTHH:MM:SS`). The scanner normalizes naive timestamps to UTC using system timezone. + +### Logging +Logs to `~/.vlm/vlm.log` by default. If log directory is unwritable, falls back to console-only logging and continues execution (does not fail). + +### Error Handling +- Configuration errors: display helpful message, create default config, continue +- Missing ffprobe: skip metadata extraction, log debug message, continue +- File access errors: log error, skip file, continue scanning +- Invalid YAML: display error, use default config + +## Testing Guidelines + +- Framework: pytest with hypothesis for property-based tests +- Test file naming: `test_*.py` +- Test function naming: `test_*` +- Test class naming: `Test*` +- Use `CliRunner` for CLI integration tests +- Prefer narrow unit tests for module logic plus targeted CLI integration tests +- Add tests with each behavior change, including error paths and edge cases + +## Important Implementation Notes + +### Safety Protocol +- NEVER permanently delete files - use quarantine instead +- All file operations create rollback logs when executed with --confirm +- Execution plans detect destination conflicts and mark operations +- Default mode is dry-run; --confirm required for actual execution +- Quarantine is reversible via restore command + +### Anime Handling +Anime files are discovered and categorized but NOT parsed in v1 (deferred for future implementation). + +### State Management +State tracking is optional and allows marking files as reviewed/ignored/planned/executed/quarantined. State is persisted to `~/.vlm/state.json`. + +### Quarantine Constraints +Only movie and series files can be quarantined in v1 (anime and other categories rejected). + +### Template Variables +Available for path/filename templates: +- Movies: `{title}`, `{year}`, `{ext}` +- Series: `{title}`, `{season}`, `{episode}`, `{ext}` + +Format specifiers like `{season:02d}` are supported. + +## Code Style + +- Python 3.10+ idioms +- 4-space indentation +- PEP 8 naming: `snake_case` for functions/variables, `PascalCase` for classes, `UPPER_SNAKE_CASE` for constants +- Type hints for public functions and non-trivial internal APIs +- Modules focused on single responsibility +- No formatter/linter enforced - maintain consistency with existing files + +## Commit Guidelines + +- Clear, imperative commit subjects (e.g., "fix logging fallback for unwritable log dir") +- Keep commits focused - avoid mixing refactors and behavior changes +- Include co-author tag: `Co-Authored-By: Claude Sonnet 4.5 ` diff --git a/README.md b/README.md index f61a8ba..085ffcb 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ A Python-based CLI tool for managing personal video collections with a safety-first, human-in-the-loop approach. +## Features + +- **Safety-First Design**: All file operations are reversible with rollback support +- **Human-in-the-Loop**: Explicit confirmation required before making any changes +- **Comprehensive Analysis**: Detect episode gaps and duplicate files +- **Rich Metadata**: Extract video resolution, codec, duration, and bitrate +- **Flexible Organization**: Customizable directory structure and naming templates +- **State Tracking**: Track file status throughout the workflow +- **Detailed Reporting**: Generate inventory, completeness, and duplicate reports + ## Installation This project uses `uv` for Python package management. To install: @@ -14,26 +24,610 @@ uv pip install -e . uv pip install -e ".[dev]" ``` -## Usage +## Quick Start + +### 1. Initialize Configuration + +First, create a configuration file: ```bash -vlm --help +vlm config init ``` -Reports and inventory exports store `modified_timestamp` values in UTC (`YYYY-MM-DDTHH:MM:SS`). +This creates `~/.vlm/config.yaml`. Edit it to set your library root: -If file logging cannot be initialized (for example, unwritable log directory), VLM falls back to console logging and continues running. +```yaml +library_root: "/mnt/Downloads" # Change this to your video library path +``` + +### 2. Scan Your Library + +Discover all video files in your library: + +```bash +vlm scan +``` + +This creates `inventory.csv` with all discovered files and their metadata. + +**What happens:** +- Discovers video files using the system `find` command +- Extracts file metadata (size, modification time) +- Categorizes files based on directory structure (movie/series/anime/other) +- Extracts video metadata using ffprobe (if available) +- Saves results to `inventory.csv` + +### 3. Parse Filenames + +Extract titles, years, seasons, and episodes from filenames: + +```bash +vlm parse +``` + +This creates `identities.json` with parsed information. + +**What it extracts:** +- **Movies**: Title and year (e.g., "Inception (2010)") +- **Series**: Title, season, and episode numbers (e.g., "Breaking Bad S01E01") +- **Confidence scores**: Indicates parsing reliability + +### 4. Analyze Your Library + +Detect episode gaps and duplicates: + +```bash +vlm analyze +``` + +This creates `analysis.json` with: +- Series with missing episodes +- Duplicate files with quality comparison + +### 5. Generate Execution Plan + +Create a reviewable plan of file operations: + +```bash +vlm plan +``` + +This creates `plan.json` with proposed operations (move, rename, quarantine). + +**Review the plan** by opening `plan.json` in your editor. You can edit it if needed. + +### 6. Execute (Dry-Run First) + +Preview what will happen without making changes: + +```bash +vlm execute +``` + +When ready to actually move/rename files: + +```bash +vlm execute --confirm +``` + +**Important**: This creates a rollback log in `~/.vlm/rollback/` for reverting changes. + +### 7. Rollback (If Needed) + +If you need to undo the operations: + +```bash +vlm rollback +``` + +This uses the most recent rollback log to restore files to their original locations. + +## Complete Workflow Example + +Here's a complete workflow from start to finish: + +```bash +# 1. Initialize configuration +vlm config init +# Edit ~/.vlm/config.yaml to set your library_root + +# 2. Scan your library +vlm scan +# Output: inventory.csv with 1234 files discovered + +# 3. Parse filenames +vlm parse +# Output: identities.json with parsed titles and episodes + +# 4. Analyze for gaps and duplicates +vlm analyze +# Output: analysis.json with 5 series with gaps, 12 duplicate groups + +# 5. Generate execution plan +vlm plan +# Output: plan.json with 456 operations proposed + +# 6. Review the plan +cat plan.json | less +# or open in your editor + +# 7. Dry-run to preview +vlm execute +# Shows what would happen without making changes + +# 8. Execute with confirmation +vlm execute --confirm +# Actually performs the file operations + +# 9. If needed, rollback +vlm rollback +``` + +## Command Reference + +### Configuration + +```bash +# Initialize config file +vlm config init + +# Show current configuration +vlm config show + +# Validate configuration +vlm config validate +``` + +### Scanning + +```bash +# Scan with default output (inventory.csv) +vlm scan + +# Scan with custom output file +vlm scan --output my_library.csv +``` + +**Scan behavior:** +- Hidden paths are skipped (any component starting with `.`) +- File matching uses configured `video_extensions` (default: `.mp4`, `.mkv`, `.avi`, `.mov`, `.wmv`, `.flv`, `.webm`, `.m4v`) +- If `ffprobe` is available, resolution/codec/duration/bitrate are included +- Timestamps stored in UTC format (`YYYY-MM-DDTHH:MM:SS`) + +### Parsing + +```bash +# Parse with default files +vlm parse + +# Parse with custom input/output +vlm parse --input my_inventory.csv --output my_identities.json +``` + +### Analysis + +```bash +# Analyze with default files +vlm analyze + +# Analyze with custom files +vlm analyze --input my_identities.json --output my_analysis.json +``` + +### Planning + +```bash +# Generate plan with default files +vlm plan + +# Generate plan with custom files +vlm plan --input my_identities.json --output my_plan.json +``` + +### Execution + +```bash +# Dry-run (preview only, no changes) +vlm execute + +# Execute with confirmation (actually makes changes) +vlm execute --confirm + +# Use custom plan file +vlm execute --plan my_plan.json --confirm +``` + +### Rollback + +```bash +# Rollback using most recent log +vlm rollback + +# Rollback using specific log +vlm rollback --log ~/.vlm/rollback/rollback_.json +``` + +### Quarantine Management + +```bash +# List quarantined files +vlm quarantine list + +# List by category +vlm quarantine list --category movie + +# Quarantine a file +vlm quarantine add /path/to/file.mkv --reason "duplicate" + +# Restore from quarantine +vlm quarantine restore /path/to/.quarantine/file.mkv +``` + +### Reporting + +```bash +# Generate inventory report (text format) +vlm report inventory + +# Generate in different formats +vlm report inventory --format csv +vlm report inventory --format json --output inventory_report.json + +# Generate completeness report (series with gaps) +vlm report completeness + +# Generate duplicates report +vlm report duplicates + +# Generate summary statistics +vlm report summary +``` + +### State Management + +```bash +# Show state for a file +vlm state show /path/to/file.mkv + +# Set state for a file +vlm state set /path/to/file.mkv --status reviewed +vlm state set /path/to/file.mkv --status ignored --reason "duplicate" + +# Query files by status +vlm state query --status ignored + +# Clear state for a file +vlm state clear /path/to/file.mkv +``` + +## Configuration File + +The configuration file (`~/.vlm/config.yaml`) controls VLM's behavior: + +```yaml +# Required: Root directory of your video library +library_root: "/mnt/nas/videos" + +# Video file extensions to recognize +video_extensions: + - .mp4 + - .mkv + - .avi + - .mov + - .wmv + - .flv + - .webm + - .m4v + +# Directory structure templates +templates: + movie_dir: "movie/{title} ({year})/" + series_dir: "series/{title}/Season {season:02d}/" + movie_filename: "{title} ({year}){ext}" + series_filename: "S{season:02d}E{episode:02d}{ext}" + +# Quarantine directory name (relative to category root) +quarantine_dir: ".quarantine" + +# Logging level (DEBUG, INFO, WARNING, ERROR) +log_level: "INFO" + +# Category mappings (directory name to category) +categories: + movie: [movie, movies, films] + series: [series, tv, shows] + anime: [anime] +``` + +### Template Variables + +**Movies:** +- `{title}` - Movie title +- `{year}` - Release year +- `{ext}` - File extension + +**Series:** +- `{title}` - Series title +- `{season}` - Season number +- `{episode}` - Episode number +- `{ext}` - File extension + +Format specifiers are supported (e.g., `{season:02d}` for zero-padded numbers). + +## File Categorization + +VLM categorizes files based on the top-level directory in your library, matched against configured category mappings. By default: + +- `movie/` → "movie" +- `series/` → "series" +- `anime/` → "anime" +- Anything else → "other" + +You can customize these mappings to support multiple directory names per category: + +```yaml +categories: + movie: [movie, movies, films] + series: [series, tv, shows, television] + anime: [anime] +``` + +This allows directories like `/library/movies/` or `/library/tv/` to be correctly categorized. Directory matching is case-insensitive. + +Example: +``` +/mnt/Downloads/ +├── movies/ # Category: movie (plural form) +│ └── Inception (2010).mkv +├── tv/ # Category: series (tv variant) +│ └── Breaking Bad S01E01.mkv +├── anime/ +│ └── Attack on Titan E01.mkv # Category: anime +└── documentaries/ + └── Planet Earth.mkv # Category: other +``` + +**Migration Note:** If you have existing directories with non-standard names (like "movies" or "tv"), update your `categories` configuration in `config.yaml` and re-run `vlm scan` to fix categorization. No files will be moved. + +## Common Scenarios + +### Scenario 1: First Time Setup + +```bash +# 1. Install +uv pip install -e . + +# 2. Initialize config +vlm config init +# Edit ~/.vlm/config.yaml + +# 3. Scan library +vlm scan + +# 4. Check what was found +vlm report summary +``` + +### Scenario 2: Finding Missing Episodes + +```bash +# 1. Scan and parse +vlm scan +vlm parse + +# 2. Analyze completeness +vlm analyze + +# 3. View report +vlm report completeness +``` + +### Scenario 3: Finding and Removing Duplicates + +```bash +# 1. Scan and parse +vlm scan +vlm parse + +# 2. Analyze for duplicates +vlm analyze + +# 3. View duplicates with quality comparison +vlm report duplicates + +# 4. Manually quarantine lower quality files +vlm quarantine add /path/to/lower/quality/file.mkv --reason "duplicate - lower quality" + +# 5. Or generate plan and let VLM suggest operations +vlm plan +vlm execute +``` + +### Scenario 4: Reorganizing Your Library + +```bash +# 1. Customize templates in config +# Edit ~/.vlm/config.yaml to set your preferred structure + +# 2. Scan and parse +vlm scan +vlm parse + +# 3. Generate plan +vlm plan + +# 4. Review plan.json carefully + +# 5. Dry-run to preview +vlm execute + +# 6. Execute when satisfied +vlm execute --confirm + +# 7. If something goes wrong, rollback +vlm rollback +``` + +## Logging + +VLM logs to `~/.vlm/vlm.log` by default. If the log directory is unwritable, it falls back to console logging and continues running. + +## Troubleshooting + +### ffprobe Not Found + +If you see warnings about ffprobe: + +```bash +# Install ffmpeg (includes ffprobe) +# Ubuntu/Debian +sudo apt install ffmpeg + +# macOS +brew install ffmpeg + +# Windows +# Download from https://ffmpeg.org/download.html +``` + +Without ffprobe, VLM still works but won't extract video metadata (resolution, codec, etc.). + +### Configuration Errors + +If VLM can't find your config: + +```bash +# Check config location +ls -la ~/.vlm/config.yaml + +# Reinitialize if needed +vlm config init + +# Validate current config +vlm config validate +``` + +### Permission Errors + +If you can't access certain files: + +```bash +# Check permissions +ls -la /path/to/library + +# VLM logs errors but continues scanning other files +# Check log file for details +cat ~/.vlm/vlm.log +``` + +### Rollback Fails + +Rollback is **best-effort** and may fail if: +- Files have been moved/deleted since execution +- Destination paths are occupied +- Permissions have changed + +Always keep backups of important files! + +## Safety Features + +- **No Permanent Deletion**: VLM never deletes files. Use quarantine instead. +- **Dry-Run Default**: Execute command defaults to dry-run mode +- **Explicit Confirmation**: `--confirm` flag required to make actual changes +- **Rollback Logs**: All operations logged for reversal +- **Conflict Detection**: Plan generation detects destination conflicts +- **Best-Effort Rollback**: Attempt to restore files to original locations + +## Limitations + +### Version 1.0 + +- **Anime parsing not implemented**: Anime files are discovered and categorized but not parsed +- **Movie/Series only for quarantine**: Only movie and series categories can be quarantined +- **Hardcoded parsing patterns**: Filename patterns are not user-configurable +- **Local files only**: Designed for local or mounted network storage +- **Best-effort rollback**: Rollback may not succeed if files have been modified ## Development -Run tests: +### Running Tests ```bash +# Run all tests pytest + +# Run specific test file +pytest tests/test_scanner.py + +# Run with verbose output +pytest -v + +# Run with coverage +pytest --cov=vlm tests/ +``` + +### Code Style + +- Python 3.10+ idioms +- PEP 8 naming conventions +- Type hints for public functions +- 4-space indentation + +### Project Structure + +``` +src/vlm/ +├── cli.py # Click-based CLI interface +├── scanner.py # File discovery and metadata extraction +├── parser.py # Filename parsing (titles, years, episodes) +├── analysis.py # Completeness and duplicate detection +├── planner.py # Execution plan generation +├── executor.py # File operations and rollback +├── quarantine.py # Quarantine management +├── state.py # File state tracking +├── reports.py # Report generation +├── config.py # Configuration management +├── models.py # Data structures +└── logging_config.py # Logging setup + +tests/ +├── test_scanner.py # Scanner tests +├── test_parser.py # Parser tests +├── test_analysis.py # Analysis tests +├── test_planner.py # Planner tests +└── ... # More test files ``` ## Requirements -- Python >= 3.10 -- uv (Python package manager) -- ffmpeg (optional, for video metadata extraction) +- **Python** >= 3.10 +- **uv** (Python package manager) +- **ffmpeg** (optional, for video metadata extraction) +- **find** command (standard on Unix-like systems) + +## Contributing + +Contributions are welcome! Please: + +1. Write tests for new features +2. Follow existing code style +3. Update documentation +4. Add clear commit messages + +## License + +[Add your license here] + +## Acknowledgments + +Built with: +- [Click](https://click.palletsprojects.com/) - CLI framework +- [PyYAML](https://pyyaml.org/) - Configuration parsing +- [pytest](https://pytest.org/) - Testing framework +- [hypothesis](https://hypothesis.readthedocs.io/) - Property-based testing + +## Support + +- Report issues: [GitHub Issues](https://github.com/yourusername/video-library-manager/issues) +- Documentation: See CLAUDE.md for architecture details +- Logs: Check `~/.vlm/vlm.log` for detailed information diff --git a/src/vlm/config.py b/src/vlm/config.py index 401311f..f564f71 100644 --- a/src/vlm/config.py +++ b/src/vlm/config.py @@ -19,6 +19,7 @@ class Config: series_filename_template: Filename template for series (e.g., "S{season:02d}E{episode:02d}{ext}") log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) quarantine_dir: Quarantine directory name relative to category root (e.g., ".quarantine") + categories: Mapping of category names to directory name lists for file categorization """ library_root: Path video_extensions: list[str] = field(default_factory=lambda: [ @@ -30,6 +31,11 @@ class Config: series_filename_template: str = "S{season:02d}E{episode:02d}{ext}" log_level: str = "INFO" quarantine_dir: str = ".quarantine" + categories: dict[str, list[str]] = field(default_factory=lambda: { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + }) def load_config(path: Path) -> Config: @@ -79,7 +85,14 @@ def load_config(path: Path) -> Config: # Extract other settings quarantine_dir = data.get('quarantine_dir', '.quarantine') log_level = data.get('log_level', 'INFO') - + + # Extract categories configuration + categories = data.get('categories', { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + }) + return Config( library_root=library_root, video_extensions=video_extensions, @@ -88,7 +101,8 @@ def load_config(path: Path) -> Config: movie_filename_template=movie_filename_template, series_filename_template=series_filename_template, log_level=log_level, - quarantine_dir=quarantine_dir + quarantine_dir=quarantine_dir, + categories=categories ) @@ -110,7 +124,12 @@ def create_default_config(path: Path) -> Config: movie_filename_template="{title} ({year}){ext}", series_filename_template="S{season:02d}E{episode:02d}{ext}", log_level="INFO", - quarantine_dir=".quarantine" + quarantine_dir=".quarantine", + categories={ + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } ) # Create YAML content @@ -124,7 +143,8 @@ def create_default_config(path: Path) -> Config: 'series_filename': default_config.series_filename_template }, 'quarantine_dir': default_config.quarantine_dir, - 'log_level': default_config.log_level + 'log_level': default_config.log_level, + 'categories': default_config.categories } # Ensure parent directory exists @@ -204,5 +224,47 @@ def validate_config(config: Config) -> list[str]: errors.append("quarantine_dir must be a string") elif config.quarantine_dir.startswith('/') or config.quarantine_dir.startswith('\\'): errors.append("quarantine_dir must be relative to category root, not absolute") - + + # Validate categories + if not config.categories: + errors.append("categories cannot be empty") + elif not isinstance(config.categories, dict): + errors.append("categories must be a dictionary") + else: + # Check required category keys exist + required_categories = {"movie", "series", "anime"} + missing = required_categories - set(config.categories.keys()) + if missing: + errors.append(f"categories must include keys: {sorted(missing)}") + + # Validate each category's directory list and check for duplicates + seen_dirs = {} + for category, dir_list in config.categories.items(): + if not isinstance(dir_list, list): + errors.append(f"categories['{category}'] must be a list") + continue + + if not dir_list: + errors.append(f"categories['{category}'] cannot be empty") + continue + + for dir_name in dir_list: + if not isinstance(dir_name, str): + errors.append(f"categories['{category}'] must contain strings") + break + + if not dir_name.strip(): + errors.append(f"categories['{category}'] contains empty directory name") + break + + # Check for duplicates (case-insensitive) + dir_lower = dir_name.lower() + if dir_lower in seen_dirs: + errors.append( + f"Duplicate directory name '{dir_name}' in categories " + f"'{category}' and '{seen_dirs[dir_lower]}'" + ) + else: + seen_dirs[dir_lower] = category + return errors diff --git a/src/vlm/scanner.py b/src/vlm/scanner.py index e4c85c4..05f0cde 100644 --- a/src/vlm/scanner.py +++ b/src/vlm/scanner.py @@ -1,8 +1,8 @@ """Inventory scanner for discovering and cataloging video files. This module implements the core scanning functionality for the Video Library Manager, -including recursive directory traversal, file filtering, metadata extraction, and -categorization based on directory structure. +including file discovery via `find`, metadata extraction, and categorization based on +directory structure. """ import csv @@ -12,7 +12,7 @@ import os import subprocess from datetime import datetime, timezone from pathlib import Path -from typing import Optional +from typing import Callable, Iterator, Optional from vlm.config import Config from vlm.models import VideoFile @@ -27,7 +27,11 @@ def _normalize_to_utc(timestamp: datetime) -> datetime: return timestamp.astimezone(timezone.utc) -def scan_library(root: Path, config: Config) -> list[VideoFile]: +def scan_library( + root: Path, + config: Config, + progress_callback: Optional[Callable[[int, int], None]] = None +) -> list[VideoFile]: """Recursively scan library for video files. Discovers all video files matching configured extensions within the library root, @@ -57,10 +61,23 @@ def scan_library(root: Path, config: Config) -> list[VideoFile]: video_files = [] file_count = 0 - # Recursively scan directory tree - for video_file in _scan_directory_recursive(root, config, root): + discovered_paths = _discover_video_paths(root, config.video_extensions) + total_paths = len(discovered_paths) + + if progress_callback is not None: + progress_callback(0, total_paths) + + for index, file_path in enumerate(discovered_paths, start=1): + video_file = _create_video_file(file_path, root, config.categories) + if video_file is None: + if progress_callback is not None: + progress_callback(index, total_paths) + continue video_files.append(video_file) file_count += 1 + + if progress_callback is not None: + progress_callback(index, total_paths) if file_count % 100 == 0: logger.debug(f"Scanned {file_count} files so far...") @@ -69,20 +86,87 @@ def scan_library(root: Path, config: Config) -> list[VideoFile]: return video_files +def _discover_video_paths(root: Path, video_extensions: list[str]) -> list[Path]: + """Discover matching video files under root. + + Uses the system `find` command for traversal speed and falls back to Python + recursion if `find` is unavailable. + """ + try: + return _discover_video_paths_with_find(root, video_extensions) + except FileNotFoundError: + logger.warning("`find` command not available - falling back to Python recursion") + return _discover_video_paths_recursive(root, video_extensions) + + +def _discover_video_paths_with_find(root: Path, video_extensions: list[str]) -> list[Path]: + """Discover matching video files using the system `find` command.""" + normalized_extensions = [ext.lower() for ext in video_extensions if ext] + if not normalized_extensions: + return [] + + command: list[str] = ["find", str(root), "-type", "f", "("] + for index, extension in enumerate(normalized_extensions): + if index > 0: + command.append("-o") + command.extend(["-iname", f"*{extension}"]) + command.extend([")", "-print0"]) + + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout, stderr = process.communicate() + + if process.returncode != 0: + stderr_text = stderr.decode(errors="replace").strip() + if stderr_text: + logger.warning(f"find reported issues while scanning: {stderr_text}") + + discovered_paths: list[Path] = [] + for path_bytes in stdout.split(b"\0"): + if not path_bytes: + continue + file_path = Path(os.fsdecode(path_bytes)) + if _is_hidden_path(file_path, root): + continue + discovered_paths.append(file_path) + + return discovered_paths + + +def _discover_video_paths_recursive(root: Path, video_extensions: list[str]) -> list[Path]: + """Fallback discovery using Python directory traversal.""" + discovered_paths: list[Path] = [] + for file_path in _scan_directory_recursive(root, video_extensions): + if _is_hidden_path(file_path, root): + continue + discovered_paths.append(file_path) + return discovered_paths + + +def _is_hidden_path(file_path: Path, library_root: Path) -> bool: + """Return True when any path component under root starts with a dot.""" + try: + relative_path = file_path.relative_to(library_root) + except ValueError: + return file_path.name.startswith(".") + return any(part.startswith(".") for part in relative_path.parts) + + def _scan_directory_recursive( directory: Path, - config: Config, - library_root: Path -) -> list[VideoFile]: + video_extensions: list[str] +) -> Iterator[Path]: """Recursively scan a directory for video files. Args: directory: Directory to scan - config: Configuration object - library_root: Root of the library (for categorization) + video_extensions: List of configured video extensions Yields: - VideoFile objects for each discovered video file + File paths for each discovered video file """ try: # Use os.scandir for efficient directory traversal @@ -96,15 +180,13 @@ def _scan_directory_recursive( if entry.is_file(follow_symlinks=False): # Check if file has video extension file_path = Path(entry.path) - if _is_video_file(file_path, config.video_extensions): - video_file = _create_video_file(file_path, library_root) - if video_file: - yield video_file + if _is_video_file(file_path, video_extensions): + yield file_path elif entry.is_dir(follow_symlinks=False): # Recursively scan subdirectory subdir_path = Path(entry.path) - yield from _scan_directory_recursive(subdir_path, config, library_root) + yield from _scan_directory_recursive(subdir_path, video_extensions) except (OSError, PermissionError) as e: # Handle inaccessible files/directories gracefully @@ -130,16 +212,21 @@ def _is_video_file(file_path: Path, video_extensions: list[str]) -> bool: return file_extension in [ext.lower() for ext in video_extensions] -def _create_video_file(file_path: Path, library_root: Path) -> Optional[VideoFile]: +def _create_video_file( + file_path: Path, + library_root: Path, + categories_config: dict[str, list[str]] +) -> Optional[VideoFile]: """Create VideoFile object from file path. - + Extracts file metadata and categorizes based on directory structure. Optionally extracts video metadata using ffprobe if available. - + Args: file_path: Path to video file library_root: Root of the library (for categorization) - + categories_config: Mapping of category names to directory name lists + Returns: VideoFile object or None if file cannot be accessed """ @@ -150,7 +237,7 @@ def _create_video_file(file_path: Path, library_root: Path) -> Optional[VideoFil modified_timestamp = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc) # Categorize based on directory structure - category = categorize_file(file_path, library_root) + category = categorize_file(file_path, library_root, categories_config) # Extract video metadata using ffprobe (optional, non-blocking) video_metadata = extract_metadata(file_path) @@ -173,43 +260,46 @@ def _create_video_file(file_path: Path, library_root: Path) -> Optional[VideoFil return None -def categorize_file(file_path: Path, library_root: Path) -> str: +def categorize_file( + file_path: Path, + library_root: Path, + categories_config: dict[str, list[str]] +) -> str: """Determine category based on directory structure. - - Categories are determined by the top-level directory within the library root: - - movie/ -> "movie" - - series/ -> "series" - - anime/ -> "anime" - - other/ or anything else -> "other" - + + Categories are determined by the top-level directory within the library root, + matched against configured category mappings. + Args: file_path: Path to video file library_root: Root of the library - + categories_config: Mapping of category names to directory name lists + Example: {"movie": ["movie", "movies"], "series": ["series", "tv"]} + Returns: Category string: "movie", "series", "anime", or "other" """ try: # Get relative path from library root relative_path = file_path.relative_to(library_root) - + # Get the first component of the relative path (top-level directory) parts = relative_path.parts if len(parts) > 0: top_level_dir = parts[0].lower() - - if top_level_dir == "movie": - return "movie" - elif top_level_dir == "series": - return "series" - elif top_level_dir == "anime": - return "anime" - else: - return "other" + + # Check against configured category mappings + for category, dir_names in categories_config.items(): + # Case-insensitive matching + if top_level_dir in [name.lower() for name in dir_names]: + return category + + # No match found + return "other" else: # File is directly in library root return "other" - + except ValueError: # File is not within library root logger.warning(f"File {file_path} is not within library root {library_root}") diff --git a/tests/test_config.py b/tests/test_config.py index ab5edda..31ad349 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -29,12 +29,36 @@ class TestConfig: movie_template="movies/{title}-{year}/", log_level="DEBUG" ) - + assert config.library_root == Path("/custom/path") assert config.video_extensions == [".mp4", ".avi"] assert config.movie_template == "movies/{title}-{year}/" assert config.log_level == "DEBUG" + def test_config_default_categories(self): + """Test Config has default categories.""" + config = Config(library_root=Path("/test")) + + assert config.categories == { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + def test_config_custom_categories(self): + """Test Config with custom category mappings.""" + config = Config( + library_root=Path("/test"), + categories={ + "movie": ["movie", "movies", "films"], + "series": ["series", "tv"], + "anime": ["anime"] + } + ) + + assert "movies" in config.categories["movie"] + assert "tv" in config.categories["series"] + class TestLoadConfig: """Test load_config function.""" @@ -127,18 +151,34 @@ class TestLoadConfig: config_data = { 'library_root': '/mnt/nas/videos' } - + with open(config_file, 'w') as f: yaml.dump(config_data, f) - + config = load_config(config_file) - + # Should use default values for missing fields assert config.library_root == Path('/mnt/nas/videos') assert ".mp4" in config.video_extensions assert config.movie_template == "movie/{title} ({year})/" assert config.log_level == "INFO" + def test_load_config_with_custom_categories(self, tmp_path): + """Test loading config with custom categories.""" + config_file = tmp_path / "config.yaml" + config_file.write_text(""" +library_root: /test/library +categories: + movie: [movie, movies, films] + series: [series, tv, shows] + anime: [anime] +""") + + config = load_config(config_file) + + assert config.categories["movie"] == ["movie", "movies", "films"] + assert config.categories["series"] == ["series", "tv", "shows"] + class TestCreateDefaultConfig: """Test create_default_config function.""" @@ -308,12 +348,127 @@ class TestValidateConfig: movie_template="", log_level="INVALID" ) - + errors = validate_config(config) - + # Should have multiple errors assert len(errors) >= 4 + def test_validate_empty_categories(self): + """Test validating config with empty categories.""" + config = Config(library_root=Path("/test"), categories={}) + errors = validate_config(config) + assert any("categories" in e and "empty" in e for e in errors) + + def test_validate_missing_required_category(self): + """Test validating config with missing required categories.""" + config = Config( + library_root=Path("/test"), + categories={"movie": ["movie"]} # Missing series, anime + ) + errors = validate_config(config) + assert any("series" in e or "anime" in e for e in errors) + + def test_validate_duplicate_directory_names(self): + """Test validating config with duplicate directory names.""" + config = Config( + library_root=Path("/test"), + categories={ + "movie": ["movie", "videos"], + "series": ["series", "videos"], # Duplicate + "anime": ["anime"] + } + ) + errors = validate_config(config) + assert any("Duplicate" in e and "videos" in e for e in errors) + + def test_validate_case_insensitive_duplicates(self): + """Test validating config with case-insensitive duplicates.""" + config = Config( + library_root=Path("/test"), + categories={ + "movie": ["Movie"], + "series": ["movie"], # Case-insensitive duplicate + "anime": ["anime"] + } + ) + errors = validate_config(config) + assert any("Duplicate" in e for e in errors) + + def test_validate_valid_custom_categories(self): + """Test validating config with valid custom categories.""" + config = Config( + library_root=Path("/test"), + categories={ + "movie": ["movie", "movies"], + "series": ["series", "tv"], + "anime": ["anime"] + } + ) + errors = validate_config(config) + assert errors == [] + + def test_validate_categories_not_dict(self): + """Test validating config with categories not a dict.""" + config = Config( + library_root=Path("/test"), + categories=["movie", "series"] # Wrong type + ) + errors = validate_config(config) + assert any("must be a dictionary" in e for e in errors) + + def test_validate_category_list_not_list(self): + """Test validating config with category value not a list.""" + config = Config( + library_root=Path("/test"), + categories={ + "movie": "movie", # Should be a list + "series": ["series"], + "anime": ["anime"] + } + ) + errors = validate_config(config) + assert any("must be a list" in e for e in errors) + + def test_validate_empty_category_list(self): + """Test validating config with empty category list.""" + config = Config( + library_root=Path("/test"), + categories={ + "movie": [], # Empty list + "series": ["series"], + "anime": ["anime"] + } + ) + errors = validate_config(config) + assert any("cannot be empty" in e for e in errors) + + def test_validate_category_list_with_non_string(self): + """Test validating config with non-string in category list.""" + config = Config( + library_root=Path("/test"), + categories={ + "movie": ["movie", 123], # Non-string + "series": ["series"], + "anime": ["anime"] + } + ) + errors = validate_config(config) + assert any("must contain strings" in e for e in errors) + + def test_validate_category_list_with_empty_string(self): + """Test validating config with empty string in category list.""" + config = Config( + library_root=Path("/test"), + categories={ + "movie": ["movie", ""], # Empty string + "series": ["series"], + "anime": ["anime"] + } + ) + errors = validate_config(config) + assert any("empty directory name" in e for e in errors) + class TestConfigIntegration: """Integration tests for configuration workflow.""" diff --git a/tests/test_scanner.py b/tests/test_scanner.py index 4326514..7b8bf5a 100644 --- a/tests/test_scanner.py +++ b/tests/test_scanner.py @@ -106,6 +106,46 @@ class TestScanLibrary: assert len(result) == 2 filenames = {vf.filename for vf in result} assert filenames == {"video.mp4", "video.mkv"} + + def test_scan_uses_find_output_and_filters_hidden_paths(self, tmp_path): + """Test scan_library filters hidden paths from find output.""" + movie_dir = tmp_path / "movie" + hidden_dir = tmp_path / ".hidden" + movie_dir.mkdir() + hidden_dir.mkdir() + + visible_file = movie_dir / "visible.mp4" + hidden_file = hidden_dir / "hidden.mp4" + visible_file.touch() + hidden_file.touch() + + fake_stdout = f"{visible_file}\0{hidden_file}\0".encode() + + with patch('subprocess.Popen') as mock_popen: + process = MagicMock() + process.communicate.return_value = (fake_stdout, b"") + process.returncode = 0 + mock_popen.return_value = process + + config = Config(library_root=tmp_path) + result = scan_library(tmp_path, config) + + assert len(result) == 1 + assert result[0].path == visible_file + + def test_scan_falls_back_when_find_is_unavailable(self, tmp_path): + """Test scan_library falls back to recursive scanning if find is unavailable.""" + movie_dir = tmp_path / "movie" + movie_dir.mkdir() + video_file = movie_dir / "fallback.mp4" + video_file.touch() + + with patch('subprocess.Popen', side_effect=FileNotFoundError): + config = Config(library_root=tmp_path) + result = scan_library(tmp_path, config) + + assert len(result) == 1 + assert result[0].path == video_file def test_scan_records_metadata(self, tmp_path): """Test scanning records file metadata correctly.""" @@ -200,86 +240,241 @@ class TestScanLibrary: # Should find both files (no permission errors in test environment) assert len(result) == 2 + def test_scan_reports_progress_callback(self, tmp_path): + """Test scanning reports progress updates for discovered files.""" + config = Config(library_root=tmp_path) + fake_paths = [tmp_path / "a.mp4", tmp_path / "b.mp4"] + + fake_video = VideoFile( + path=fake_paths[0], + filename="a.mp4", + size_bytes=1, + modified_timestamp=datetime.now(timezone.utc), + category="movie", + ) + + progress_events: list[tuple[int, int]] = [] + + with patch("vlm.scanner._discover_video_paths", return_value=fake_paths), patch( + "vlm.scanner._create_video_file", + side_effect=[fake_video, None] + ): + result = scan_library( + tmp_path, + config, + progress_callback=lambda current, total: progress_events.append((current, total)) + ) + + assert len(result) == 1 + assert progress_events == [(0, 2), (1, 2), (2, 2)] + + def test_scan_reports_progress_for_empty_discovery(self, tmp_path): + """Test scanning reports zero progress when no files are discovered.""" + config = Config(library_root=tmp_path) + progress_events: list[tuple[int, int]] = [] + + with patch("vlm.scanner._discover_video_paths", return_value=[]): + result = scan_library( + tmp_path, + config, + progress_callback=lambda current, total: progress_events.append((current, total)) + ) + + assert result == [] + assert progress_events == [(0, 0)] + class TestCategorizeFile: """Tests for the categorize_file function.""" - + def test_categorize_movie(self, tmp_path): """Test categorizing a file in movie directory.""" movie_dir = tmp_path / "movie" movie_dir.mkdir() video_file = movie_dir / "test.mp4" video_file.touch() - - category = categorize_file(video_file, tmp_path) + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) assert category == "movie" - + def test_categorize_series(self, tmp_path): """Test categorizing a file in series directory.""" series_dir = tmp_path / "series" series_dir.mkdir() video_file = series_dir / "test.mkv" video_file.touch() - - category = categorize_file(video_file, tmp_path) + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) assert category == "series" - + def test_categorize_anime(self, tmp_path): """Test categorizing a file in anime directory.""" anime_dir = tmp_path / "anime" anime_dir.mkdir() video_file = anime_dir / "test.avi" video_file.touch() - - category = categorize_file(video_file, tmp_path) + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) assert category == "anime" - + def test_categorize_other(self, tmp_path): """Test categorizing a file in other directory.""" other_dir = tmp_path / "other" other_dir.mkdir() video_file = other_dir / "test.mov" video_file.touch() - - category = categorize_file(video_file, tmp_path) + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) assert category == "other" - + def test_categorize_nested_file(self, tmp_path): """Test categorizing a file in nested subdirectory.""" movie_dir = tmp_path / "movie" / "subdir" / "nested" movie_dir.mkdir(parents=True) video_file = movie_dir / "test.mp4" video_file.touch() - - category = categorize_file(video_file, tmp_path) + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) assert category == "movie" - + def test_categorize_case_insensitive(self, tmp_path): """Test categorization is case-insensitive.""" movie_dir = tmp_path / "Movie" movie_dir.mkdir() video_file = movie_dir / "test.mp4" video_file.touch() - - category = categorize_file(video_file, tmp_path) + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) assert category == "movie" - + def test_categorize_file_in_root(self, tmp_path): """Test categorizing a file directly in library root.""" video_file = tmp_path / "test.mp4" video_file.touch() - - category = categorize_file(video_file, tmp_path) + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) assert category == "other" - + def test_categorize_unknown_directory(self, tmp_path): """Test categorizing a file in unknown directory.""" unknown_dir = tmp_path / "random" unknown_dir.mkdir() video_file = unknown_dir / "test.mp4" video_file.touch() - - category = categorize_file(video_file, tmp_path) + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) + assert category == "other" + + def test_categorize_plural_movies(self, tmp_path): + """Test recognizing plural 'movies' directory.""" + movies_dir = tmp_path / "movies" + movies_dir.mkdir() + video_file = movies_dir / "test.mp4" + video_file.touch() + + categories_config = { + "movie": ["movie", "movies"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) + assert category == "movie" + + def test_categorize_tv_directory(self, tmp_path): + """Test recognizing 'tv' as series category.""" + tv_dir = tmp_path / "tv" + tv_dir.mkdir() + video_file = tv_dir / "show.mkv" + video_file.touch() + + categories_config = { + "movie": ["movie"], + "series": ["series", "tv", "shows"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) + assert category == "series" + + def test_categorize_custom_case_insensitive(self, tmp_path): + """Test case-insensitive matching with custom mappings.""" + movies_dir = tmp_path / "MOVIES" + movies_dir.mkdir() + video_file = movies_dir / "test.mp4" + video_file.touch() + + categories_config = { + "movie": ["movie", "movies"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) + assert category == "movie" + + def test_categorize_unmapped_returns_other(self, tmp_path): + """Test unmapped directory returns 'other'.""" + downloads_dir = tmp_path / "downloads" + downloads_dir.mkdir() + video_file = downloads_dir / "file.mp4" + video_file.touch() + + categories_config = { + "movie": ["movie"], + "series": ["series"], + "anime": ["anime"] + } + + category = categorize_file(video_file, tmp_path, categories_config) assert category == "other"