windy/dl-organizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add configurable category mappings for directory recognition

Browse Source 
Allow users to configure multiple directory names per category (movie/series/anime) to support variations like "movies", "tv", "films". This enables proper categorization of files in directories that don't match the hardcoded singular forms, solving the issue where 635 files in "/mnt/Downloads/movies/" were incorrectly categorized as "other".

Configuration example:
  categories:
    movie: [movie, movies, films]
    series: [series, tv, shows]
    anime: [anime]

Changes include comprehensive validation, backward-compatible defaults, case-insensitive matching, and full test coverage (395 tests passing).

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
windyboy
2026-02-09 22:40:48 +08:00
co-authored by Claude Sonnet 4.5
parent aa0dc8ec47
commit 259e7506d7
6 changed files with 1380 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vlm/config.py
+67 -5
View File
@@ -19,6 +19,7 @@ class Config:
series_filename_template: Filename template for series (e.g., "S{season:02d}E{episode:02d}{ext}")
log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
quarantine_dir: Quarantine directory name relative to category root (e.g., ".quarantine")
categories: Mapping of category names to directory name lists for file categorization
"""
library_root: Path
video_extensions: list[str] = field(default_factory=lambda: [
@@ -30,6 +31,11 @@ class Config:
series_filename_template: str = "S{season:02d}E{episode:02d}{ext}"
log_level: str = "INFO"
quarantine_dir: str = ".quarantine"
categories: dict[str, list[str]] = field(default_factory=lambda: {
"movie": ["movie"],
"series": ["series"],
"anime": ["anime"]
})
def load_config(path: Path) -> Config:
@@ -79,7 +85,14 @@ def load_config(path: Path) -> Config:
# Extract other settings
quarantine_dir = data.get('quarantine_dir', '.quarantine')
log_level = data.get('log_level', 'INFO')
# Extract categories configuration
categories = data.get('categories', {
"movie": ["movie"],
"series": ["series"],
"anime": ["anime"]
})
return Config(
library_root=library_root,
video_extensions=video_extensions,
@@ -88,7 +101,8 @@ def load_config(path: Path) -> Config:
movie_filename_template=movie_filename_template,
series_filename_template=series_filename_template,
log_level=log_level,
quarantine_dir=quarantine_dir
quarantine_dir=quarantine_dir,
categories=categories
)
@@ -110,7 +124,12 @@ def create_default_config(path: Path) -> Config:
movie_filename_template="{title} ({year}){ext}",
series_filename_template="S{season:02d}E{episode:02d}{ext}",
log_level="INFO",
quarantine_dir=".quarantine"
quarantine_dir=".quarantine",
categories={
"movie": ["movie"],
"series": ["series"],
"anime": ["anime"]
}
)
# Create YAML content
@@ -124,7 +143,8 @@ def create_default_config(path: Path) -> Config:
'series_filename': default_config.series_filename_template
},
'quarantine_dir': default_config.quarantine_dir,
'log_level': default_config.log_level
'log_level': default_config.log_level,
'categories': default_config.categories
}
# Ensure parent directory exists
@@ -204,5 +224,47 @@ def validate_config(config: Config) -> list[str]:
errors.append("quarantine_dir must be a string")
elif config.quarantine_dir.startswith('/') or config.quarantine_dir.startswith('\\'):
errors.append("quarantine_dir must be relative to category root, not absolute")
# Validate categories
if not config.categories:
errors.append("categories cannot be empty")
elif not isinstance(config.categories, dict):
errors.append("categories must be a dictionary")
else:
# Check required category keys exist
required_categories = {"movie", "series", "anime"}
missing = required_categories - set(config.categories.keys())
if missing:
errors.append(f"categories must include keys: {sorted(missing)}")
# Validate each category's directory list and check for duplicates
seen_dirs = {}
for category, dir_list in config.categories.items():
if not isinstance(dir_list, list):
errors.append(f"categories['{category}'] must be a list")
continue
if not dir_list:
errors.append(f"categories['{category}'] cannot be empty")
continue
for dir_name in dir_list:
if not isinstance(dir_name, str):
errors.append(f"categories['{category}'] must contain strings")
break
if not dir_name.strip():
errors.append(f"categories['{category}'] contains empty directory name")
break
# Check for duplicates (case-insensitive)
dir_lower = dir_name.lower()
if dir_lower in seen_dirs:
errors.append(
f"Duplicate directory name '{dir_name}' in categories "
f"'{category}' and '{seen_dirs[dir_lower]}'"
)
else:
seen_dirs[dir_lower] = category
return errors
src/vlm/scanner.py
+132 -42
View File
@@ -1,8 +1,8 @@
"""Inventory scanner for discovering and cataloging video files.
This module implements the core scanning functionality for the Video Library Manager,
including recursive directory traversal, file filtering, metadata extraction, and
categorization based on directory structure.
including file discovery via `find`, metadata extraction, and categorization based on
directory structure.
"""
import csv
@@ -12,7 +12,7 @@ import os
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional
from typing import Callable, Iterator, Optional
from vlm.config import Config
from vlm.models import VideoFile
@@ -27,7 +27,11 @@ def _normalize_to_utc(timestamp: datetime) -> datetime:
return timestamp.astimezone(timezone.utc)
def scan_library(root: Path, config: Config) -> list[VideoFile]:
def scan_library(
root: Path,
config: Config,
progress_callback: Optional[Callable[[int, int], None]] = None
) -> list[VideoFile]:
"""Recursively scan library for video files.
Discovers all video files matching configured extensions within the library root,
@@ -57,10 +61,23 @@ def scan_library(root: Path, config: Config) -> list[VideoFile]:
video_files = []
file_count = 0
# Recursively scan directory tree
for video_file in _scan_directory_recursive(root, config, root):
discovered_paths = _discover_video_paths(root, config.video_extensions)
total_paths = len(discovered_paths)
if progress_callback is not None:
progress_callback(0, total_paths)
for index, file_path in enumerate(discovered_paths, start=1):
video_file = _create_video_file(file_path, root, config.categories)
if video_file is None:
if progress_callback is not None:
progress_callback(index, total_paths)
continue
video_files.append(video_file)
file_count += 1
if progress_callback is not None:
progress_callback(index, total_paths)
if file_count % 100 == 0:
logger.debug(f"Scanned {file_count} files so far...")
@@ -69,20 +86,87 @@ def scan_library(root: Path, config: Config) -> list[VideoFile]:
return video_files
def _discover_video_paths(root: Path, video_extensions: list[str]) -> list[Path]:
"""Discover matching video files under root.
Uses the system `find` command for traversal speed and falls back to Python
recursion if `find` is unavailable.
"""
try:
return _discover_video_paths_with_find(root, video_extensions)
except FileNotFoundError:
logger.warning("`find` command not available - falling back to Python recursion")
return _discover_video_paths_recursive(root, video_extensions)
def _discover_video_paths_with_find(root: Path, video_extensions: list[str]) -> list[Path]:
"""Discover matching video files using the system `find` command."""
normalized_extensions = [ext.lower() for ext in video_extensions if ext]
if not normalized_extensions:
return []
command: list[str] = ["find", str(root), "-type", "f", "("]
for index, extension in enumerate(normalized_extensions):
if index > 0:
command.append("-o")
command.extend(["-iname", f"*{extension}"])
command.extend([")", "-print0"])
process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
stdout, stderr = process.communicate()
if process.returncode != 0:
stderr_text = stderr.decode(errors="replace").strip()
if stderr_text:
logger.warning(f"find reported issues while scanning: {stderr_text}")
discovered_paths: list[Path] = []
for path_bytes in stdout.split(b"\0"):
if not path_bytes:
continue
file_path = Path(os.fsdecode(path_bytes))
if _is_hidden_path(file_path, root):
continue
discovered_paths.append(file_path)
return discovered_paths
def _discover_video_paths_recursive(root: Path, video_extensions: list[str]) -> list[Path]:
"""Fallback discovery using Python directory traversal."""
discovered_paths: list[Path] = []
for file_path in _scan_directory_recursive(root, video_extensions):
if _is_hidden_path(file_path, root):
continue
discovered_paths.append(file_path)
return discovered_paths
def _is_hidden_path(file_path: Path, library_root: Path) -> bool:
"""Return True when any path component under root starts with a dot."""
try:
relative_path = file_path.relative_to(library_root)
except ValueError:
return file_path.name.startswith(".")
return any(part.startswith(".") for part in relative_path.parts)
def _scan_directory_recursive(
directory: Path,
config: Config,
library_root: Path
) -> list[VideoFile]:
video_extensions: list[str]
) -> Iterator[Path]:
"""Recursively scan a directory for video files.
Args:
directory: Directory to scan
config: Configuration object
library_root: Root of the library (for categorization)
video_extensions: List of configured video extensions
Yields:
VideoFile objects for each discovered video file
File paths for each discovered video file
"""
try:
# Use os.scandir for efficient directory traversal
@@ -96,15 +180,13 @@ def _scan_directory_recursive(
if entry.is_file(follow_symlinks=False):
# Check if file has video extension
file_path = Path(entry.path)
if _is_video_file(file_path, config.video_extensions):
video_file = _create_video_file(file_path, library_root)
if video_file:
yield video_file
if _is_video_file(file_path, video_extensions):
yield file_path
elif entry.is_dir(follow_symlinks=False):
# Recursively scan subdirectory
subdir_path = Path(entry.path)
yield from _scan_directory_recursive(subdir_path, config, library_root)
yield from _scan_directory_recursive(subdir_path, video_extensions)
except (OSError, PermissionError) as e:
# Handle inaccessible files/directories gracefully
@@ -130,16 +212,21 @@ def _is_video_file(file_path: Path, video_extensions: list[str]) -> bool:
return file_extension in [ext.lower() for ext in video_extensions]
def _create_video_file(file_path: Path, library_root: Path) -> Optional[VideoFile]:
def _create_video_file(
file_path: Path,
library_root: Path,
categories_config: dict[str, list[str]]
) -> Optional[VideoFile]:
"""Create VideoFile object from file path.
Extracts file metadata and categorizes based on directory structure.
Optionally extracts video metadata using ffprobe if available.
Args:
file_path: Path to video file
library_root: Root of the library (for categorization)
categories_config: Mapping of category names to directory name lists
Returns:
VideoFile object or None if file cannot be accessed
"""
@@ -150,7 +237,7 @@ def _create_video_file(file_path: Path, library_root: Path) -> Optional[VideoFil
modified_timestamp = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
# Categorize based on directory structure
category = categorize_file(file_path, library_root)
category = categorize_file(file_path, library_root, categories_config)
# Extract video metadata using ffprobe (optional, non-blocking)
video_metadata = extract_metadata(file_path)
@@ -173,43 +260,46 @@ def _create_video_file(file_path: Path, library_root: Path) -> Optional[VideoFil
return None
def categorize_file(file_path: Path, library_root: Path) -> str:
def categorize_file(
file_path: Path,
library_root: Path,
categories_config: dict[str, list[str]]
) -> str:
"""Determine category based on directory structure.
Categories are determined by the top-level directory within the library root:
- movie/ -> "movie"
- series/ -> "series"
- anime/ -> "anime"
- other/ or anything else -> "other"
Categories are determined by the top-level directory within the library root,
matched against configured category mappings.
Args:
file_path: Path to video file
library_root: Root of the library
categories_config: Mapping of category names to directory name lists
Example: {"movie": ["movie", "movies"], "series": ["series", "tv"]}
Returns:
Category string: "movie", "series", "anime", or "other"
"""
try:
# Get relative path from library root
relative_path = file_path.relative_to(library_root)
# Get the first component of the relative path (top-level directory)
parts = relative_path.parts
if len(parts) > 0:
top_level_dir = parts[0].lower()
if top_level_dir == "movie":
return "movie"
elif top_level_dir == "series":
return "series"
elif top_level_dir == "anime":
return "anime"
else:
return "other"
# Check against configured category mappings
for category, dir_names in categories_config.items():
# Case-insensitive matching
if top_level_dir in [name.lower() for name in dir_names]:
return category
# No match found
return "other"
else:
# File is directly in library root
return "other"
except ValueError:
# File is not within library root
logger.warning(f"File {file_path} is not within library root {library_root}")