Initial commit: Video Library Manager
- Add core VLM modules (scanner, parser, planner, executor, analysis) - Add CLI with quarantine, reports, rollback, and state management - Add comprehensive test suite - Add project configuration and documentation - Add .gitignore for Python project
This commit is contained in:
windyboy
@@ -0,0 +1,460 @@
|
||||
"""Inventory scanner for discovering and cataloging video files.
|
||||
|
||||
This module implements the core scanning functionality for the Video Library Manager,
|
||||
including recursive directory traversal, file filtering, metadata extraction, and
|
||||
categorization based on directory structure.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from vlm.config import Config
|
||||
from vlm.models import VideoFile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def scan_library(root: Path, config: Config) -> list[VideoFile]:
|
||||
"""Recursively scan library for video files.
|
||||
|
||||
Discovers all video files matching configured extensions within the library root,
|
||||
records their metadata, and categorizes them based on directory structure.
|
||||
|
||||
Args:
|
||||
root: Root directory to scan
|
||||
config: Configuration object with video extensions and settings
|
||||
|
||||
Returns:
|
||||
List of VideoFile objects representing discovered files
|
||||
|
||||
Note:
|
||||
- Handles inaccessible files gracefully by logging errors and continuing
|
||||
- Performs read-only operations without modifying any files or directories
|
||||
- Categorizes files based on parent directory structure (movie/series/anime/other)
|
||||
"""
|
||||
logger.info(f"Starting library scan at: {root}")
|
||||
|
||||
if not root.exists():
|
||||
logger.error(f"Library root does not exist: {root}")
|
||||
return []
|
||||
|
||||
if not root.is_dir():
|
||||
logger.error(f"Library root is not a directory: {root}")
|
||||
return []
|
||||
|
||||
video_files = []
|
||||
file_count = 0
|
||||
error_count = 0
|
||||
|
||||
# Recursively scan directory tree
|
||||
for video_file in _scan_directory_recursive(root, config, root):
|
||||
video_files.append(video_file)
|
||||
file_count += 1
|
||||
|
||||
if file_count % 100 == 0:
|
||||
logger.debug(f"Scanned {file_count} files so far...")
|
||||
|
||||
logger.info(f"Scan complete. Found {file_count} video files")
|
||||
if error_count > 0:
|
||||
logger.warning(f"Encountered {error_count} errors during scan (see log for details)")
|
||||
|
||||
return video_files
|
||||
|
||||
|
||||
def _scan_directory_recursive(
|
||||
directory: Path,
|
||||
config: Config,
|
||||
library_root: Path
|
||||
) -> list[VideoFile]:
|
||||
"""Recursively scan a directory for video files.
|
||||
|
||||
Args:
|
||||
directory: Directory to scan
|
||||
config: Configuration object
|
||||
library_root: Root of the library (for categorization)
|
||||
|
||||
Yields:
|
||||
VideoFile objects for each discovered video file
|
||||
"""
|
||||
try:
|
||||
# Use os.scandir for efficient directory traversal
|
||||
with os.scandir(directory) as entries:
|
||||
for entry in entries:
|
||||
try:
|
||||
# Skip hidden files and directories (starting with .)
|
||||
if entry.name.startswith('.'):
|
||||
continue
|
||||
|
||||
if entry.is_file(follow_symlinks=False):
|
||||
# Check if file has video extension
|
||||
file_path = Path(entry.path)
|
||||
if _is_video_file(file_path, config.video_extensions):
|
||||
video_file = _create_video_file(file_path, library_root)
|
||||
if video_file:
|
||||
yield video_file
|
||||
|
||||
elif entry.is_dir(follow_symlinks=False):
|
||||
# Recursively scan subdirectory
|
||||
subdir_path = Path(entry.path)
|
||||
yield from _scan_directory_recursive(subdir_path, config, library_root)
|
||||
|
||||
except (OSError, PermissionError) as e:
|
||||
# Handle inaccessible files/directories gracefully
|
||||
logger.error(f"Cannot access {entry.path}: {e}")
|
||||
continue
|
||||
|
||||
except (OSError, PermissionError) as e:
|
||||
# Handle inaccessible directory
|
||||
logger.error(f"Cannot access directory {directory}: {e}")
|
||||
|
||||
|
||||
def _is_video_file(file_path: Path, video_extensions: list[str]) -> bool:
|
||||
"""Check if file has a video extension.
|
||||
|
||||
Args:
|
||||
file_path: Path to file
|
||||
video_extensions: List of valid video extensions (e.g., [".mp4", ".mkv"])
|
||||
|
||||
Returns:
|
||||
True if file has a video extension, False otherwise
|
||||
"""
|
||||
file_extension = file_path.suffix.lower()
|
||||
return file_extension in [ext.lower() for ext in video_extensions]
|
||||
|
||||
|
||||
def _create_video_file(file_path: Path, library_root: Path) -> Optional[VideoFile]:
|
||||
"""Create VideoFile object from file path.
|
||||
|
||||
Extracts file metadata and categorizes based on directory structure.
|
||||
Optionally extracts video metadata using ffprobe if available.
|
||||
|
||||
Args:
|
||||
file_path: Path to video file
|
||||
library_root: Root of the library (for categorization)
|
||||
|
||||
Returns:
|
||||
VideoFile object or None if file cannot be accessed
|
||||
"""
|
||||
try:
|
||||
# Get file stats
|
||||
stat = file_path.stat()
|
||||
size_bytes = stat.st_size
|
||||
modified_timestamp = datetime.fromtimestamp(stat.st_mtime)
|
||||
|
||||
# Categorize based on directory structure
|
||||
category = categorize_file(file_path, library_root)
|
||||
|
||||
# Extract video metadata using ffprobe (optional, non-blocking)
|
||||
video_metadata = extract_metadata(file_path)
|
||||
|
||||
# Create VideoFile object with optional metadata
|
||||
return VideoFile(
|
||||
path=file_path,
|
||||
filename=file_path.name,
|
||||
size_bytes=size_bytes,
|
||||
modified_timestamp=modified_timestamp,
|
||||
category=category,
|
||||
resolution=video_metadata.get('resolution'),
|
||||
codec=video_metadata.get('codec'),
|
||||
duration_seconds=video_metadata.get('duration_seconds'),
|
||||
bitrate_kbps=video_metadata.get('bitrate_kbps')
|
||||
)
|
||||
|
||||
except (OSError, PermissionError) as e:
|
||||
logger.error(f"Cannot read file metadata for {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def categorize_file(file_path: Path, library_root: Path) -> str:
|
||||
"""Determine category based on directory structure.
|
||||
|
||||
Categories are determined by the top-level directory within the library root:
|
||||
- movie/ -> "movie"
|
||||
- series/ -> "series"
|
||||
- anime/ -> "anime"
|
||||
- other/ or anything else -> "other"
|
||||
|
||||
Args:
|
||||
file_path: Path to video file
|
||||
library_root: Root of the library
|
||||
|
||||
Returns:
|
||||
Category string: "movie", "series", "anime", or "other"
|
||||
"""
|
||||
try:
|
||||
# Get relative path from library root
|
||||
relative_path = file_path.relative_to(library_root)
|
||||
|
||||
# Get the first component of the relative path (top-level directory)
|
||||
parts = relative_path.parts
|
||||
if len(parts) > 0:
|
||||
top_level_dir = parts[0].lower()
|
||||
|
||||
if top_level_dir == "movie":
|
||||
return "movie"
|
||||
elif top_level_dir == "series":
|
||||
return "series"
|
||||
elif top_level_dir == "anime":
|
||||
return "anime"
|
||||
else:
|
||||
return "other"
|
||||
else:
|
||||
# File is directly in library root
|
||||
return "other"
|
||||
|
||||
except ValueError:
|
||||
# File is not within library root
|
||||
logger.warning(f"File {file_path} is not within library root {library_root}")
|
||||
return "other"
|
||||
|
||||
|
||||
def extract_metadata(file_path: Path) -> dict:
|
||||
"""Extract video metadata using ffprobe.
|
||||
|
||||
Attempts to extract resolution, codec, duration, and bitrate from video file
|
||||
using ffprobe. If ffprobe is not available or fails, returns empty dict.
|
||||
This is a non-blocking operation that gracefully handles failures.
|
||||
|
||||
Args:
|
||||
file_path: Path to video file
|
||||
|
||||
Returns:
|
||||
Dictionary with optional keys:
|
||||
- resolution: str (e.g., "1920x1080")
|
||||
- codec: str (e.g., "h264")
|
||||
- duration_seconds: float
|
||||
- bitrate_kbps: int
|
||||
|
||||
Note:
|
||||
- Returns empty dict if ffprobe is not available
|
||||
- Returns empty dict if ffprobe fails to extract metadata
|
||||
- Logs warnings for failures but does not raise exceptions
|
||||
"""
|
||||
try:
|
||||
# Run ffprobe to get video stream information in JSON format
|
||||
result = subprocess.run(
|
||||
[
|
||||
'ffprobe',
|
||||
'-v', 'quiet', # Suppress ffprobe output
|
||||
'-print_format', 'json', # Output as JSON
|
||||
'-show_streams', # Show stream information
|
||||
'-show_format', # Show format information
|
||||
str(file_path)
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10 # 10 second timeout to prevent hanging
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.debug(f"ffprobe failed for {file_path.name}: {result.stderr}")
|
||||
return {}
|
||||
|
||||
# Parse JSON output
|
||||
probe_data = json.loads(result.stdout)
|
||||
|
||||
# Extract metadata from the first video stream
|
||||
metadata = {}
|
||||
|
||||
# Find the first video stream
|
||||
video_stream = None
|
||||
for stream in probe_data.get('streams', []):
|
||||
if stream.get('codec_type') == 'video':
|
||||
video_stream = stream
|
||||
break
|
||||
|
||||
if video_stream:
|
||||
# Extract resolution
|
||||
width = video_stream.get('width')
|
||||
height = video_stream.get('height')
|
||||
if width and height:
|
||||
metadata['resolution'] = f"{width}x{height}"
|
||||
|
||||
# Extract codec
|
||||
codec_name = video_stream.get('codec_name')
|
||||
if codec_name:
|
||||
metadata['codec'] = codec_name
|
||||
|
||||
# Extract duration and bitrate from format section
|
||||
format_info = probe_data.get('format', {})
|
||||
|
||||
# Extract duration
|
||||
duration = format_info.get('duration')
|
||||
if duration:
|
||||
try:
|
||||
metadata['duration_seconds'] = float(duration)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Extract bitrate
|
||||
bitrate = format_info.get('bit_rate')
|
||||
if bitrate:
|
||||
try:
|
||||
# Convert from bits/sec to kbits/sec
|
||||
metadata['bitrate_kbps'] = int(float(bitrate) / 1000)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if metadata:
|
||||
logger.debug(f"Extracted metadata for {file_path.name}: {metadata}")
|
||||
|
||||
return metadata
|
||||
|
||||
except FileNotFoundError:
|
||||
# ffprobe not installed or not in PATH
|
||||
logger.debug("ffprobe not available - skipping metadata extraction")
|
||||
return {}
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(f"ffprobe timeout for {file_path.name} - skipping metadata")
|
||||
return {}
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Failed to parse ffprobe output for {file_path.name}: {e}")
|
||||
return {}
|
||||
|
||||
except Exception as e:
|
||||
# Catch any other unexpected errors
|
||||
logger.warning(f"Unexpected error extracting metadata for {file_path.name}: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
def save_inventory_csv(files: list[VideoFile], output: Path, library_root: Path) -> None:
|
||||
"""Save inventory to CSV format (primary format).
|
||||
|
||||
Generates a CSV file with all file metadata following the defined schema:
|
||||
path, filename, size_bytes, modified_timestamp, category, resolution, codec,
|
||||
duration_seconds, bitrate_kbps
|
||||
|
||||
Args:
|
||||
files: List of VideoFile objects to export
|
||||
output: Path to output CSV file
|
||||
library_root: Root of the library (included in report metadata)
|
||||
|
||||
Note:
|
||||
- Timestamps are formatted as ISO 8601 (YYYY-MM-DDTHH:MM:SS) in UTC
|
||||
- Missing optional values are represented as empty strings
|
||||
- Header row is always present with column names
|
||||
- Generation timestamp and library root are included as comment lines
|
||||
"""
|
||||
logger.info(f"Saving inventory to CSV: {output}")
|
||||
|
||||
# Ensure output directory exists
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get generation timestamp in UTC
|
||||
generation_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
with open(output, 'w', newline='', encoding='utf-8') as csvfile:
|
||||
# Write metadata as comments
|
||||
csvfile.write(f"# Generated: {generation_timestamp}\n")
|
||||
csvfile.write(f"# Library Root: {library_root}\n")
|
||||
|
||||
# Define CSV schema
|
||||
fieldnames = [
|
||||
'path',
|
||||
'filename',
|
||||
'size_bytes',
|
||||
'modified_timestamp',
|
||||
'category',
|
||||
'resolution',
|
||||
'codec',
|
||||
'duration_seconds',
|
||||
'bitrate_kbps'
|
||||
]
|
||||
|
||||
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
# Write each file
|
||||
for video_file in files:
|
||||
# Format timestamp as ISO 8601 in UTC
|
||||
if video_file.modified_timestamp.tzinfo is None:
|
||||
# Assume local time, convert to UTC
|
||||
modified_utc = video_file.modified_timestamp.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
modified_utc = video_file.modified_timestamp.astimezone(timezone.utc)
|
||||
|
||||
row = {
|
||||
'path': str(video_file.path),
|
||||
'filename': video_file.filename,
|
||||
'size_bytes': video_file.size_bytes,
|
||||
'modified_timestamp': modified_utc.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
'category': video_file.category,
|
||||
'resolution': video_file.resolution or '',
|
||||
'codec': video_file.codec or '',
|
||||
'duration_seconds': video_file.duration_seconds if video_file.duration_seconds is not None else '',
|
||||
'bitrate_kbps': video_file.bitrate_kbps if video_file.bitrate_kbps is not None else ''
|
||||
}
|
||||
writer.writerow(row)
|
||||
|
||||
logger.info(f"Saved {len(files)} files to CSV inventory")
|
||||
|
||||
|
||||
def save_inventory_json(files: list[VideoFile], output: Path, library_root: Path) -> None:
|
||||
"""Save inventory to JSON format (optional export format).
|
||||
|
||||
Generates a JSON file with all file metadata and report metadata.
|
||||
|
||||
Args:
|
||||
files: List of VideoFile objects to export
|
||||
output: Path to output JSON file
|
||||
library_root: Root of the library (included in report metadata)
|
||||
|
||||
Note:
|
||||
- Timestamps are formatted as ISO 8601 strings
|
||||
- Missing optional values are represented as null
|
||||
- Generation timestamp and library root are included in metadata section
|
||||
"""
|
||||
logger.info(f"Saving inventory to JSON: {output}")
|
||||
|
||||
# Ensure output directory exists
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get generation timestamp in UTC
|
||||
generation_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
# Build JSON structure
|
||||
inventory_data = {
|
||||
'metadata': {
|
||||
'generated': generation_timestamp,
|
||||
'library_root': str(library_root),
|
||||
'file_count': len(files)
|
||||
},
|
||||
'files': []
|
||||
}
|
||||
|
||||
# Add each file
|
||||
for video_file in files:
|
||||
# Format timestamp as ISO 8601 in UTC
|
||||
if video_file.modified_timestamp.tzinfo is None:
|
||||
# Assume local time, convert to UTC
|
||||
modified_utc = video_file.modified_timestamp.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
modified_utc = video_file.modified_timestamp.astimezone(timezone.utc)
|
||||
|
||||
file_data = {
|
||||
'path': str(video_file.path),
|
||||
'filename': video_file.filename,
|
||||
'size_bytes': video_file.size_bytes,
|
||||
'modified_timestamp': modified_utc.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
'category': video_file.category,
|
||||
'resolution': video_file.resolution,
|
||||
'codec': video_file.codec,
|
||||
'duration_seconds': video_file.duration_seconds,
|
||||
'bitrate_kbps': video_file.bitrate_kbps
|
||||
}
|
||||
inventory_data['files'].append(file_data)
|
||||
|
||||
# Write JSON file with pretty formatting
|
||||
with open(output, 'w', encoding='utf-8') as jsonfile:
|
||||
json.dump(inventory_data, jsonfile, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.info(f"Saved {len(files)} files to JSON inventory")
|
||||
Reference in New Issue
Block a user