add incremental enrich controls with progress and retry limits
This commit is contained in:
windyboy
+138
-118
@@ -8,19 +8,8 @@ import yaml
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
"""Configuration for Video Library Manager.
|
||||
|
||||
Attributes:
|
||||
library_root: Root directory of the video library
|
||||
video_extensions: List of video file extensions to recognize
|
||||
movie_template: Directory template for movies (e.g., "movie/{title} ({year})/")
|
||||
series_template: Directory template for series (e.g., "series/{title}/Season {season:02d}/")
|
||||
movie_filename_template: Filename template for movies (e.g., "{title} ({year}){ext}")
|
||||
series_filename_template: Filename template for series (e.g., "S{season:02d}E{episode:02d}{ext}")
|
||||
log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
|
||||
quarantine_dir: Quarantine directory name relative to category root (e.g., ".quarantine")
|
||||
categories: Mapping of category names to directory name lists for file categorization
|
||||
"""
|
||||
"""Configuration for Video Library Manager."""
|
||||
|
||||
library_root: Path
|
||||
video_extensions: list[str] = field(default_factory=lambda: [
|
||||
".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".m4v"
|
||||
@@ -32,67 +21,75 @@ class Config:
|
||||
log_level: str = "INFO"
|
||||
quarantine_dir: str = ".quarantine"
|
||||
categories: dict[str, list[str]] = field(default_factory=lambda: {
|
||||
"movie": ["movie"],
|
||||
"series": ["series"],
|
||||
"movie": ["movie", "movies"],
|
||||
"series": ["series", "tv", "shows"],
|
||||
"anime": ["anime"]
|
||||
})
|
||||
|
||||
# Enrichment settings
|
||||
enrichment_enabled: bool = True
|
||||
enrichment_incremental: bool = True
|
||||
enrichment_refresh_mode: str = "manual"
|
||||
enrichment_providers: list[str] = field(default_factory=lambda: ["tmdb", "douban"])
|
||||
enrichment_cache_db: Path = field(default_factory=lambda: Path.home() / ".vlm" / "enrichment_cache.db")
|
||||
enrichment_max_concurrency: int = 6
|
||||
enrichment_min_match_score: float = 0.75
|
||||
translation_mode: str = "bidirectional"
|
||||
translation_fallback_machine: bool = True
|
||||
tmdb_api_key: Optional[str] = None
|
||||
douban_api_key: Optional[str] = None
|
||||
douban_api_endpoint: Optional[str] = None
|
||||
openai_api_key: Optional[str] = None
|
||||
reputation_min_votes: int = 50
|
||||
reputation_low_score_threshold: float = 6.0
|
||||
reputation_policy: str = "flag_for_review"
|
||||
naming_title_format: str = "{title_zh} {title_en}"
|
||||
|
||||
|
||||
def load_config(path: Path) -> Config:
|
||||
"""Load configuration from YAML file.
|
||||
|
||||
Args:
|
||||
path: Path to configuration file
|
||||
|
||||
Returns:
|
||||
Config object with loaded settings
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If config file doesn't exist (caller should handle by creating default)
|
||||
yaml.YAMLError: If YAML syntax is invalid (caller should handle by using defaults)
|
||||
"""
|
||||
"""Load configuration from YAML file."""
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Configuration file not found: {path}")
|
||||
|
||||
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
except yaml.YAMLError as e:
|
||||
raise yaml.YAMLError(f"Invalid YAML syntax in configuration file: {e}")
|
||||
|
||||
|
||||
if data is None:
|
||||
data = {}
|
||||
|
||||
# Extract library_root (required field)
|
||||
library_root_str = data.get('library_root')
|
||||
|
||||
library_root_str = data.get("library_root")
|
||||
if not library_root_str:
|
||||
raise ValueError("Configuration must specify 'library_root'")
|
||||
|
||||
|
||||
library_root = Path(library_root_str).expanduser()
|
||||
|
||||
# Extract optional fields with defaults
|
||||
video_extensions = data.get('video_extensions', [
|
||||
|
||||
video_extensions = data.get("video_extensions", [
|
||||
".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".m4v"
|
||||
])
|
||||
|
||||
# Extract templates
|
||||
templates = data.get('templates', {})
|
||||
movie_template = templates.get('movie_dir', "movie/{title} ({year})/")
|
||||
series_template = templates.get('series_dir', "series/{title}/Season {season:02d}/")
|
||||
movie_filename_template = templates.get('movie_filename', "{title} ({year}){ext}")
|
||||
series_filename_template = templates.get('series_filename', "S{season:02d}E{episode:02d}{ext}")
|
||||
|
||||
# Extract other settings
|
||||
quarantine_dir = data.get('quarantine_dir', '.quarantine')
|
||||
log_level = data.get('log_level', 'INFO')
|
||||
|
||||
# Extract categories configuration
|
||||
categories = data.get('categories', {
|
||||
"movie": ["movie"],
|
||||
"series": ["series"],
|
||||
templates = data.get("templates", {})
|
||||
movie_template = templates.get("movie_dir", "movie/{title} ({year})/")
|
||||
series_template = templates.get("series_dir", "series/{title}/Season {season:02d}/")
|
||||
movie_filename_template = templates.get("movie_filename", "{title} ({year}){ext}")
|
||||
series_filename_template = templates.get("series_filename", "S{season:02d}E{episode:02d}{ext}")
|
||||
|
||||
quarantine_dir = data.get("quarantine_dir", ".quarantine")
|
||||
log_level = data.get("log_level", "INFO")
|
||||
categories = data.get("categories", {
|
||||
"movie": ["movie", "movies"],
|
||||
"series": ["series", "tv", "shows"],
|
||||
"anime": ["anime"]
|
||||
})
|
||||
|
||||
enrichment = data.get("enrichment", {})
|
||||
translation = enrichment.get("translation", {})
|
||||
api_keys = enrichment.get("api_keys", {})
|
||||
reputation = enrichment.get("reputation", {})
|
||||
naming = enrichment.get("naming", {})
|
||||
|
||||
return Config(
|
||||
library_root=library_root,
|
||||
video_extensions=video_extensions,
|
||||
@@ -102,79 +99,94 @@ def load_config(path: Path) -> Config:
|
||||
series_filename_template=series_filename_template,
|
||||
log_level=log_level,
|
||||
quarantine_dir=quarantine_dir,
|
||||
categories=categories
|
||||
categories=categories,
|
||||
enrichment_enabled=enrichment.get("enabled", True),
|
||||
enrichment_incremental=enrichment.get("incremental", True),
|
||||
enrichment_refresh_mode=enrichment.get("refresh_mode", "manual"),
|
||||
enrichment_providers=enrichment.get("providers", ["tmdb", "douban"]),
|
||||
enrichment_cache_db=Path(
|
||||
enrichment.get("cache_db", str(Path.home() / ".vlm" / "enrichment_cache.db"))
|
||||
).expanduser(),
|
||||
enrichment_max_concurrency=enrichment.get("max_concurrency", 6),
|
||||
enrichment_min_match_score=enrichment.get("min_match_score", 0.75),
|
||||
translation_mode=translation.get("mode", "bidirectional"),
|
||||
translation_fallback_machine=translation.get("fallback_machine", True),
|
||||
tmdb_api_key=api_keys.get("tmdb"),
|
||||
douban_api_key=api_keys.get("douban"),
|
||||
douban_api_endpoint=enrichment.get("douban_endpoint"),
|
||||
openai_api_key=api_keys.get("openai"),
|
||||
reputation_min_votes=reputation.get("min_votes", 50),
|
||||
reputation_low_score_threshold=reputation.get("low_score_threshold", 6.0),
|
||||
reputation_policy=reputation.get("policy", "flag_for_review"),
|
||||
naming_title_format=naming.get("title_format", "{title_zh} {title_en}"),
|
||||
)
|
||||
|
||||
|
||||
def create_default_config(path: Path) -> Config:
|
||||
"""Create a default configuration file and return the Config object.
|
||||
|
||||
Args:
|
||||
path: Path where configuration file should be created
|
||||
|
||||
Returns:
|
||||
Config object with default settings
|
||||
"""
|
||||
# Create default config object
|
||||
"""Create a default configuration file and return the Config object."""
|
||||
default_config = Config(
|
||||
library_root=Path.home() / "Videos",
|
||||
video_extensions=[".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".m4v"],
|
||||
movie_template="movie/{title} ({year})/",
|
||||
series_template="series/{title}/Season {season:02d}/",
|
||||
movie_filename_template="{title} ({year}){ext}",
|
||||
series_filename_template="S{season:02d}E{episode:02d}{ext}",
|
||||
log_level="INFO",
|
||||
quarantine_dir=".quarantine",
|
||||
categories={
|
||||
"movie": ["movie"],
|
||||
"series": ["series"],
|
||||
"anime": ["anime"]
|
||||
}
|
||||
)
|
||||
|
||||
# Create YAML content
|
||||
|
||||
yaml_content = {
|
||||
'library_root': str(default_config.library_root),
|
||||
'video_extensions': default_config.video_extensions,
|
||||
'templates': {
|
||||
'movie_dir': default_config.movie_template,
|
||||
'series_dir': default_config.series_template,
|
||||
'movie_filename': default_config.movie_filename_template,
|
||||
'series_filename': default_config.series_filename_template
|
||||
"library_root": str(default_config.library_root),
|
||||
"video_extensions": default_config.video_extensions,
|
||||
"templates": {
|
||||
"movie_dir": default_config.movie_template,
|
||||
"series_dir": default_config.series_template,
|
||||
"movie_filename": default_config.movie_filename_template,
|
||||
"series_filename": default_config.series_filename_template,
|
||||
},
|
||||
"quarantine_dir": default_config.quarantine_dir,
|
||||
"log_level": default_config.log_level,
|
||||
"categories": default_config.categories,
|
||||
"enrichment": {
|
||||
"enabled": default_config.enrichment_enabled,
|
||||
"incremental": default_config.enrichment_incremental,
|
||||
"refresh_mode": default_config.enrichment_refresh_mode,
|
||||
"providers": default_config.enrichment_providers,
|
||||
"cache_db": str(default_config.enrichment_cache_db),
|
||||
"max_concurrency": default_config.enrichment_max_concurrency,
|
||||
"min_match_score": default_config.enrichment_min_match_score,
|
||||
"translation": {
|
||||
"mode": default_config.translation_mode,
|
||||
"fallback_machine": default_config.translation_fallback_machine,
|
||||
},
|
||||
"api_keys": {
|
||||
"tmdb": default_config.tmdb_api_key,
|
||||
"douban": default_config.douban_api_key,
|
||||
"openai": default_config.openai_api_key,
|
||||
},
|
||||
"douban_endpoint": default_config.douban_api_endpoint,
|
||||
"reputation": {
|
||||
"min_votes": default_config.reputation_min_votes,
|
||||
"low_score_threshold": default_config.reputation_low_score_threshold,
|
||||
"policy": default_config.reputation_policy,
|
||||
},
|
||||
"naming": {
|
||||
"title_format": default_config.naming_title_format,
|
||||
},
|
||||
},
|
||||
'quarantine_dir': default_config.quarantine_dir,
|
||||
'log_level': default_config.log_level,
|
||||
'categories': default_config.categories
|
||||
}
|
||||
|
||||
# Ensure parent directory exists
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write configuration file
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(yaml_content, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
|
||||
return default_config
|
||||
|
||||
|
||||
def validate_config(config: Config) -> list[str]:
|
||||
"""Validate configuration and return list of error messages.
|
||||
|
||||
Args:
|
||||
config: Configuration object to validate
|
||||
|
||||
Returns:
|
||||
List of error messages (empty if valid)
|
||||
"""
|
||||
"""Validate configuration and return list of error messages."""
|
||||
errors = []
|
||||
|
||||
# Validate library_root
|
||||
|
||||
if not isinstance(config.library_root, Path):
|
||||
errors.append("library_root must be a Path object")
|
||||
elif not str(config.library_root) or str(config.library_root) == ".":
|
||||
errors.append("library_root cannot be empty")
|
||||
|
||||
# Validate video_extensions
|
||||
|
||||
if not config.video_extensions:
|
||||
errors.append("video_extensions cannot be empty")
|
||||
elif not isinstance(config.video_extensions, list):
|
||||
@@ -184,31 +196,29 @@ def validate_config(config: Config) -> list[str]:
|
||||
if not isinstance(ext, str):
|
||||
errors.append(f"video_extensions must contain strings, found: {type(ext)}")
|
||||
break
|
||||
if not ext.startswith('.'):
|
||||
if not ext.startswith("."):
|
||||
errors.append(f"video extension must start with '.': {ext}")
|
||||
|
||||
# Validate templates
|
||||
|
||||
if not config.movie_template:
|
||||
errors.append("movie_template cannot be empty")
|
||||
elif not isinstance(config.movie_template, str):
|
||||
errors.append("movie_template must be a string")
|
||||
|
||||
|
||||
if not config.series_template:
|
||||
errors.append("series_template cannot be empty")
|
||||
elif not isinstance(config.series_template, str):
|
||||
errors.append("series_template must be a string")
|
||||
|
||||
|
||||
if not config.movie_filename_template:
|
||||
errors.append("movie_filename_template cannot be empty")
|
||||
elif not isinstance(config.movie_filename_template, str):
|
||||
errors.append("movie_filename_template must be a string")
|
||||
|
||||
|
||||
if not config.series_filename_template:
|
||||
errors.append("series_filename_template cannot be empty")
|
||||
elif not isinstance(config.series_filename_template, str):
|
||||
errors.append("series_filename_template must be a string")
|
||||
|
||||
# Validate log_level
|
||||
|
||||
valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
if not config.log_level:
|
||||
errors.append("log_level cannot be empty")
|
||||
@@ -216,28 +226,24 @@ def validate_config(config: Config) -> list[str]:
|
||||
errors.append("log_level must be a string")
|
||||
elif config.log_level.upper() not in valid_log_levels:
|
||||
errors.append(f"log_level must be one of {valid_log_levels}, got: {config.log_level}")
|
||||
|
||||
# Validate quarantine_dir
|
||||
|
||||
if not config.quarantine_dir:
|
||||
errors.append("quarantine_dir cannot be empty")
|
||||
elif not isinstance(config.quarantine_dir, str):
|
||||
errors.append("quarantine_dir must be a string")
|
||||
elif config.quarantine_dir.startswith('/') or config.quarantine_dir.startswith('\\'):
|
||||
elif config.quarantine_dir.startswith("/") or config.quarantine_dir.startswith("\\"):
|
||||
errors.append("quarantine_dir must be relative to category root, not absolute")
|
||||
|
||||
# Validate categories
|
||||
if not config.categories:
|
||||
errors.append("categories cannot be empty")
|
||||
elif not isinstance(config.categories, dict):
|
||||
errors.append("categories must be a dictionary")
|
||||
else:
|
||||
# Check required category keys exist
|
||||
required_categories = {"movie", "series", "anime"}
|
||||
missing = required_categories - set(config.categories.keys())
|
||||
if missing:
|
||||
errors.append(f"categories must include keys: {sorted(missing)}")
|
||||
|
||||
# Validate each category's directory list and check for duplicates
|
||||
seen_dirs = {}
|
||||
for category, dir_list in config.categories.items():
|
||||
if not isinstance(dir_list, list):
|
||||
@@ -257,7 +263,6 @@ def validate_config(config: Config) -> list[str]:
|
||||
errors.append(f"categories['{category}'] contains empty directory name")
|
||||
break
|
||||
|
||||
# Check for duplicates (case-insensitive)
|
||||
dir_lower = dir_name.lower()
|
||||
if dir_lower in seen_dirs:
|
||||
errors.append(
|
||||
@@ -267,4 +272,19 @@ def validate_config(config: Config) -> list[str]:
|
||||
else:
|
||||
seen_dirs[dir_lower] = category
|
||||
|
||||
if not isinstance(config.enrichment_cache_db, Path):
|
||||
errors.append("enrichment_cache_db must be a Path object")
|
||||
if not isinstance(config.enrichment_providers, list) or not config.enrichment_providers:
|
||||
errors.append("enrichment_providers must be a non-empty list")
|
||||
if config.enrichment_max_concurrency < 1:
|
||||
errors.append("enrichment_max_concurrency must be >= 1")
|
||||
if not (0.0 <= config.enrichment_min_match_score <= 1.0):
|
||||
errors.append("enrichment_min_match_score must be between 0.0 and 1.0")
|
||||
if config.enrichment_refresh_mode not in {"manual"}:
|
||||
errors.append("enrichment_refresh_mode must be 'manual'")
|
||||
if config.reputation_min_votes < 0:
|
||||
errors.append("reputation_min_votes must be >= 0")
|
||||
if not (0.0 <= config.reputation_low_score_threshold <= 10.0):
|
||||
errors.append("reputation_low_score_threshold must be between 0.0 and 10.0")
|
||||
|
||||
return errors
|
||||
|
||||
Reference in New Issue
Block a user