windy/journal_organizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
ed4fe731dfbc68f161328e1930f4fa95eeee82ea
journal_organizer/config_validation.py
T

1316 lines
55 KiB
Python
Raw Normal View History

Initial project setup: Obsidian intelligent journal organizer
2025-12-31 17:55:10 +08:00
"""
Enhanced configuration validation using Pydantic models
Provides comprehensive validation, environment variable expansion, and error handling
"""
import json
import os
import re
import logging
from pathlib import Path
from typing import Dict, Any, List, Optional, Union
try:
from .dependency_manager import get_dependency_manager
from .configuration_loader import ConfigurationLoader, ConfigurationError, EnvironmentVariableError
from .configuration_migrator import ConfigurationMigrator
from .error_handling import (
ClaudeConfigurationError, ClaudeAPIURLError, ClaudeModelValidationError,
ClaudeAPIKeyError, transform_pydantic_validation_error, get_error_handler
)
except ImportError:
from dependency_manager import get_dependency_manager
from configuration_loader import ConfigurationLoader, ConfigurationError, EnvironmentVariableError
from configuration_migrator import ConfigurationMigrator
try:
from error_handling import (
ClaudeConfigurationError, ClaudeAPIURLError, ClaudeModelValidationError,
ClaudeAPIKeyError, transform_pydantic_validation_error, get_error_handler
)
except ImportError:
# Fallback if error handling not available
ClaudeConfigurationError = Exception
ClaudeAPIURLError = Exception
ClaudeModelValidationError = Exception
ClaudeAPIKeyError = Exception
def transform_pydantic_validation_error(e, section=""): return e
def get_error_handler(): return None
# Try to import dependencies with graceful degradation
dependency_manager = get_dependency_manager()
yaml = dependency_manager.get_module('yaml')
# Try to import pydantic with graceful degradation
pydantic_module = dependency_manager.get_module('pydantic')
if pydantic_module is not None:
from pydantic import BaseModel, Field, validator, root_validator
from pydantic.error_wrappers import ValidationError
else:
# Create fallback classes if pydantic is not available
class BaseModel:
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
def Field(*args, **kwargs):
return None
def validator(*args, **kwargs):
def decorator(func):
return func
return decorator
def root_validator(*args, **kwargs):
def decorator(func):
return func
return decorator
class ValidationError(Exception):
def __init__(self, message):
super().__init__(message)
self.errors = lambda: [{'loc': ['unknown'], 'msg': message}]
class ObsidianRestAPIConfig(BaseModel):
"""Obsidian REST API configuration with validation"""
url: str = Field(..., description="REST API server URL")
api_key: str = Field(..., min_length=1, description="API key for authentication")
verify_ssl: bool = Field(default=False, description="Whether to verify SSL certificates")
@validator('url')
def validate_url(cls, v: str) -> str:
"""Validate URL format"""
if not v.startswith(('http://', 'https://')):
raise ValueError('URL must start with http:// or https://')
return v
@validator('api_key')
def validate_api_key(cls, v: str) -> str:
"""Validate API key format"""
if not v or not v.strip():
raise ValueError('API key cannot be empty')
# Basic format validation - should be alphanumeric with possible dashes/underscores
if not re.match(r'^[a-zA-Z0-9_-]+$', v.strip()):
raise ValueError('API key contains invalid characters')
return v.strip()
class ObsidianConfig(BaseModel):
"""Obsidian configuration with path validation"""
vault_path: str = Field(..., description="Path to Obsidian vault")
rest_api: ObsidianRestAPIConfig = Field(..., description="REST API configuration")
@validator('vault_path')
def validate_vault_path(cls, v: str) -> str:
"""Validate vault path exists and is accessible"""
if not v or not v.strip():
raise ValueError('Vault path cannot be empty')
# Expand environment variables and user home
expanded_path = os.path.expandvars(os.path.expanduser(v.strip()))
path = Path(expanded_path)
if not path.exists():
raise ValueError(f'Vault path does not exist: {expanded_path}')
if not path.is_dir():
raise ValueError(f'Vault path is not a directory: {expanded_path}')
# Check if it's readable
if not os.access(path, os.R_OK):
raise ValueError(f'Vault path is not readable: {expanded_path}')
return str(path.resolve())
class ClaudeAPIConfig(BaseModel):
"""Enhanced Claude API configuration with URL and model validation"""
api_key: str = Field(..., min_length=1, description="Claude API key")
api_url: Optional[str] = Field(
default="https://api.anthropic.com",
description="Claude API base URL"
)
model: str = Field(
default="claude-3-5-sonnet-20241022",
description="Claude model name"
)
max_tokens: int = Field(default=4096, ge=1, le=200000, description="Maximum tokens")
temperature: float = Field(default=0.7, ge=0.0, le=1.0, description="Temperature parameter")
@validator('api_url')
def validate_api_url(cls, v: Optional[str]) -> str:
"""Validate API URL format and accessibility"""
if v is None:
return "https://api.anthropic.com"
# Parse URL to validate format
from urllib.parse import urlparse
parsed = urlparse(v)
if not parsed.scheme or not parsed.netloc:
raise ValueError(f"Invalid URL format: {v}")
if parsed.scheme not in ['http', 'https']:
raise ValueError(f"URL must use http or https protocol: {v}")
# Remove trailing slash for consistency
return v.rstrip('/')
@validator('model')
def validate_model_name(cls, v: str) -> str:
"""Validate Claude model name and provide suggestions"""
valid_models = [
# Claude 3 series
"claude-3-opus-20240229",
"claude-3-sonnet-20240229",
"claude-3-haiku-20240307",
# Claude 3.5 series
"claude-3-5-sonnet-20241022",
"claude-3-5-haiku-20241022",
# Latest aliases
"claude-3-opus-latest",
"claude-3-sonnet-latest",
"claude-3-haiku-latest",
"claude-3-5-sonnet-latest",
"claude-3-5-haiku-latest"
]
if v not in valid_models:
# Check if it follows Claude naming pattern
claude_pattern = r'^claude-\d+(\.\d+)?-(opus|sonnet|haiku)(-\d{8}|-latest)?$'
if not re.match(claude_pattern, v, re.IGNORECASE):
suggestions = ", ".join(valid_models[:5])
raise ValueError(
f"Invalid model name: {v}. "
f"Valid models include: {suggestions}. "
f"Model names should follow pattern: claude-X-Y-YYYYMMDD or claude-X-Y-latest"
)
return v
@validator('api_key')
def validate_api_key(cls, v: str) -> str:
"""Validate Claude API key format"""
if not v or not v.strip():
raise ValueError('Claude API key cannot be empty')
# Claude API keys typically start with 'sk-ant-'
stripped = v.strip()
if not stripped.startswith('sk-ant-'):
raise ValueError('Claude API key should start with "sk-ant-"')
# Should be at least 50 characters long
if len(stripped) < 50:
raise ValueError('Claude API key appears to be too short')
return stripped
# Keep the old ClaudeConfig for backward compatibility
class ClaudeConfig(ClaudeAPIConfig):
"""Legacy Claude API configuration - use ClaudeAPIConfig for new implementations"""
pass
class JournalConfig(BaseModel):
"""Journal configuration with validation"""
daily_notes_folder: str = Field(default="Daily", description="Folder for daily notes")
date_format: str = Field(default="YYYY-MM-DD", description="Date format for files")
file_extension: str = Field(default=".md", description="File extension for notes")
@validator('daily_notes_folder')
def validate_folder(cls, v: str) -> str:
"""Validate folder name"""
if not v or not v.strip():
raise ValueError('Daily notes folder cannot be empty')
return v.strip()
@validator('file_extension')
def validate_extension(cls, v: str) -> str:
"""Validate file extension"""
if not v.startswith('.'):
raise ValueError('File extension must start with a dot')
return v
class OutputConfig(BaseModel):
"""Output configuration with folder validation"""
experiences_folder: str = Field(default="Knowledge/Experiences", description="Experiences folder")
lessons_folder: str = Field(default="Knowledge/Lessons", description="Lessons folder")
tasks_folder: str = Field(default="Tasks/Daily", description="Tasks folder")
problems_folder: str = Field(default="Knowledge/Problems", description="Problems folder")
achievements_folder: str = Field(default="Knowledge/Achievements", description="Achievements folder")
improvements_folder: str = Field(default="Knowledge/Improvements", description="Improvements folder")
@validator('*')
def validate_folder_paths(cls, v: str) -> str:
"""Validate all folder paths are non-empty"""
if not v or not v.strip():
raise ValueError('Folder path cannot be empty')
return v.strip()
class AnalysisConfig(BaseModel):
"""Analysis configuration with validation"""
categories: List[str] = Field(default_factory=list, description="Analysis categories")
extraction_rules: Dict[str, Dict[str, Any]] = Field(default_factory=dict, description="Extraction rules")
@validator('categories')
def validate_categories(cls, v: List[str]) -> List[str]:
"""Validate categories list"""
if not isinstance(v, list):
raise ValueError('Categories must be a list')
# Remove empty categories
valid_categories = [cat.strip() for cat in v if cat and cat.strip()]
return valid_categories
class LoggingConfig(BaseModel):
"""Logging configuration with validation"""
level: str = Field(default="INFO", description="Logging level")
file: str = Field(default="logs/journal_organizer.log", description="Log file path")
@validator('level')
def validate_level(cls, v: str) -> str:
"""Validate logging level"""
valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
if v.upper() not in valid_levels:
raise ValueError(f'Logging level must be one of: {", ".join(valid_levels)}')
return v.upper()
@validator('file')
def validate_file_path(cls, v: str) -> str:
"""Validate log file path"""
if not v or not v.strip():
raise ValueError('Log file path cannot be empty')
# Ensure parent directory exists or can be created
log_path = Path(v.strip())
try:
log_path.parent.mkdir(parents=True, exist_ok=True)
except (OSError, PermissionError) as e:
raise ValueError(f'Cannot create log directory: {e}')
return str(log_path)
class SystemConfig(BaseModel):
"""Complete system configuration with validation"""
obsidian: ObsidianConfig = Field(..., description="Obsidian configuration")
claude: ClaudeAPIConfig = Field(..., description="Claude API configuration")
journal: JournalConfig = Field(default_factory=JournalConfig, description="Journal configuration")
output: OutputConfig = Field(default_factory=OutputConfig, description="Output configuration")
analysis: AnalysisConfig = Field(default_factory=AnalysisConfig, description="Analysis configuration")
logging: LoggingConfig = Field(default_factory=LoggingConfig, description="Logging configuration")
class Config:
"""Pydantic configuration"""
extra = 'forbid' # Don't allow extra fields
validate_assignment = True # Validate on assignment
@root_validator
def validate_paths_relative_to_vault(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validate that output paths make sense relative to vault"""
obsidian_config = values.get('obsidian')
output_config = values.get('output')
if obsidian_config and output_config:
vault_path = Path(obsidian_config.vault_path)
# Check if output folders would be accessible within vault
for folder_name in ['experiences_folder', 'lessons_folder', 'tasks_folder',
'problems_folder', 'achievements_folder', 'improvements_folder']:
folder_path = getattr(output_config, folder_name)
if folder_path:
# Ensure the folder path doesn't try to escape the vault
full_path = vault_path / folder_path
try:
full_path.resolve().relative_to(vault_path.resolve())
except ValueError:
raise ValueError(f'Output folder {folder_name} ({folder_path}) would be outside vault')
return values
class ConfigurationValidator:
"""Configuration validation and loading utility"""
def __init__(self, logger: Optional[logging.Logger] = None):
self.env_var_pattern = re.compile(r'\$\{([^}]+)\}')
self.config_loader = ConfigurationLoader()
self.migrator = ConfigurationMigrator()
self.logger = logger or logging.getLogger(__name__)
self.error_handler = get_error_handler()
def expand_environment_variables(self, data: Any) -> Any:
"""
Recursively expand environment variables in configuration data
DEPRECATED: Use ConfigurationLoader.expand_environment_variables instead
This method is kept for backward compatibility
"""
return self.config_loader.expand_environment_variables(data) if isinstance(data, dict) else self.config_loader._expand_value(data)
def load_and_validate_config(self, config_file: Union[str, Path]) -> SystemConfig:
"""Load and validate configuration from file"""
self.logger.info(f"Loading configuration from: {config_file}")
try:
# Use the new ConfigurationLoader for loading and environment variable expansion
self.logger.debug("Expanding environment variables in configuration")
expanded_config = self.config_loader.load_config(config_file)
self.logger.debug("Environment variable expansion completed successfully")
except (ConfigurationError, EnvironmentVariableError) as e:
self.logger.error(f"Configuration loading failed: {str(e)}")
if self.error_handler:
error_response = self.error_handler.handle_configuration_error(e, "config_file")
self.logger.error(f"Configuration error details: {error_response}")
raise ValueError(str(e))
# Apply migration before validation to ensure backward compatibility
if self.migrator.check_migration_needed(expanded_config):
self.logger.info("Configuration migration needed, applying migration")
migration_preview = self.migrator.get_migration_preview(expanded_config)
for action in migration_preview:
self.logger.info(f"Migration action: {action}")
expanded_config = self.migrator.migrate_configuration(expanded_config)
self.logger.info("Configuration migration completed successfully")
else:
self.logger.debug("No configuration migration needed")
# Apply default values for missing sections and fields
self.logger.debug("Applying default values to configuration")
expanded_config = self._apply_default_values(expanded_config)
self.logger.debug("Default values applied successfully")
# Validate configuration
try:
self.logger.debug("Starting configuration validation")
if pydantic_module is None:
self.logger.warning("Pydantic not available, using fallback validation")
# Fallback validation without pydantic
config = self._validate_config_without_pydantic(expanded_config)
self.logger.info("Configuration validation completed (fallback mode)")
return config
config = SystemConfig(**expanded_config)
self.logger.info("Configuration validation completed successfully")
# Log configuration summary (without sensitive data)
self._log_configuration_summary(config)
return config
except ValidationError as e:
self.logger.error("Configuration validation failed")
# Transform pydantic errors into user-friendly Claude errors
claude_error = transform_pydantic_validation_error(e, "claude")
# Log detailed validation errors
self._log_validation_errors(e)
# Use error handler if available
if self.error_handler:
error_response = self.error_handler.handle_claude_configuration_error(
claude_error, operation="validate_config"
)
self.logger.error(f"Validation error details: {error_response}")
# Format validation errors nicely
error_messages = []
for error in e.errors():
field_path = ' -> '.join(str(loc) for loc in error['loc'])
error_messages.append(f"{field_path}: {error['msg']}")
formatted_error = f'Configuration validation failed:\n' + '\n'.join(error_messages)
self.logger.error(f"Formatted validation error: {formatted_error}")
raise ValueError(formatted_error)
def _log_configuration_summary(self, config: SystemConfig) -> None:
"""Log a summary of the loaded configuration (without sensitive data)"""
try:
summary = {
"obsidian": {
"vault_path": config.obsidian.vault_path,
"rest_api_url": config.obsidian.rest_api.url,
"verify_ssl": config.obsidian.rest_api.verify_ssl
},
"claude": {
"api_url": config.claude.api_url,
"model": config.claude.model,
"max_tokens": config.claude.max_tokens,
"temperature": config.claude.temperature,
"is_custom_endpoint": config.claude.api_url != "https://api.anthropic.com"
},
"journal": {
"daily_notes_folder": config.journal.daily_notes_folder,
"date_format": config.journal.date_format,
"file_extension": config.journal.file_extension
},
"logging": {
"level": config.logging.level,
"file": config.logging.file
}
}
self.logger.info(f"Configuration summary: {json.dumps(summary, indent=2)}")
# Log specific warnings for custom configurations
if config.claude.api_url != "https://api.anthropic.com":
self.logger.warning(f"Using custom Claude API endpoint: {config.claude.api_url}")
if not config.obsidian.rest_api.verify_ssl:
self.logger.warning("SSL verification disabled for Obsidian REST API")
except Exception as e:
self.logger.debug(f"Could not log configuration summary: {e}")
def _log_validation_errors(self, validation_error) -> None:
"""Log detailed validation errors for debugging"""
try:
errors = validation_error.errors()
self.logger.debug(f"Total validation errors: {len(errors)}")
for i, error in enumerate(errors):
field_path = ' -> '.join(str(loc) for loc in error.get('loc', []))
error_type = error.get('type', 'unknown')
error_msg = error.get('msg', 'Unknown error')
input_value = error.get('input', 'N/A')
# Sanitize input value for logging
if any(sensitive in field_path.lower() for sensitive in ['key', 'password', 'token']):
input_value = '[REDACTED]'
elif isinstance(input_value, str) and len(input_value) > 100:
input_value = input_value[:100] + '...'
self.logger.debug(
f"Validation error {i+1}: "
f"field='{field_path}', type='{error_type}', "
f"message='{error_msg}', input='{input_value}'"
)
except Exception as e:
self.logger.debug(f"Could not log validation error details: {e}")
def _apply_default_values(self, config_dict: Dict[str, Any]) -> Dict[str, Any]:
"""
Apply default values for missing configuration sections and fields
Note: Most default value application is now handled by ConfigurationMigrator.
This method provides additional validation and ensures consistency.
Args:
config_dict: Configuration dictionary loaded from file
Returns:
Configuration dictionary with default values applied
"""
# Create a copy to avoid modifying the original
config_with_defaults = config_dict.copy()
# The migrator should have already applied most defaults,
# but we'll ensure critical defaults are present as a safety measure
# Ensure Claude API defaults are present (critical for functionality)
if 'claude' not in config_with_defaults:
config_with_defaults['claude'] = {}
claude_config = config_with_defaults['claude']
# These are critical defaults that must be present
if 'api_url' not in claude_config:
claude_config['api_url'] = "https://api.anthropic.com"
if 'model' not in claude_config:
claude_config['model'] = "claude-3-5-sonnet-20241022"
if 'max_tokens' not in claude_config:
claude_config['max_tokens'] = 4096
if 'temperature' not in claude_config:
claude_config['temperature'] = 0.7
# Ensure other critical sections exist (migrator should have handled this)
self._ensure_section_exists(config_with_defaults, 'journal', {
'daily_notes_folder': "Daily",
'date_format': "YYYY-MM-DD",
'file_extension': ".md"
})
self._ensure_section_exists(config_with_defaults, 'output', {
'experiences_folder': "Knowledge/Experiences",
'lessons_folder': "Knowledge/Lessons",
'tasks_folder': "Tasks/Daily",
'problems_folder': "Knowledge/Problems",
'achievements_folder': "Knowledge/Achievements",
'improvements_folder': "Knowledge/Improvements"
})
self._ensure_section_exists(config_with_defaults, 'analysis', {
'categories': [],
'extraction_rules': {}
})
self._ensure_section_exists(config_with_defaults, 'logging', {
'level': "INFO",
'file': "logs/journal_organizer.log"
})
return config_with_defaults
def _ensure_section_exists(self, config_dict: Dict[str, Any], section_name: str, defaults: Dict[str, Any]) -> None:
"""Ensure a configuration section exists with default values"""
if section_name not in config_dict:
config_dict[section_name] = defaults.copy()
else:
section_config = config_dict[section_name]
for field, default_value in defaults.items():
if field not in section_config:
section_config[field] = default_value
def _validate_config_without_pydantic(self, config_data: Dict[str, Any]) -> 'SystemConfig':
"""Fallback validation when pydantic is not available"""
# Basic validation without pydantic
required_sections = ['obsidian', 'claude']
for section in required_sections:
if section not in config_data:
raise ValueError(f'Missing required configuration section: {section}')
# Basic obsidian validation
obsidian = config_data.get('obsidian', {})
if not obsidian.get('vault_path'):
raise ValueError('obsidian.vault_path is required')
if not obsidian.get('rest_api', {}).get('api_key'):
raise ValueError('obsidian.rest_api.api_key is required')
# Basic claude validation - now with defaults applied
claude = config_data.get('claude', {})
if not claude.get('api_key'):
raise ValueError('claude.api_key is required')
# Validate that defaults were applied correctly
if not claude.get('api_url'):
raise ValueError('claude.api_url should have default value')
if not claude.get('model'):
raise ValueError('claude.model should have default value')
# Create a simple config object (fallback)
class SimpleConfig:
def __init__(self, data):
for key, value in data.items():
if isinstance(value, dict):
setattr(self, key, SimpleConfig(value))
else:
setattr(self, key, value)
return SimpleConfig(config_data)
def validate_api_keys(self, config: SystemConfig) -> List[str]:
"""Validate API keys and return setup instructions if needed"""
self.logger.debug("Starting API key validation")
issues = []
# Check Claude API key
self.logger.debug("Validating Claude API key")
claude_key = config.claude.api_key
if not claude_key or claude_key.startswith('${'):
self.logger.warning("Claude API key is not set or uses environment variable placeholder")
issues.append(self._get_claude_api_key_setup_instructions())
else:
# Validate Claude API key format
claude_issues = self._validate_claude_api_key_format(claude_key)
if claude_issues:
self.logger.warning(f"Claude API key format issues found: {len(claude_issues)} issues")
for issue in claude_issues:
self.logger.debug(f"Claude API key issue: {issue}")
issues.extend(claude_issues)
else:
self.logger.debug("Claude API key format validation passed")
# Check Obsidian API key
self.logger.debug("Validating Obsidian API key")
obsidian_key = config.obsidian.rest_api.api_key
if not obsidian_key or obsidian_key.startswith('${'):
self.logger.warning("Obsidian API key is not set or uses environment variable placeholder")
issues.append(self._get_obsidian_api_key_setup_instructions())
else:
# Validate Obsidian API key format
obsidian_issues = self._validate_obsidian_api_key_format(obsidian_key)
if obsidian_issues:
self.logger.warning(f"Obsidian API key format issues found: {len(obsidian_issues)} issues")
for issue in obsidian_issues:
self.logger.debug(f"Obsidian API key issue: {issue}")
issues.extend(obsidian_issues)
else:
self.logger.debug("Obsidian API key format validation passed")
if issues:
self.logger.warning(f"API key validation completed with {len(issues)} issues")
else:
self.logger.info("API key validation completed successfully")
return issues
def _get_claude_api_key_setup_instructions(self) -> str:
"""Get detailed setup instructions for Claude API key"""
return """Claude API Key Setup Required:
1. Visit https://console.anthropic.com/
2. Sign in or create an account
3. Navigate to 'API Keys' section
4. Click 'Create Key' and give it a name
5. Copy the generated key (starts with 'sk-ant-')
6. Set the environment variable:
export ANTHROPIC_API_KEY="sk-ant-your-key-here"
Or add it to your shell profile (~/.bashrc, ~/.zshrc):
echo 'export ANTHROPIC_API_KEY="sk-ant-your-key-here"' >> ~/.zshrc
7. Restart your terminal or run: source ~/.zshrc
Note: Keep your API key secure and never commit it to version control."""
def _get_obsidian_api_key_setup_instructions(self) -> str:
"""Get detailed setup instructions for Obsidian API key"""
return """Obsidian Local REST API Setup Required:
1. Install the 'Local REST API' plugin in Obsidian:
- Open Obsidian
- Go to Settings → Community Plugins
- Browse and search for 'Local REST API'
- Install and enable the plugin
2. Configure the plugin:
- Go to Settings → Local REST API
- Enable the API server
- Set a secure API key (recommended: generate a random string)
- Note the server URL (usually https://localhost:27123)
3. Update your configuration:
- Set the API key in your config.yaml file
- Or use environment variable: export OBSIDIAN_API_KEY="your-key-here"
4. Security note:
- The plugin uses HTTPS with a self-signed certificate
- Set verify_ssl: false in your config for local development"""
def _validate_claude_api_key_format(self, api_key: str) -> List[str]:
"""Validate Claude API key format and provide specific feedback"""
issues = []
if not api_key.startswith('sk-ant-'):
issues.append(
"Claude API key format issue: Key should start with 'sk-ant-'\n"
"Please verify you copied the complete key from https://console.anthropic.com/"
)
if len(api_key) < 50:
issues.append(
"Claude API key appears too short. Please verify you copied the complete key.\n"
"Claude API keys are typically 100+ characters long."
)
# Check for common copy-paste issues
if ' ' in api_key:
issues.append(
"Claude API key contains spaces. Please remove any whitespace.\n"
"API keys should be a single continuous string."
)
if api_key.endswith('...') or '...' in api_key:
issues.append(
"Claude API key appears to be truncated (contains '...').\n"
"Please copy the complete key from the Anthropic console."
)
return issues
def _validate_obsidian_api_key_format(self, api_key: str) -> List[str]:
"""Validate Obsidian API key format and provide specific feedback"""
issues = []
# Obsidian API keys are typically user-generated, so less strict validation
if len(api_key) < 8:
issues.append(
"Obsidian API key appears very short. For security, consider using a longer key.\n"
"Recommended: Generate a random string of at least 16 characters."
)
# Check for obviously insecure keys
insecure_keys = ['password', '123456', 'admin', 'test', 'key', 'secret']
if api_key.lower() in insecure_keys:
issues.append(
f"Obsidian API key '{api_key}' is not secure.\n"
"Please generate a random, unique API key for better security."
)
# Check for common copy-paste issues
if ' ' in api_key:
issues.append(
"Obsidian API key contains spaces. Please remove any whitespace.\n"
"API keys should be a single continuous string."
)
return issues
async def test_api_key_connectivity(self, config: SystemConfig) -> Dict[str, Any]:
"""Test API key connectivity (optional, for advanced validation)"""
self.logger.debug("Starting API key connectivity testing")
results = {
'claude': {'status': 'unknown', 'message': ''},
'obsidian': {'status': 'unknown', 'message': ''}
}
# Note: This is a placeholder for actual connectivity testing
# In a real implementation, you might want to make test API calls
# but that requires additional dependencies and network access
# For now, just validate the format
self.logger.debug("Testing Claude API key format")
claude_issues = self._validate_claude_api_key_format(config.claude.api_key)
if not claude_issues:
results['claude'] = {'status': 'format_valid', 'message': 'API key format appears valid'}
self.logger.debug("Claude API key format validation passed")
else:
results['claude'] = {'status': 'format_invalid', 'message': '; '.join(claude_issues)}
self.logger.warning(f"Claude API key format validation failed: {results['claude']['message']}")
self.logger.debug("Testing Obsidian API key format")
obsidian_issues = self._validate_obsidian_api_key_format(config.obsidian.rest_api.api_key)
if not obsidian_issues:
results['obsidian'] = {'status': 'format_valid', 'message': 'API key format appears valid'}
self.logger.debug("Obsidian API key format validation passed")
else:
results['obsidian'] = {'status': 'format_invalid', 'message': '; '.join(obsidian_issues)}
self.logger.warning(f"Obsidian API key format validation failed: {results['obsidian']['message']}")
self.logger.info(f"API key connectivity testing completed: Claude={results['claude']['status']}, Obsidian={results['obsidian']['status']}")
return results
def test_environment_variables(self, config_data: Dict[str, Any]) -> List[str]:
"""Test environment variable expansion and return any issues"""
self.logger.debug("Testing environment variable expansion")
try:
issues = self.config_loader.validate_environment_variables(config_data)
if issues:
self.logger.warning(f"Environment variable validation found {len(issues)} issues")
for issue in issues:
self.logger.debug(f"Environment variable issue: {issue}")
else:
self.logger.debug("Environment variable validation passed")
return issues
except Exception as e:
self.logger.error(f"Environment variable testing failed: {e}")
return [f"Environment variable testing failed: {str(e)}"]
def get_environment_variable_help(self) -> str:
"""Get help text for setting up environment variables"""
return """
Environment Variable Setup:
1. Claude API Key:
export ANTHROPIC_API_KEY="sk-ant-your-key-here"
Get your key from: https://console.anthropic.com/
2. For other API keys, set them as:
export YOUR_API_KEY="your-key-value"
3. You can also use default values in your config:
api_key: "${API_KEY:-default-value}"
4. Or make variables required with custom error messages:
api_key: "${API_KEY:?Please set your API key}"
5. Check current environment variables:
env | grep API_KEY
"""
def validate_file_paths(self, config: SystemConfig) -> List[str]:
"""Validate file paths and return issues if any"""
self.logger.debug("Starting file path validation")
issues = []
# Validate vault path (already validated by pydantic, but let's check write access)
self.logger.debug(f"Validating vault path: {config.obsidian.vault_path}")
vault_issues = self._validate_vault_path(config.obsidian.vault_path)
if vault_issues:
self.logger.warning(f"Vault path validation issues: {len(vault_issues)} issues")
for issue in vault_issues:
self.logger.debug(f"Vault path issue: {issue}")
else:
self.logger.debug("Vault path validation passed")
issues.extend(vault_issues)
if not vault_issues: # Only check other paths if vault path is valid
vault_path = Path(config.obsidian.vault_path)
# Validate daily notes folder
self.logger.debug(f"Validating daily notes folder: {config.journal.daily_notes_folder}")
daily_notes_issues = self._validate_daily_notes_folder(vault_path, config.journal.daily_notes_folder)
if daily_notes_issues:
self.logger.info(f"Daily notes folder validation: {len(daily_notes_issues)} messages")
for issue in daily_notes_issues:
self.logger.debug(f"Daily notes folder: {issue}")
else:
self.logger.debug("Daily notes folder validation passed")
issues.extend(daily_notes_issues)
# Validate output folders
self.logger.debug("Validating output folders")
output_issues = self._validate_output_folders(vault_path, config.output)
if output_issues:
self.logger.warning(f"Output folders validation issues: {len(output_issues)} issues")
for issue in output_issues:
self.logger.debug(f"Output folder issue: {issue}")
else:
self.logger.debug("Output folders validation passed")
issues.extend(output_issues)
# Validate log file path
self.logger.debug(f"Validating log file path: {config.logging.file}")
log_issues = self._validate_log_file_path(config.logging.file)
if log_issues:
self.logger.warning(f"Log file path validation issues: {len(log_issues)} issues")
for issue in log_issues:
self.logger.debug(f"Log file path issue: {issue}")
else:
self.logger.debug("Log file path validation passed")
issues.extend(log_issues)
if issues:
self.logger.warning(f"File path validation completed with {len(issues)} issues")
else:
self.logger.info("File path validation completed successfully")
return issues
def _validate_vault_path(self, vault_path: str) -> List[str]:
"""Validate Obsidian vault path with detailed checks"""
issues = []
path = Path(vault_path)
# Check if path exists (already validated by pydantic)
if not path.exists():
issues.append(f"Vault path does not exist: {vault_path}")
return issues
# Check if it's a directory
if not path.is_dir():
issues.append(f"Vault path is not a directory: {vault_path}")
return issues
# Check read access
if not os.access(path, os.R_OK):
issues.append(f"Vault path is not readable: {vault_path}")
# Check write access
if not os.access(path, os.W_OK):
issues.append(
f"Vault path is not writable: {vault_path}\n"
"The application needs write access to create and update notes.\n"
"Please check file permissions or run with appropriate privileges."
)
# Check if it looks like an Obsidian vault
obsidian_config_path = path / ".obsidian"
if not obsidian_config_path.exists():
issues.append(
f"Path does not appear to be an Obsidian vault: {vault_path}\n"
"Expected to find .obsidian folder. Please verify the vault path is correct."
)
# Check available disk space (basic check)
try:
stat = os.statvfs(path)
free_space_mb = (stat.f_bavail * stat.f_frsize) / (1024 * 1024)
if free_space_mb < 100: # Less than 100MB
issues.append(
f"Low disk space in vault directory: {free_space_mb:.1f}MB available\n"
"Consider freeing up disk space to avoid issues when creating notes."
)
except (OSError, AttributeError):
# statvfs not available on all systems (e.g., Windows)
pass
return issues
def _validate_daily_notes_folder(self, vault_path: Path, daily_notes_folder: str) -> List[str]:
"""Validate daily notes folder"""
issues = []
daily_notes_path = vault_path / daily_notes_folder
if not daily_notes_path.exists():
try:
daily_notes_path.mkdir(parents=True, exist_ok=True)
issues.append(f"Created daily notes folder: {daily_notes_path}")
except (OSError, PermissionError) as e:
issues.append(
f"Cannot create daily notes folder: {daily_notes_path}\n"
f"Error: {e}\n"
"Please create this folder manually or check permissions."
)
elif not daily_notes_path.is_dir():
issues.append(
f"Daily notes path exists but is not a directory: {daily_notes_path}\n"
"Please remove this file or choose a different folder name."
)
elif not os.access(daily_notes_path, os.W_OK):
issues.append(
f"Daily notes folder is not writable: {daily_notes_path}\n"
"Please check folder permissions."
)
return issues
def _validate_output_folders(self, vault_path: Path, output_config: OutputConfig) -> List[str]:
"""Validate all output folders"""
issues = []
output_folders = {
'experiences': output_config.experiences_folder,
'lessons': output_config.lessons_folder,
'tasks': output_config.tasks_folder,
'problems': output_config.problems_folder,
'achievements': output_config.achievements_folder,
'improvements': output_config.improvements_folder
}
for folder_type, folder_path in output_folders.items():
full_path = vault_path / folder_path
# Check if path would be outside vault (security check)
try:
full_path.resolve().relative_to(vault_path.resolve())
except ValueError:
issues.append(
f"Output folder '{folder_type}' ({folder_path}) would be outside vault\n"
"This is not allowed for security reasons. Please use a relative path within the vault."
)
continue
if not full_path.exists():
try:
full_path.mkdir(parents=True, exist_ok=True)
# This is informational, not an error
pass
except (OSError, PermissionError) as e:
issues.append(
f"Cannot create {folder_type} folder: {full_path}\n"
f"Error: {e}\n"
"Please create this folder manually or check permissions."
)
elif not full_path.is_dir():
issues.append(
f"Output path for {folder_type} exists but is not a directory: {full_path}\n"
"Please remove this file or choose a different folder name."
)
elif not os.access(full_path, os.W_OK):
issues.append(
f"Output folder for {folder_type} is not writable: {full_path}\n"
"Please check folder permissions."
)
return issues
def _validate_log_file_path(self, log_file_path: str) -> List[str]:
"""Validate log file path"""
issues = []
log_path = Path(log_file_path)
# Check if parent directory exists or can be created
try:
log_path.parent.mkdir(parents=True, exist_ok=True)
except (OSError, PermissionError) as e:
issues.append(
f"Cannot create log directory: {log_path.parent}\n"
f"Error: {e}\n"
"Please create the directory manually or choose a different log file path."
)
return issues
# Check if log file is writable (if it exists)
if log_path.exists():
if not os.access(log_path, os.W_OK):
issues.append(
f"Log file is not writable: {log_path}\n"
"Please check file permissions or choose a different log file path."
)
else:
# Check if we can create the log file
try:
log_path.touch()
log_path.unlink() # Remove the test file
except (OSError, PermissionError) as e:
issues.append(
f"Cannot create log file: {log_path}\n"
f"Error: {e}\n"
"Please check directory permissions or choose a different log file path."
)
return issues
def get_path_setup_help(self) -> str:
"""Get help text for setting up file paths"""
return """
File Path Setup Guide:
1. Obsidian Vault Path:
- Must point to an existing Obsidian vault directory
- Should contain a .obsidian folder
- Must have read and write permissions
- Example: "/Users/username/Documents/MyVault"
2. Daily Notes Folder:
- Relative path within your vault
- Will be created if it doesn't exist
- Example: "Daily" or "Journal/Daily"
3. Output Folders:
- All paths are relative to your vault
- Will be created automatically if they don't exist
- Must be within the vault for security
- Examples: "Knowledge/Experiences", "Tasks/Daily"
4. Log File:
- Can be absolute or relative path
- Parent directory will be created if needed
- Must have write permissions
- Example: "logs/journal_organizer.log"
5. Common Issues:
- Permission denied: Check folder/file permissions
- Path not found: Verify the path exists and is correct
- Outside vault: Use relative paths within your vault
- Disk space: Ensure sufficient free space
6. Fixing Permissions (Unix/Linux/macOS):
chmod 755 /path/to/vault # For directories
chmod 644 /path/to/file # For files
"""
def check_migration_needed(self, config_dict: Dict[str, Any]) -> bool:
"""
Check if configuration needs migration
Args:
config_dict: Configuration dictionary to check
Returns:
True if migration is needed, False otherwise
"""
return self.migrator.check_migration_needed(config_dict)
def get_migration_preview(self, config_dict: Dict[str, Any]) -> List[str]:
"""
Get a preview of what migration actions would be taken
Args:
config_dict: Configuration dictionary to analyze
Returns:
List of migration actions that would be taken
"""
return self.migrator.get_migration_preview(config_dict)
def get_supported_model_names(self) -> List[str]:
"""
Get list of all supported model names (both current and legacy)
Returns:
List of supported model names
"""
return self.migrator.get_supported_model_names()
def validate_configuration_comprehensive(self, config: SystemConfig) -> Dict[str, Any]:
"""
Perform comprehensive configuration validation with detailed logging and reporting
Args:
config: SystemConfig instance to validate
Returns:
Dictionary with comprehensive validation results
"""
self.logger.info("Starting comprehensive configuration validation")
validation_results = {
'overall_status': 'unknown',
'validation_timestamp': None,
'api_keys': {'status': 'unknown', 'issues': []},
'file_paths': {'status': 'unknown', 'issues': []},
'environment_variables': {'status': 'unknown', 'issues': []},
'configuration_summary': {},
'warnings': [],
'errors': [],
'recommendations': []
}
try:
from datetime import datetime
validation_results['validation_timestamp'] = datetime.now().isoformat()
# 1. Validate API keys
self.logger.info("Phase 1: API key validation")
api_key_issues = self.validate_api_keys(config)
validation_results['api_keys'] = {
'status': 'failed' if api_key_issues else 'passed',
'issues': api_key_issues
}
if api_key_issues:
validation_results['errors'].extend(api_key_issues)
self.logger.warning(f"API key validation failed with {len(api_key_issues)} issues")
else:
self.logger.info("API key validation passed")
# 2. Validate file paths
self.logger.info("Phase 2: File path validation")
file_path_issues = self.validate_file_paths(config)
validation_results['file_paths'] = {
'status': 'failed' if file_path_issues else 'passed',
'issues': file_path_issues
}
if file_path_issues:
validation_results['errors'].extend(file_path_issues)
self.logger.warning(f"File path validation failed with {len(file_path_issues)} issues")
else:
self.logger.info("File path validation passed")
# 3. Test environment variables (if we have the raw config data)
# Note: This would require the original config dict, which we don't have here
# So we'll mark it as skipped for now
validation_results['environment_variables'] = {
'status': 'skipped',
'issues': ['Environment variable testing requires raw configuration data']
}
self.logger.debug("Environment variable validation skipped (requires raw config data)")
# 4. Generate configuration summary
try:
validation_results['configuration_summary'] = {
'claude_api_url': config.claude.api_url,
'claude_model': config.claude.model,
'is_custom_claude_endpoint': config.claude.api_url != "https://api.anthropic.com",
'obsidian_vault_path': config.obsidian.vault_path,
'obsidian_api_url': config.obsidian.rest_api.url,
'ssl_verification_disabled': not config.obsidian.rest_api.verify_ssl,
'log_level': config.logging.level,
'daily_notes_folder': config.journal.daily_notes_folder
}
self.logger.debug("Configuration summary generated")
except Exception as e:
self.logger.warning(f"Could not generate configuration summary: {e}")
# 5. Generate warnings and recommendations
if config.claude.api_url != "https://api.anthropic.com":
warning = f"Using custom Claude API endpoint: {config.claude.api_url}"
validation_results['warnings'].append(warning)
self.logger.warning(warning)
if not config.obsidian.rest_api.verify_ssl:
warning = "SSL verification is disabled for Obsidian REST API"
validation_results['warnings'].append(warning)
self.logger.warning(warning)
if config.logging.level == "DEBUG":
recommendation = "Debug logging is enabled. Consider using INFO level for production."
validation_results['recommendations'].append(recommendation)
self.logger.info(recommendation)
# 6. Determine overall status
has_errors = bool(validation_results['errors'])
has_warnings = bool(validation_results['warnings'])
if has_errors:
validation_results['overall_status'] = 'failed'
self.logger.error(f"Configuration validation failed with {len(validation_results['errors'])} errors")
elif has_warnings:
validation_results['overall_status'] = 'passed_with_warnings'
self.logger.warning(f"Configuration validation passed with {len(validation_results['warnings'])} warnings")
else:
validation_results['overall_status'] = 'passed'
self.logger.info("Configuration validation passed successfully")
return validation_results
except Exception as e:
error_msg = f"Comprehensive validation failed: {str(e)}"
self.logger.error(error_msg)
validation_results['overall_status'] = 'error'
validation_results['errors'].append(error_msg)
return validation_results
def create_example_config() -> Dict[str, Any]:
"""Create an example configuration dictionary"""
return {
"obsidian": {
"vault_path": "/path/to/your/obsidian/vault",
"rest_api": {
"url": "https://localhost:27123",
"api_key": "your-api-key-here",
"verify_ssl": False
}
},
"claude": {
"api_key": "${ANTHROPIC_API_KEY}",
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 4096,
"temperature": 0.7
},
"journal": {
"daily_notes_folder": "Daily",
"date_format": "YYYY-MM-DD",
"file_extension": ".md"
},
"output": {
"experiences_folder": "Knowledge/Experiences",
"lessons_folder": "Knowledge/Lessons",
"tasks_folder": "Tasks/Daily",
"problems_folder": "Knowledge/Problems",
"achievements_folder": "Knowledge/Achievements",
"improvements_folder": "Knowledge/Improvements"
},
"analysis": {
"categories": [
"技术学习",
"项目管理",
"个人成长",
"团队协作",
"问题解决"
],
"extraction_rules": {
"experiences": {
"enabled": True,
"description": "提取日记中的重要经验和见解"
},
"lessons": {
"enabled": True,
"description": "提取学到的知识点和最佳实践"
}
}
},
"logging": {
"level": "INFO",
"file": "logs/journal_organizer.log"
}
}
Reference in New Issue Copy Permalink