484 lines
18 KiB
Python
484 lines
18 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
Application startup and basic functionality validation script.
|
||
|
|
Tests both v1.0 CLI and v2.0 conversational interfaces.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
import traceback
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Dict, Any, Optional, List, Tuple
|
||
|
|
|
||
|
|
# Add the current directory to Python path for imports
|
||
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
||
|
|
|
||
|
|
# Configure logging for validation
|
||
|
|
logging.basicConfig(
|
||
|
|
level=logging.INFO,
|
||
|
|
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||
|
|
)
|
||
|
|
logger = logging.getLogger("StartupValidation")
|
||
|
|
|
||
|
|
|
||
|
|
class ValidationResult:
|
||
|
|
"""Container for validation test results"""
|
||
|
|
|
||
|
|
def __init__(self, test_name: str):
|
||
|
|
self.test_name = test_name
|
||
|
|
self.success = False
|
||
|
|
self.message = ""
|
||
|
|
self.details: Dict[str, Any] = {}
|
||
|
|
self.error: Optional[Exception] = None
|
||
|
|
|
||
|
|
def set_success(self, message: str, details: Optional[Dict[str, Any]] = None):
|
||
|
|
self.success = True
|
||
|
|
self.message = message
|
||
|
|
self.details = details or {}
|
||
|
|
|
||
|
|
def set_failure(self, message: str, error: Optional[Exception] = None, details: Optional[Dict[str, Any]] = None):
|
||
|
|
self.success = False
|
||
|
|
self.message = message
|
||
|
|
self.error = error
|
||
|
|
self.details = details or {}
|
||
|
|
|
||
|
|
def __str__(self) -> str:
|
||
|
|
status = "✅ PASS" if self.success else "❌ FAIL"
|
||
|
|
return f"{status} {self.test_name}: {self.message}"
|
||
|
|
|
||
|
|
|
||
|
|
class StartupValidator:
|
||
|
|
"""Validates application startup and basic functionality"""
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.results: List[ValidationResult] = []
|
||
|
|
self.temp_config_path: Optional[Path] = None
|
||
|
|
|
||
|
|
def create_test_config(self) -> Path:
|
||
|
|
"""Create a temporary test configuration file"""
|
||
|
|
config_content = {
|
||
|
|
'obsidian': {
|
||
|
|
'vault_path': '/tmp/test_vault',
|
||
|
|
'rest_api': {
|
||
|
|
'url': 'https://localhost:27123',
|
||
|
|
'api_key': 'test-api-key',
|
||
|
|
'verify_ssl': False
|
||
|
|
}
|
||
|
|
},
|
||
|
|
'claude': {
|
||
|
|
'api_key': '${ANTHROPIC_API_KEY}',
|
||
|
|
'model': 'claude-3-5-sonnet-20241022',
|
||
|
|
'max_tokens': 4096,
|
||
|
|
'temperature': 0.7
|
||
|
|
},
|
||
|
|
'journal': {
|
||
|
|
'daily_notes_folder': 'Daily',
|
||
|
|
'date_format': 'YYYY-MM-DD',
|
||
|
|
'file_extension': '.md'
|
||
|
|
},
|
||
|
|
'output': {
|
||
|
|
'experiences_folder': 'Knowledge/Experiences',
|
||
|
|
'lessons_folder': 'Knowledge/Lessons',
|
||
|
|
'tasks_folder': 'Tasks/Daily',
|
||
|
|
'problems_folder': 'Knowledge/Problems',
|
||
|
|
'achievements_folder': 'Knowledge/Achievements',
|
||
|
|
'improvements_folder': 'Knowledge/Improvements'
|
||
|
|
},
|
||
|
|
'logging': {
|
||
|
|
'level': 'INFO',
|
||
|
|
'file': 'logs/journal_organizer.log'
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
# Create temporary config file
|
||
|
|
temp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False)
|
||
|
|
json.dump(config_content, temp_file, indent=2)
|
||
|
|
temp_file.close()
|
||
|
|
|
||
|
|
self.temp_config_path = Path(temp_file.name)
|
||
|
|
return self.temp_config_path
|
||
|
|
|
||
|
|
def cleanup_test_config(self):
|
||
|
|
"""Clean up temporary test configuration"""
|
||
|
|
if self.temp_config_path and self.temp_config_path.exists():
|
||
|
|
self.temp_config_path.unlink()
|
||
|
|
|
||
|
|
async def test_import_main_modules(self) -> ValidationResult:
|
||
|
|
"""Test importing main application modules"""
|
||
|
|
result = ValidationResult("Import Main Modules")
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Test importing core modules
|
||
|
|
modules_tested = []
|
||
|
|
|
||
|
|
# Test core modules
|
||
|
|
try:
|
||
|
|
import main
|
||
|
|
modules_tested.append("main")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import main: {e}")
|
||
|
|
|
||
|
|
try:
|
||
|
|
import chat_main
|
||
|
|
modules_tested.append("chat_main")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import chat_main: {e}")
|
||
|
|
|
||
|
|
try:
|
||
|
|
import agent_core
|
||
|
|
modules_tested.append("agent_core")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import agent_core: {e}")
|
||
|
|
|
||
|
|
try:
|
||
|
|
import config
|
||
|
|
modules_tested.append("config")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import config: {e}")
|
||
|
|
|
||
|
|
# Test command modules
|
||
|
|
try:
|
||
|
|
from commands import organize_command
|
||
|
|
modules_tested.append("commands.organize_command")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import commands.organize_command: {e}")
|
||
|
|
|
||
|
|
# Test skill modules
|
||
|
|
try:
|
||
|
|
from skills import obsidian_skill
|
||
|
|
modules_tested.append("skills.obsidian_skill")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import skills.obsidian_skill: {e}")
|
||
|
|
|
||
|
|
try:
|
||
|
|
from skills import claude_skill
|
||
|
|
modules_tested.append("skills.claude_skill")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import skills.claude_skill: {e}")
|
||
|
|
|
||
|
|
# Test conversation modules
|
||
|
|
try:
|
||
|
|
from conversation import conversational_agent
|
||
|
|
modules_tested.append("conversation.conversational_agent")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import conversation.conversational_agent: {e}")
|
||
|
|
|
||
|
|
try:
|
||
|
|
from conversation import conversation_state
|
||
|
|
modules_tested.append("conversation.conversation_state")
|
||
|
|
except ImportError as e:
|
||
|
|
logger.warning(f"Could not import conversation.conversation_state: {e}")
|
||
|
|
|
||
|
|
if len(modules_tested) >= 4: # At least core modules should work
|
||
|
|
result.set_success(f"Successfully imported {len(modules_tested)} modules", {
|
||
|
|
"modules_tested": modules_tested,
|
||
|
|
"total_attempted": 9
|
||
|
|
})
|
||
|
|
else:
|
||
|
|
result.set_failure(f"Only imported {len(modules_tested)} out of 9 modules", details={
|
||
|
|
"modules_tested": modules_tested
|
||
|
|
})
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
result.set_failure(f"Failed to import modules: {str(e)}", e)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
async def test_agent_initialization(self) -> ValidationResult:
|
||
|
|
"""Test basic agent initialization"""
|
||
|
|
result = ValidationResult("Agent Initialization")
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Try to import and test agent initialization
|
||
|
|
try:
|
||
|
|
from main import JournalOrganizerAgent
|
||
|
|
except ImportError:
|
||
|
|
# If relative import fails, skip this test
|
||
|
|
result.set_failure("Could not import JournalOrganizerAgent - likely due to package structure")
|
||
|
|
return result
|
||
|
|
|
||
|
|
# Create test config
|
||
|
|
config_path = self.create_test_config()
|
||
|
|
|
||
|
|
# Initialize agent
|
||
|
|
agent = JournalOrganizerAgent(str(config_path))
|
||
|
|
|
||
|
|
# Test basic properties
|
||
|
|
assert hasattr(agent, 'config'), "Agent should have config attribute"
|
||
|
|
assert hasattr(agent, 'agent'), "Agent should have agent attribute"
|
||
|
|
assert hasattr(agent, 'logger'), "Agent should have logger attribute"
|
||
|
|
|
||
|
|
# Test command listing
|
||
|
|
commands = agent.list_commands()
|
||
|
|
assert isinstance(commands, list), "Commands should be a list"
|
||
|
|
assert len(commands) > 0, "Should have at least one command"
|
||
|
|
|
||
|
|
result.set_success("Agent initialized successfully", {
|
||
|
|
"available_commands": commands,
|
||
|
|
"config_loaded": bool(agent.config)
|
||
|
|
})
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
result.set_failure(f"Agent initialization failed: {str(e)}", e)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
async def test_configuration_loading(self) -> ValidationResult:
|
||
|
|
"""Test configuration loading with various scenarios"""
|
||
|
|
result = ValidationResult("Configuration Loading")
|
||
|
|
|
||
|
|
try:
|
||
|
|
from main import JournalOrganizerAgent
|
||
|
|
|
||
|
|
test_results = {}
|
||
|
|
|
||
|
|
# Test 1: Load from specific config file
|
||
|
|
config_path = self.create_test_config()
|
||
|
|
agent1 = JournalOrganizerAgent(str(config_path))
|
||
|
|
test_results["specific_config"] = bool(agent1.config)
|
||
|
|
|
||
|
|
# Test 2: Load with no config file (should use defaults)
|
||
|
|
agent2 = JournalOrganizerAgent(None)
|
||
|
|
test_results["default_config"] = isinstance(agent2.config, dict)
|
||
|
|
|
||
|
|
# Test 3: Load with non-existent config file (should use defaults)
|
||
|
|
agent3 = JournalOrganizerAgent("/non/existent/config.yaml")
|
||
|
|
test_results["fallback_config"] = isinstance(agent3.config, dict)
|
||
|
|
|
||
|
|
result.set_success("Configuration loading works correctly", test_results)
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
result.set_failure(f"Configuration loading failed: {str(e)}", e)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
async def test_command_registration(self) -> ValidationResult:
|
||
|
|
"""Test command registration and basic info retrieval"""
|
||
|
|
result = ValidationResult("Command Registration")
|
||
|
|
|
||
|
|
try:
|
||
|
|
from main import JournalOrganizerAgent
|
||
|
|
|
||
|
|
config_path = self.create_test_config()
|
||
|
|
agent = JournalOrganizerAgent(str(config_path))
|
||
|
|
|
||
|
|
# Test command listing
|
||
|
|
commands = agent.list_commands()
|
||
|
|
assert "organize" in commands, "Should have 'organize' command"
|
||
|
|
|
||
|
|
# Test command info retrieval
|
||
|
|
organize_info = agent.get_command_info("organize")
|
||
|
|
assert isinstance(organize_info, dict), "Command info should be a dict"
|
||
|
|
assert "name" in organize_info, "Command info should have name"
|
||
|
|
|
||
|
|
# Test all commands info
|
||
|
|
all_info = agent.get_all_commands_info()
|
||
|
|
assert isinstance(all_info, dict), "All commands info should be a dict"
|
||
|
|
|
||
|
|
result.set_success("Command registration working correctly", {
|
||
|
|
"registered_commands": commands,
|
||
|
|
"organize_command_info": bool(organize_info),
|
||
|
|
"all_commands_info": bool(all_info)
|
||
|
|
})
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
result.set_failure(f"Command registration failed: {str(e)}", e)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
async def test_conversational_agent_init(self) -> ValidationResult:
|
||
|
|
"""Test conversational agent initialization"""
|
||
|
|
result = ValidationResult("Conversational Agent Initialization")
|
||
|
|
|
||
|
|
try:
|
||
|
|
from main import JournalOrganizerAgent
|
||
|
|
from conversation import ConversationalAgent
|
||
|
|
|
||
|
|
config_path = self.create_test_config()
|
||
|
|
journal_agent = JournalOrganizerAgent(str(config_path))
|
||
|
|
|
||
|
|
# Initialize conversational agent
|
||
|
|
conv_agent = ConversationalAgent(journal_agent.agent, journal_agent.config)
|
||
|
|
|
||
|
|
# Test basic properties
|
||
|
|
assert hasattr(conv_agent, 'agent'), "Should have agent attribute"
|
||
|
|
assert hasattr(conv_agent, 'config'), "Should have config attribute"
|
||
|
|
|
||
|
|
# Test initialization
|
||
|
|
welcome_msg = await conv_agent.initialize()
|
||
|
|
assert isinstance(welcome_msg, str), "Welcome message should be a string"
|
||
|
|
assert len(welcome_msg) > 0, "Welcome message should not be empty"
|
||
|
|
|
||
|
|
result.set_success("Conversational agent initialized successfully", {
|
||
|
|
"welcome_message_length": len(welcome_msg),
|
||
|
|
"has_required_attributes": True
|
||
|
|
})
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
result.set_failure(f"Conversational agent initialization failed: {str(e)}", e)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
async def test_dependency_management(self) -> ValidationResult:
|
||
|
|
"""Test dependency management and graceful degradation"""
|
||
|
|
result = ValidationResult("Dependency Management")
|
||
|
|
|
||
|
|
try:
|
||
|
|
from dependency_manager import get_dependency_manager
|
||
|
|
|
||
|
|
dep_manager = get_dependency_manager()
|
||
|
|
|
||
|
|
# Test dependency status
|
||
|
|
status_report = dep_manager.get_dependency_status_report()
|
||
|
|
assert isinstance(status_report, str), "Status report should be a string"
|
||
|
|
|
||
|
|
# Test missing dependencies check
|
||
|
|
missing_deps = dep_manager.get_missing_dependencies()
|
||
|
|
assert isinstance(missing_deps, list), "Missing deps should be a list"
|
||
|
|
|
||
|
|
# Test module retrieval (should work even if module is missing)
|
||
|
|
yaml_module = dep_manager.get_module('yaml')
|
||
|
|
# yaml_module could be None if not installed, which is fine
|
||
|
|
|
||
|
|
result.set_success("Dependency management working correctly", {
|
||
|
|
"status_report_generated": bool(status_report),
|
||
|
|
"missing_dependencies_count": len(missing_deps),
|
||
|
|
"yaml_module_available": yaml_module is not None
|
||
|
|
})
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
result.set_failure(f"Dependency management failed: {str(e)}", e)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
async def test_error_handling_framework(self) -> ValidationResult:
|
||
|
|
"""Test error handling framework"""
|
||
|
|
result = ValidationResult("Error Handling Framework")
|
||
|
|
|
||
|
|
try:
|
||
|
|
from error_handling import (
|
||
|
|
JournalOrganizerError, ConfigurationError,
|
||
|
|
APIError, ValidationError, ErrorHandler
|
||
|
|
)
|
||
|
|
|
||
|
|
# Test custom exceptions
|
||
|
|
test_exceptions = [
|
||
|
|
JournalOrganizerError("test"),
|
||
|
|
ConfigurationError("test config error"),
|
||
|
|
APIError("test api error"),
|
||
|
|
ValidationError("test validation error")
|
||
|
|
]
|
||
|
|
|
||
|
|
for exc in test_exceptions:
|
||
|
|
assert isinstance(exc, Exception), f"{type(exc).__name__} should be an Exception"
|
||
|
|
assert str(exc), f"{type(exc).__name__} should have string representation"
|
||
|
|
|
||
|
|
# Test ErrorHandler
|
||
|
|
import logging
|
||
|
|
logger = logging.getLogger("test")
|
||
|
|
error_handler = ErrorHandler(logger)
|
||
|
|
|
||
|
|
assert hasattr(error_handler, 'handle_api_error'), "Should have handle_api_error method"
|
||
|
|
assert hasattr(error_handler, 'handle_validation_error'), "Should have handle_validation_error method"
|
||
|
|
|
||
|
|
result.set_success("Error handling framework working correctly", {
|
||
|
|
"custom_exceptions_count": len(test_exceptions),
|
||
|
|
"error_handler_methods": ["handle_api_error", "handle_validation_error"]
|
||
|
|
})
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
result.set_failure(f"Error handling framework test failed: {str(e)}", e)
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
async def run_all_tests(self) -> List[ValidationResult]:
|
||
|
|
"""Run all validation tests"""
|
||
|
|
logger.info("🚀 Starting application startup validation...")
|
||
|
|
|
||
|
|
test_methods = [
|
||
|
|
self.test_import_main_modules,
|
||
|
|
self.test_agent_initialization,
|
||
|
|
self.test_configuration_loading,
|
||
|
|
self.test_command_registration,
|
||
|
|
self.test_conversational_agent_init,
|
||
|
|
self.test_dependency_management,
|
||
|
|
self.test_error_handling_framework
|
||
|
|
]
|
||
|
|
|
||
|
|
for test_method in test_methods:
|
||
|
|
try:
|
||
|
|
logger.info(f"Running {test_method.__name__}...")
|
||
|
|
result = await test_method()
|
||
|
|
self.results.append(result)
|
||
|
|
logger.info(str(result))
|
||
|
|
except Exception as e:
|
||
|
|
error_result = ValidationResult(test_method.__name__)
|
||
|
|
error_result.set_failure(f"Test execution failed: {str(e)}", e)
|
||
|
|
self.results.append(error_result)
|
||
|
|
logger.error(str(error_result))
|
||
|
|
|
||
|
|
# Cleanup
|
||
|
|
self.cleanup_test_config()
|
||
|
|
|
||
|
|
return self.results
|
||
|
|
|
||
|
|
def generate_report(self) -> str:
|
||
|
|
"""Generate a comprehensive validation report"""
|
||
|
|
total_tests = len(self.results)
|
||
|
|
passed_tests = sum(1 for r in self.results if r.success)
|
||
|
|
failed_tests = total_tests - passed_tests
|
||
|
|
|
||
|
|
report = f"""
|
||
|
|
{'='*60}
|
||
|
|
APPLICATION STARTUP VALIDATION REPORT
|
||
|
|
{'='*60}
|
||
|
|
|
||
|
|
Summary:
|
||
|
|
Total Tests: {total_tests}
|
||
|
|
Passed: {passed_tests}
|
||
|
|
Failed: {failed_tests}
|
||
|
|
Success Rate: {(passed_tests/total_tests*100):.1f}%
|
||
|
|
|
||
|
|
Test Results:
|
||
|
|
"""
|
||
|
|
|
||
|
|
for result in self.results:
|
||
|
|
report += f"\n{str(result)}"
|
||
|
|
if result.details:
|
||
|
|
for key, value in result.details.items():
|
||
|
|
report += f"\n - {key}: {value}"
|
||
|
|
if not result.success and result.error:
|
||
|
|
report += f"\n - Error: {str(result.error)}"
|
||
|
|
|
||
|
|
report += f"\n\n{'='*60}\n"
|
||
|
|
|
||
|
|
return report
|
||
|
|
|
||
|
|
|
||
|
|
async def main():
|
||
|
|
"""Main validation function"""
|
||
|
|
validator = StartupValidator()
|
||
|
|
|
||
|
|
try:
|
||
|
|
results = await validator.run_all_tests()
|
||
|
|
report = validator.generate_report()
|
||
|
|
|
||
|
|
print(report)
|
||
|
|
|
||
|
|
# Return appropriate exit code
|
||
|
|
failed_count = sum(1 for r in results if not r.success)
|
||
|
|
return 0 if failed_count == 0 else 1
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Validation failed with unexpected error: {str(e)}")
|
||
|
|
traceback.print_exc()
|
||
|
|
return 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
exit_code = asyncio.run(main())
|
||
|
|
sys.exit(exit_code)
|