Initial project setup: Obsidian intelligent journal organizer
- Add core agent architecture with Command + Skill pattern - Implement Claude API integration for content analysis - Add Obsidian REST API integration for vault operations - Create conversational interface (v2.0) with natural language processing - Add comprehensive configuration management and validation - Include project documentation and developer guides - Set up testing framework with unit, integration, and property tests - Add Kiro specs for Claude API configuration and code quality improvements - Configure project steering files for development guidelines
This commit is contained in:
windyboy
@@ -0,0 +1,496 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Performance and reliability testing script.
|
||||
Tests various input sizes, edge cases, memory usage, and error recovery.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Tuple, Optional
|
||||
|
||||
# Try to import psutil, fallback to basic memory tracking
|
||||
try:
|
||||
import psutil
|
||||
HAS_PSUTIL = True
|
||||
except ImportError:
|
||||
HAS_PSUTIL = False
|
||||
psutil = None
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
logger = logging.getLogger("PerformanceReliability")
|
||||
|
||||
|
||||
class PerformanceReliabilityTest:
|
||||
"""Test performance and reliability of core components"""
|
||||
|
||||
def __init__(self):
|
||||
self.test_results: List[Tuple[str, bool, str, Dict[str, Any]]] = []
|
||||
if HAS_PSUTIL:
|
||||
self.process = psutil.Process()
|
||||
else:
|
||||
self.process = None
|
||||
|
||||
def get_memory_usage(self) -> float:
|
||||
"""Get current memory usage in MB"""
|
||||
if HAS_PSUTIL and self.process:
|
||||
return self.process.memory_info().rss / 1024 / 1024
|
||||
else:
|
||||
# Fallback to basic memory tracking
|
||||
return 0.0 # Can't measure without psutil
|
||||
|
||||
def test_dependency_manager_performance(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||||
"""Test dependency manager performance with multiple calls"""
|
||||
test_name = "Dependency Manager Performance Test"
|
||||
metrics = {}
|
||||
|
||||
try:
|
||||
import dependency_manager
|
||||
|
||||
# Measure initial memory
|
||||
initial_memory = self.get_memory_usage()
|
||||
|
||||
# Test multiple instantiations
|
||||
start_time = time.time()
|
||||
managers = []
|
||||
for i in range(100):
|
||||
manager = dependency_manager.get_dependency_manager()
|
||||
managers.append(manager)
|
||||
|
||||
creation_time = time.time() - start_time
|
||||
|
||||
# Test status report generation performance
|
||||
start_time = time.time()
|
||||
for manager in managers[:10]: # Test first 10
|
||||
status = manager.get_dependency_status_report()
|
||||
report_time = time.time() - start_time
|
||||
|
||||
# Measure final memory
|
||||
final_memory = self.get_memory_usage()
|
||||
memory_increase = final_memory - initial_memory if HAS_PSUTIL else 0.0
|
||||
|
||||
# Cleanup
|
||||
del managers
|
||||
gc.collect()
|
||||
|
||||
metrics = {
|
||||
"creation_time_100_instances": f"{creation_time:.3f}s",
|
||||
"report_generation_time_10_calls": f"{report_time:.3f}s",
|
||||
"memory_increase": f"{memory_increase:.2f}MB" if HAS_PSUTIL else "N/A (psutil not available)",
|
||||
"avg_creation_time": f"{creation_time/100*1000:.2f}ms"
|
||||
}
|
||||
|
||||
# Performance thresholds
|
||||
if creation_time > 5.0: # 5 seconds for 100 instances
|
||||
return test_name, False, "Dependency manager creation too slow", metrics
|
||||
|
||||
if HAS_PSUTIL and memory_increase > 50: # 50MB increase
|
||||
return test_name, False, "Excessive memory usage", metrics
|
||||
|
||||
return test_name, True, "Dependency manager performance acceptable", metrics
|
||||
|
||||
except Exception as e:
|
||||
return test_name, False, f"Performance test failed: {str(e)}", metrics
|
||||
|
||||
def test_error_handling_reliability(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||||
"""Test error handling reliability with various error conditions"""
|
||||
test_name = "Error Handling Reliability Test"
|
||||
metrics = {}
|
||||
|
||||
try:
|
||||
import error_handling
|
||||
import logging
|
||||
|
||||
# Create error handler
|
||||
logger = logging.getLogger("test_reliability")
|
||||
error_handler = error_handling.ErrorHandler(logger)
|
||||
|
||||
# Test various error scenarios
|
||||
test_errors = [
|
||||
Exception("Generic error"),
|
||||
ValueError("Value error"),
|
||||
TypeError("Type error"),
|
||||
RuntimeError("Runtime error"),
|
||||
ConnectionError("Connection error")
|
||||
]
|
||||
|
||||
successful_handles = 0
|
||||
start_time = time.time()
|
||||
|
||||
for i, error in enumerate(test_errors * 20): # 100 total errors
|
||||
try:
|
||||
result = error_handler.handle_api_error(error, "test_api", f"test_operation_{i}")
|
||||
# ErrorHandler returns a dict with success, message, etc.
|
||||
if isinstance(result, dict) and 'success' in result:
|
||||
successful_handles += 1
|
||||
elif result is not None: # Any non-None result counts as handled
|
||||
successful_handles += 1
|
||||
except Exception:
|
||||
pass # Error in error handling - not good but continue
|
||||
|
||||
handling_time = time.time() - start_time
|
||||
|
||||
metrics = {
|
||||
"total_errors_processed": len(test_errors) * 20,
|
||||
"successful_handles": successful_handles,
|
||||
"handling_time": f"{handling_time:.3f}s",
|
||||
"avg_handling_time": f"{handling_time/(len(test_errors)*20)*1000:.2f}ms",
|
||||
"success_rate": f"{successful_handles/(len(test_errors)*20)*100:.1f}%"
|
||||
}
|
||||
|
||||
# Reliability thresholds
|
||||
success_rate = successful_handles / (len(test_errors) * 20)
|
||||
if success_rate < 0.95: # 95% success rate
|
||||
return test_name, False, "Error handling success rate too low", metrics
|
||||
|
||||
if handling_time > 2.0: # 2 seconds for 100 errors
|
||||
return test_name, False, "Error handling too slow", metrics
|
||||
|
||||
return test_name, True, "Error handling reliability acceptable", metrics
|
||||
|
||||
except Exception as e:
|
||||
return test_name, False, f"Reliability test failed: {str(e)}", metrics
|
||||
|
||||
def test_configuration_edge_cases(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||||
"""Test configuration handling with edge cases"""
|
||||
test_name = "Configuration Edge Cases Test"
|
||||
metrics = {}
|
||||
|
||||
try:
|
||||
# Test various configuration scenarios
|
||||
test_configs = [
|
||||
{}, # Empty config
|
||||
{"invalid": "structure"}, # Invalid structure
|
||||
{"obsidian": {}}, # Partial config
|
||||
{"obsidian": {"vault_path": ""}}, # Empty values
|
||||
{"obsidian": {"vault_path": "/nonexistent/path"}}, # Invalid paths
|
||||
{"claude": {"api_key": ""}}, # Empty API key
|
||||
{"claude": {"api_key": "x" * 1000}}, # Very long API key
|
||||
]
|
||||
|
||||
successful_loads = 0
|
||||
start_time = time.time()
|
||||
|
||||
for i, config in enumerate(test_configs):
|
||||
try:
|
||||
# Create temporary config file
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
json.dump(config, f)
|
||||
temp_path = f.name
|
||||
|
||||
# Try to load config (this would normally be done by main.py)
|
||||
with open(temp_path, 'r') as f:
|
||||
loaded_config = json.load(f)
|
||||
|
||||
# Basic validation that it's a dict
|
||||
if isinstance(loaded_config, dict):
|
||||
successful_loads += 1
|
||||
|
||||
# Cleanup
|
||||
os.unlink(temp_path)
|
||||
|
||||
except Exception:
|
||||
# Expected for some edge cases
|
||||
pass
|
||||
|
||||
processing_time = time.time() - start_time
|
||||
|
||||
metrics = {
|
||||
"total_configs_tested": len(test_configs),
|
||||
"successful_loads": successful_loads,
|
||||
"processing_time": f"{processing_time:.3f}s",
|
||||
"avg_processing_time": f"{processing_time/len(test_configs)*1000:.2f}ms"
|
||||
}
|
||||
|
||||
# Should handle at least basic configs
|
||||
if successful_loads < 3:
|
||||
return test_name, False, "Too many configuration failures", metrics
|
||||
|
||||
return test_name, True, "Configuration edge cases handled", metrics
|
||||
|
||||
except Exception as e:
|
||||
return test_name, False, f"Edge case test failed: {str(e)}", metrics
|
||||
|
||||
def test_memory_usage_patterns(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||||
"""Test memory usage patterns and potential leaks"""
|
||||
test_name = "Memory Usage Patterns Test"
|
||||
metrics = {}
|
||||
|
||||
try:
|
||||
# Measure baseline memory
|
||||
gc.collect()
|
||||
baseline_memory = self.get_memory_usage()
|
||||
|
||||
# Test repeated operations that might cause memory leaks
|
||||
operations_memory = []
|
||||
|
||||
for iteration in range(10):
|
||||
# Simulate typical operations
|
||||
temp_data = []
|
||||
|
||||
# Create temporary objects
|
||||
for i in range(1000):
|
||||
temp_data.append({
|
||||
"id": i,
|
||||
"data": "x" * 100, # 100 chars
|
||||
"nested": {"value": i * 2}
|
||||
})
|
||||
|
||||
# Process data
|
||||
processed = [item for item in temp_data if item["id"] % 2 == 0]
|
||||
|
||||
# Measure memory after each iteration
|
||||
current_memory = self.get_memory_usage()
|
||||
operations_memory.append(current_memory)
|
||||
|
||||
# Cleanup
|
||||
del temp_data, processed
|
||||
gc.collect()
|
||||
|
||||
# Final memory measurement
|
||||
final_memory = self.get_memory_usage()
|
||||
memory_increase = final_memory - baseline_memory
|
||||
|
||||
# Calculate memory growth trend
|
||||
if len(operations_memory) > 1:
|
||||
memory_growth = operations_memory[-1] - operations_memory[0]
|
||||
else:
|
||||
memory_growth = 0
|
||||
|
||||
metrics = {
|
||||
"baseline_memory": f"{baseline_memory:.2f}MB" if HAS_PSUTIL else "N/A",
|
||||
"final_memory": f"{final_memory:.2f}MB" if HAS_PSUTIL else "N/A",
|
||||
"total_memory_increase": f"{memory_increase:.2f}MB" if HAS_PSUTIL else "N/A",
|
||||
"memory_growth_trend": f"{memory_growth:.2f}MB" if HAS_PSUTIL else "N/A",
|
||||
"max_memory_during_test": f"{max(operations_memory):.2f}MB" if HAS_PSUTIL and operations_memory else "N/A",
|
||||
"iterations_tested": 10
|
||||
}
|
||||
|
||||
# Memory usage thresholds (only check if psutil available)
|
||||
if HAS_PSUTIL:
|
||||
if memory_increase > 20: # 20MB increase after cleanup
|
||||
return test_name, False, "Potential memory leak detected", metrics
|
||||
|
||||
if memory_growth > 15: # 15MB growth trend
|
||||
return test_name, False, "Memory growth trend concerning", metrics
|
||||
|
||||
return test_name, True, "Memory usage patterns acceptable", metrics
|
||||
|
||||
except Exception as e:
|
||||
return test_name, False, f"Memory test failed: {str(e)}", metrics
|
||||
|
||||
def test_graceful_degradation(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||||
"""Test graceful degradation when dependencies are missing"""
|
||||
test_name = "Graceful Degradation Test"
|
||||
metrics = {}
|
||||
|
||||
try:
|
||||
import dependency_manager
|
||||
|
||||
# Test dependency manager with missing modules
|
||||
dep_manager = dependency_manager.get_dependency_manager()
|
||||
|
||||
# Test getting non-existent modules
|
||||
test_modules = [
|
||||
'nonexistent_module',
|
||||
'fake_dependency',
|
||||
'missing_package',
|
||||
'invalid.module.name'
|
||||
]
|
||||
|
||||
successful_degradations = 0
|
||||
start_time = time.time()
|
||||
|
||||
for module_name in test_modules:
|
||||
try:
|
||||
result = dep_manager.get_module(module_name)
|
||||
# Should return None for missing modules, not crash
|
||||
if result is None:
|
||||
successful_degradations += 1
|
||||
except Exception:
|
||||
# Should not raise exceptions for missing modules
|
||||
pass
|
||||
|
||||
degradation_time = time.time() - start_time
|
||||
|
||||
# Test status report with missing dependencies
|
||||
try:
|
||||
status_report = dep_manager.get_dependency_status_report()
|
||||
status_report_works = isinstance(status_report, str) and len(status_report) > 0
|
||||
except Exception:
|
||||
status_report_works = False
|
||||
|
||||
metrics = {
|
||||
"modules_tested": len(test_modules),
|
||||
"successful_degradations": successful_degradations,
|
||||
"degradation_time": f"{degradation_time:.3f}s",
|
||||
"status_report_works": status_report_works,
|
||||
"degradation_rate": f"{successful_degradations/len(test_modules)*100:.1f}%"
|
||||
}
|
||||
|
||||
# Should gracefully handle all missing modules
|
||||
if successful_degradations < len(test_modules):
|
||||
return test_name, False, "Not all missing modules handled gracefully", metrics
|
||||
|
||||
if not status_report_works:
|
||||
return test_name, False, "Status report fails with missing dependencies", metrics
|
||||
|
||||
return test_name, True, "Graceful degradation working correctly", metrics
|
||||
|
||||
except Exception as e:
|
||||
return test_name, False, f"Degradation test failed: {str(e)}", metrics
|
||||
|
||||
def test_concurrent_operations(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||||
"""Test concurrent operations and thread safety"""
|
||||
test_name = "Concurrent Operations Test"
|
||||
metrics = {}
|
||||
|
||||
try:
|
||||
import dependency_manager
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
|
||||
async def async_operation(operation_id: int) -> bool:
|
||||
"""Simulate async operation"""
|
||||
try:
|
||||
dep_manager = dependency_manager.get_dependency_manager()
|
||||
status = dep_manager.get_dependency_status_report()
|
||||
missing = dep_manager.get_missing_dependencies()
|
||||
|
||||
# Simulate some processing time
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
return len(status) > 0 and isinstance(missing, list)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Test concurrent async operations
|
||||
start_time = time.time()
|
||||
|
||||
async def run_concurrent_test():
|
||||
tasks = [async_operation(i) for i in range(20)]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
return results
|
||||
|
||||
results = asyncio.run(run_concurrent_test())
|
||||
concurrent_time = time.time() - start_time
|
||||
|
||||
successful_operations = sum(1 for r in results if r is True)
|
||||
|
||||
metrics = {
|
||||
"concurrent_operations": 20,
|
||||
"successful_operations": successful_operations,
|
||||
"concurrent_time": f"{concurrent_time:.3f}s",
|
||||
"avg_operation_time": f"{concurrent_time/20*1000:.2f}ms",
|
||||
"success_rate": f"{successful_operations/20*100:.1f}%"
|
||||
}
|
||||
|
||||
# Concurrency thresholds
|
||||
if successful_operations < 18: # 90% success rate
|
||||
return test_name, False, "Concurrent operations success rate too low", metrics
|
||||
|
||||
if concurrent_time > 5.0: # 5 seconds for 20 operations
|
||||
return test_name, False, "Concurrent operations too slow", metrics
|
||||
|
||||
return test_name, True, "Concurrent operations working correctly", metrics
|
||||
|
||||
except Exception as e:
|
||||
return test_name, False, f"Concurrency test failed: {str(e)}", metrics
|
||||
|
||||
def run_all_tests(self) -> List[Tuple[str, bool, str, Dict[str, Any]]]:
|
||||
"""Run all performance and reliability tests"""
|
||||
logger.info("🚀 Starting performance and reliability tests...")
|
||||
|
||||
test_methods = [
|
||||
self.test_dependency_manager_performance,
|
||||
self.test_error_handling_reliability,
|
||||
self.test_configuration_edge_cases,
|
||||
self.test_memory_usage_patterns,
|
||||
self.test_graceful_degradation,
|
||||
self.test_concurrent_operations
|
||||
]
|
||||
|
||||
for test_method in test_methods:
|
||||
try:
|
||||
logger.info(f"Running {test_method.__name__}...")
|
||||
result = test_method()
|
||||
self.test_results.append(result)
|
||||
|
||||
status = "✅ PASS" if result[1] else "❌ FAIL"
|
||||
logger.info(f"{status} {result[0]}: {result[2]}")
|
||||
|
||||
except Exception as e:
|
||||
error_result = (test_method.__name__, False, f"Test execution failed: {str(e)}", {})
|
||||
self.test_results.append(error_result)
|
||||
logger.error(f"❌ FAIL {error_result[0]}: {error_result[2]}")
|
||||
|
||||
return self.test_results
|
||||
|
||||
def generate_report(self) -> str:
|
||||
"""Generate performance and reliability test report"""
|
||||
total_tests = len(self.test_results)
|
||||
passed_tests = sum(1 for _, success, _, _ in self.test_results if success)
|
||||
failed_tests = total_tests - passed_tests
|
||||
|
||||
report = f"""
|
||||
{'='*60}
|
||||
PERFORMANCE AND RELIABILITY TEST REPORT
|
||||
{'='*60}
|
||||
|
||||
Summary:
|
||||
Total Tests: {total_tests}
|
||||
Passed: {passed_tests}
|
||||
Failed: {failed_tests}
|
||||
Success Rate: {(passed_tests/total_tests*100):.1f}%
|
||||
|
||||
Test Results:
|
||||
"""
|
||||
|
||||
for test_name, success, message, metrics in self.test_results:
|
||||
status = "✅ PASS" if success else "❌ FAIL"
|
||||
report += f"\n{status} {test_name}: {message}"
|
||||
|
||||
if metrics:
|
||||
report += "\n Metrics:"
|
||||
for key, value in metrics.items():
|
||||
report += f"\n - {key}: {value}"
|
||||
|
||||
report += f"\n\n{'='*60}\n"
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
"""Main test function"""
|
||||
tester = PerformanceReliabilityTest()
|
||||
|
||||
try:
|
||||
results = tester.run_all_tests()
|
||||
report = tester.generate_report()
|
||||
|
||||
print(report)
|
||||
|
||||
# Return appropriate exit code
|
||||
failed_count = sum(1 for _, success, _, _ in results if not success)
|
||||
return 0 if failed_count == 0 else 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test execution failed: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = main()
|
||||
sys.exit(exit_code)
|
||||
Reference in New Issue
Block a user