496 lines
19 KiB
Python
496 lines
19 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
Performance and reliability testing script.
|
||
|
|
Tests various input sizes, edge cases, memory usage, and error recovery.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import gc
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
import time
|
||
|
|
import traceback
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Dict, Any, List, Tuple, Optional
|
||
|
|
|
||
|
|
# Try to import psutil, fallback to basic memory tracking
|
||
|
|
try:
|
||
|
|
import psutil
|
||
|
|
HAS_PSUTIL = True
|
||
|
|
except ImportError:
|
||
|
|
HAS_PSUTIL = False
|
||
|
|
psutil = None
|
||
|
|
|
||
|
|
# Configure logging
|
||
|
|
logging.basicConfig(
|
||
|
|
level=logging.INFO,
|
||
|
|
format="%(asctime)s - %(levelname)s - %(message)s"
|
||
|
|
)
|
||
|
|
logger = logging.getLogger("PerformanceReliability")
|
||
|
|
|
||
|
|
|
||
|
|
class PerformanceReliabilityTest:
|
||
|
|
"""Test performance and reliability of core components"""
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.test_results: List[Tuple[str, bool, str, Dict[str, Any]]] = []
|
||
|
|
if HAS_PSUTIL:
|
||
|
|
self.process = psutil.Process()
|
||
|
|
else:
|
||
|
|
self.process = None
|
||
|
|
|
||
|
|
def get_memory_usage(self) -> float:
|
||
|
|
"""Get current memory usage in MB"""
|
||
|
|
if HAS_PSUTIL and self.process:
|
||
|
|
return self.process.memory_info().rss / 1024 / 1024
|
||
|
|
else:
|
||
|
|
# Fallback to basic memory tracking
|
||
|
|
return 0.0 # Can't measure without psutil
|
||
|
|
|
||
|
|
def test_dependency_manager_performance(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||
|
|
"""Test dependency manager performance with multiple calls"""
|
||
|
|
test_name = "Dependency Manager Performance Test"
|
||
|
|
metrics = {}
|
||
|
|
|
||
|
|
try:
|
||
|
|
import dependency_manager
|
||
|
|
|
||
|
|
# Measure initial memory
|
||
|
|
initial_memory = self.get_memory_usage()
|
||
|
|
|
||
|
|
# Test multiple instantiations
|
||
|
|
start_time = time.time()
|
||
|
|
managers = []
|
||
|
|
for i in range(100):
|
||
|
|
manager = dependency_manager.get_dependency_manager()
|
||
|
|
managers.append(manager)
|
||
|
|
|
||
|
|
creation_time = time.time() - start_time
|
||
|
|
|
||
|
|
# Test status report generation performance
|
||
|
|
start_time = time.time()
|
||
|
|
for manager in managers[:10]: # Test first 10
|
||
|
|
status = manager.get_dependency_status_report()
|
||
|
|
report_time = time.time() - start_time
|
||
|
|
|
||
|
|
# Measure final memory
|
||
|
|
final_memory = self.get_memory_usage()
|
||
|
|
memory_increase = final_memory - initial_memory if HAS_PSUTIL else 0.0
|
||
|
|
|
||
|
|
# Cleanup
|
||
|
|
del managers
|
||
|
|
gc.collect()
|
||
|
|
|
||
|
|
metrics = {
|
||
|
|
"creation_time_100_instances": f"{creation_time:.3f}s",
|
||
|
|
"report_generation_time_10_calls": f"{report_time:.3f}s",
|
||
|
|
"memory_increase": f"{memory_increase:.2f}MB" if HAS_PSUTIL else "N/A (psutil not available)",
|
||
|
|
"avg_creation_time": f"{creation_time/100*1000:.2f}ms"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Performance thresholds
|
||
|
|
if creation_time > 5.0: # 5 seconds for 100 instances
|
||
|
|
return test_name, False, "Dependency manager creation too slow", metrics
|
||
|
|
|
||
|
|
if HAS_PSUTIL and memory_increase > 50: # 50MB increase
|
||
|
|
return test_name, False, "Excessive memory usage", metrics
|
||
|
|
|
||
|
|
return test_name, True, "Dependency manager performance acceptable", metrics
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
return test_name, False, f"Performance test failed: {str(e)}", metrics
|
||
|
|
|
||
|
|
def test_error_handling_reliability(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||
|
|
"""Test error handling reliability with various error conditions"""
|
||
|
|
test_name = "Error Handling Reliability Test"
|
||
|
|
metrics = {}
|
||
|
|
|
||
|
|
try:
|
||
|
|
import error_handling
|
||
|
|
import logging
|
||
|
|
|
||
|
|
# Create error handler
|
||
|
|
logger = logging.getLogger("test_reliability")
|
||
|
|
error_handler = error_handling.ErrorHandler(logger)
|
||
|
|
|
||
|
|
# Test various error scenarios
|
||
|
|
test_errors = [
|
||
|
|
Exception("Generic error"),
|
||
|
|
ValueError("Value error"),
|
||
|
|
TypeError("Type error"),
|
||
|
|
RuntimeError("Runtime error"),
|
||
|
|
ConnectionError("Connection error")
|
||
|
|
]
|
||
|
|
|
||
|
|
successful_handles = 0
|
||
|
|
start_time = time.time()
|
||
|
|
|
||
|
|
for i, error in enumerate(test_errors * 20): # 100 total errors
|
||
|
|
try:
|
||
|
|
result = error_handler.handle_api_error(error, "test_api", f"test_operation_{i}")
|
||
|
|
# ErrorHandler returns a dict with success, message, etc.
|
||
|
|
if isinstance(result, dict) and 'success' in result:
|
||
|
|
successful_handles += 1
|
||
|
|
elif result is not None: # Any non-None result counts as handled
|
||
|
|
successful_handles += 1
|
||
|
|
except Exception:
|
||
|
|
pass # Error in error handling - not good but continue
|
||
|
|
|
||
|
|
handling_time = time.time() - start_time
|
||
|
|
|
||
|
|
metrics = {
|
||
|
|
"total_errors_processed": len(test_errors) * 20,
|
||
|
|
"successful_handles": successful_handles,
|
||
|
|
"handling_time": f"{handling_time:.3f}s",
|
||
|
|
"avg_handling_time": f"{handling_time/(len(test_errors)*20)*1000:.2f}ms",
|
||
|
|
"success_rate": f"{successful_handles/(len(test_errors)*20)*100:.1f}%"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Reliability thresholds
|
||
|
|
success_rate = successful_handles / (len(test_errors) * 20)
|
||
|
|
if success_rate < 0.95: # 95% success rate
|
||
|
|
return test_name, False, "Error handling success rate too low", metrics
|
||
|
|
|
||
|
|
if handling_time > 2.0: # 2 seconds for 100 errors
|
||
|
|
return test_name, False, "Error handling too slow", metrics
|
||
|
|
|
||
|
|
return test_name, True, "Error handling reliability acceptable", metrics
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
return test_name, False, f"Reliability test failed: {str(e)}", metrics
|
||
|
|
|
||
|
|
def test_configuration_edge_cases(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||
|
|
"""Test configuration handling with edge cases"""
|
||
|
|
test_name = "Configuration Edge Cases Test"
|
||
|
|
metrics = {}
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Test various configuration scenarios
|
||
|
|
test_configs = [
|
||
|
|
{}, # Empty config
|
||
|
|
{"invalid": "structure"}, # Invalid structure
|
||
|
|
{"obsidian": {}}, # Partial config
|
||
|
|
{"obsidian": {"vault_path": ""}}, # Empty values
|
||
|
|
{"obsidian": {"vault_path": "/nonexistent/path"}}, # Invalid paths
|
||
|
|
{"claude": {"api_key": ""}}, # Empty API key
|
||
|
|
{"claude": {"api_key": "x" * 1000}}, # Very long API key
|
||
|
|
]
|
||
|
|
|
||
|
|
successful_loads = 0
|
||
|
|
start_time = time.time()
|
||
|
|
|
||
|
|
for i, config in enumerate(test_configs):
|
||
|
|
try:
|
||
|
|
# Create temporary config file
|
||
|
|
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||
|
|
json.dump(config, f)
|
||
|
|
temp_path = f.name
|
||
|
|
|
||
|
|
# Try to load config (this would normally be done by main.py)
|
||
|
|
with open(temp_path, 'r') as f:
|
||
|
|
loaded_config = json.load(f)
|
||
|
|
|
||
|
|
# Basic validation that it's a dict
|
||
|
|
if isinstance(loaded_config, dict):
|
||
|
|
successful_loads += 1
|
||
|
|
|
||
|
|
# Cleanup
|
||
|
|
os.unlink(temp_path)
|
||
|
|
|
||
|
|
except Exception:
|
||
|
|
# Expected for some edge cases
|
||
|
|
pass
|
||
|
|
|
||
|
|
processing_time = time.time() - start_time
|
||
|
|
|
||
|
|
metrics = {
|
||
|
|
"total_configs_tested": len(test_configs),
|
||
|
|
"successful_loads": successful_loads,
|
||
|
|
"processing_time": f"{processing_time:.3f}s",
|
||
|
|
"avg_processing_time": f"{processing_time/len(test_configs)*1000:.2f}ms"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Should handle at least basic configs
|
||
|
|
if successful_loads < 3:
|
||
|
|
return test_name, False, "Too many configuration failures", metrics
|
||
|
|
|
||
|
|
return test_name, True, "Configuration edge cases handled", metrics
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
return test_name, False, f"Edge case test failed: {str(e)}", metrics
|
||
|
|
|
||
|
|
def test_memory_usage_patterns(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||
|
|
"""Test memory usage patterns and potential leaks"""
|
||
|
|
test_name = "Memory Usage Patterns Test"
|
||
|
|
metrics = {}
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Measure baseline memory
|
||
|
|
gc.collect()
|
||
|
|
baseline_memory = self.get_memory_usage()
|
||
|
|
|
||
|
|
# Test repeated operations that might cause memory leaks
|
||
|
|
operations_memory = []
|
||
|
|
|
||
|
|
for iteration in range(10):
|
||
|
|
# Simulate typical operations
|
||
|
|
temp_data = []
|
||
|
|
|
||
|
|
# Create temporary objects
|
||
|
|
for i in range(1000):
|
||
|
|
temp_data.append({
|
||
|
|
"id": i,
|
||
|
|
"data": "x" * 100, # 100 chars
|
||
|
|
"nested": {"value": i * 2}
|
||
|
|
})
|
||
|
|
|
||
|
|
# Process data
|
||
|
|
processed = [item for item in temp_data if item["id"] % 2 == 0]
|
||
|
|
|
||
|
|
# Measure memory after each iteration
|
||
|
|
current_memory = self.get_memory_usage()
|
||
|
|
operations_memory.append(current_memory)
|
||
|
|
|
||
|
|
# Cleanup
|
||
|
|
del temp_data, processed
|
||
|
|
gc.collect()
|
||
|
|
|
||
|
|
# Final memory measurement
|
||
|
|
final_memory = self.get_memory_usage()
|
||
|
|
memory_increase = final_memory - baseline_memory
|
||
|
|
|
||
|
|
# Calculate memory growth trend
|
||
|
|
if len(operations_memory) > 1:
|
||
|
|
memory_growth = operations_memory[-1] - operations_memory[0]
|
||
|
|
else:
|
||
|
|
memory_growth = 0
|
||
|
|
|
||
|
|
metrics = {
|
||
|
|
"baseline_memory": f"{baseline_memory:.2f}MB" if HAS_PSUTIL else "N/A",
|
||
|
|
"final_memory": f"{final_memory:.2f}MB" if HAS_PSUTIL else "N/A",
|
||
|
|
"total_memory_increase": f"{memory_increase:.2f}MB" if HAS_PSUTIL else "N/A",
|
||
|
|
"memory_growth_trend": f"{memory_growth:.2f}MB" if HAS_PSUTIL else "N/A",
|
||
|
|
"max_memory_during_test": f"{max(operations_memory):.2f}MB" if HAS_PSUTIL and operations_memory else "N/A",
|
||
|
|
"iterations_tested": 10
|
||
|
|
}
|
||
|
|
|
||
|
|
# Memory usage thresholds (only check if psutil available)
|
||
|
|
if HAS_PSUTIL:
|
||
|
|
if memory_increase > 20: # 20MB increase after cleanup
|
||
|
|
return test_name, False, "Potential memory leak detected", metrics
|
||
|
|
|
||
|
|
if memory_growth > 15: # 15MB growth trend
|
||
|
|
return test_name, False, "Memory growth trend concerning", metrics
|
||
|
|
|
||
|
|
return test_name, True, "Memory usage patterns acceptable", metrics
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
return test_name, False, f"Memory test failed: {str(e)}", metrics
|
||
|
|
|
||
|
|
def test_graceful_degradation(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||
|
|
"""Test graceful degradation when dependencies are missing"""
|
||
|
|
test_name = "Graceful Degradation Test"
|
||
|
|
metrics = {}
|
||
|
|
|
||
|
|
try:
|
||
|
|
import dependency_manager
|
||
|
|
|
||
|
|
# Test dependency manager with missing modules
|
||
|
|
dep_manager = dependency_manager.get_dependency_manager()
|
||
|
|
|
||
|
|
# Test getting non-existent modules
|
||
|
|
test_modules = [
|
||
|
|
'nonexistent_module',
|
||
|
|
'fake_dependency',
|
||
|
|
'missing_package',
|
||
|
|
'invalid.module.name'
|
||
|
|
]
|
||
|
|
|
||
|
|
successful_degradations = 0
|
||
|
|
start_time = time.time()
|
||
|
|
|
||
|
|
for module_name in test_modules:
|
||
|
|
try:
|
||
|
|
result = dep_manager.get_module(module_name)
|
||
|
|
# Should return None for missing modules, not crash
|
||
|
|
if result is None:
|
||
|
|
successful_degradations += 1
|
||
|
|
except Exception:
|
||
|
|
# Should not raise exceptions for missing modules
|
||
|
|
pass
|
||
|
|
|
||
|
|
degradation_time = time.time() - start_time
|
||
|
|
|
||
|
|
# Test status report with missing dependencies
|
||
|
|
try:
|
||
|
|
status_report = dep_manager.get_dependency_status_report()
|
||
|
|
status_report_works = isinstance(status_report, str) and len(status_report) > 0
|
||
|
|
except Exception:
|
||
|
|
status_report_works = False
|
||
|
|
|
||
|
|
metrics = {
|
||
|
|
"modules_tested": len(test_modules),
|
||
|
|
"successful_degradations": successful_degradations,
|
||
|
|
"degradation_time": f"{degradation_time:.3f}s",
|
||
|
|
"status_report_works": status_report_works,
|
||
|
|
"degradation_rate": f"{successful_degradations/len(test_modules)*100:.1f}%"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Should gracefully handle all missing modules
|
||
|
|
if successful_degradations < len(test_modules):
|
||
|
|
return test_name, False, "Not all missing modules handled gracefully", metrics
|
||
|
|
|
||
|
|
if not status_report_works:
|
||
|
|
return test_name, False, "Status report fails with missing dependencies", metrics
|
||
|
|
|
||
|
|
return test_name, True, "Graceful degradation working correctly", metrics
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
return test_name, False, f"Degradation test failed: {str(e)}", metrics
|
||
|
|
|
||
|
|
def test_concurrent_operations(self) -> Tuple[str, bool, str, Dict[str, Any]]:
|
||
|
|
"""Test concurrent operations and thread safety"""
|
||
|
|
test_name = "Concurrent Operations Test"
|
||
|
|
metrics = {}
|
||
|
|
|
||
|
|
try:
|
||
|
|
import dependency_manager
|
||
|
|
import asyncio
|
||
|
|
import concurrent.futures
|
||
|
|
|
||
|
|
async def async_operation(operation_id: int) -> bool:
|
||
|
|
"""Simulate async operation"""
|
||
|
|
try:
|
||
|
|
dep_manager = dependency_manager.get_dependency_manager()
|
||
|
|
status = dep_manager.get_dependency_status_report()
|
||
|
|
missing = dep_manager.get_missing_dependencies()
|
||
|
|
|
||
|
|
# Simulate some processing time
|
||
|
|
await asyncio.sleep(0.01)
|
||
|
|
|
||
|
|
return len(status) > 0 and isinstance(missing, list)
|
||
|
|
except Exception:
|
||
|
|
return False
|
||
|
|
|
||
|
|
# Test concurrent async operations
|
||
|
|
start_time = time.time()
|
||
|
|
|
||
|
|
async def run_concurrent_test():
|
||
|
|
tasks = [async_operation(i) for i in range(20)]
|
||
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||
|
|
return results
|
||
|
|
|
||
|
|
results = asyncio.run(run_concurrent_test())
|
||
|
|
concurrent_time = time.time() - start_time
|
||
|
|
|
||
|
|
successful_operations = sum(1 for r in results if r is True)
|
||
|
|
|
||
|
|
metrics = {
|
||
|
|
"concurrent_operations": 20,
|
||
|
|
"successful_operations": successful_operations,
|
||
|
|
"concurrent_time": f"{concurrent_time:.3f}s",
|
||
|
|
"avg_operation_time": f"{concurrent_time/20*1000:.2f}ms",
|
||
|
|
"success_rate": f"{successful_operations/20*100:.1f}%"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Concurrency thresholds
|
||
|
|
if successful_operations < 18: # 90% success rate
|
||
|
|
return test_name, False, "Concurrent operations success rate too low", metrics
|
||
|
|
|
||
|
|
if concurrent_time > 5.0: # 5 seconds for 20 operations
|
||
|
|
return test_name, False, "Concurrent operations too slow", metrics
|
||
|
|
|
||
|
|
return test_name, True, "Concurrent operations working correctly", metrics
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
return test_name, False, f"Concurrency test failed: {str(e)}", metrics
|
||
|
|
|
||
|
|
def run_all_tests(self) -> List[Tuple[str, bool, str, Dict[str, Any]]]:
|
||
|
|
"""Run all performance and reliability tests"""
|
||
|
|
logger.info("🚀 Starting performance and reliability tests...")
|
||
|
|
|
||
|
|
test_methods = [
|
||
|
|
self.test_dependency_manager_performance,
|
||
|
|
self.test_error_handling_reliability,
|
||
|
|
self.test_configuration_edge_cases,
|
||
|
|
self.test_memory_usage_patterns,
|
||
|
|
self.test_graceful_degradation,
|
||
|
|
self.test_concurrent_operations
|
||
|
|
]
|
||
|
|
|
||
|
|
for test_method in test_methods:
|
||
|
|
try:
|
||
|
|
logger.info(f"Running {test_method.__name__}...")
|
||
|
|
result = test_method()
|
||
|
|
self.test_results.append(result)
|
||
|
|
|
||
|
|
status = "✅ PASS" if result[1] else "❌ FAIL"
|
||
|
|
logger.info(f"{status} {result[0]}: {result[2]}")
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
error_result = (test_method.__name__, False, f"Test execution failed: {str(e)}", {})
|
||
|
|
self.test_results.append(error_result)
|
||
|
|
logger.error(f"❌ FAIL {error_result[0]}: {error_result[2]}")
|
||
|
|
|
||
|
|
return self.test_results
|
||
|
|
|
||
|
|
def generate_report(self) -> str:
|
||
|
|
"""Generate performance and reliability test report"""
|
||
|
|
total_tests = len(self.test_results)
|
||
|
|
passed_tests = sum(1 for _, success, _, _ in self.test_results if success)
|
||
|
|
failed_tests = total_tests - passed_tests
|
||
|
|
|
||
|
|
report = f"""
|
||
|
|
{'='*60}
|
||
|
|
PERFORMANCE AND RELIABILITY TEST REPORT
|
||
|
|
{'='*60}
|
||
|
|
|
||
|
|
Summary:
|
||
|
|
Total Tests: {total_tests}
|
||
|
|
Passed: {passed_tests}
|
||
|
|
Failed: {failed_tests}
|
||
|
|
Success Rate: {(passed_tests/total_tests*100):.1f}%
|
||
|
|
|
||
|
|
Test Results:
|
||
|
|
"""
|
||
|
|
|
||
|
|
for test_name, success, message, metrics in self.test_results:
|
||
|
|
status = "✅ PASS" if success else "❌ FAIL"
|
||
|
|
report += f"\n{status} {test_name}: {message}"
|
||
|
|
|
||
|
|
if metrics:
|
||
|
|
report += "\n Metrics:"
|
||
|
|
for key, value in metrics.items():
|
||
|
|
report += f"\n - {key}: {value}"
|
||
|
|
|
||
|
|
report += f"\n\n{'='*60}\n"
|
||
|
|
|
||
|
|
return report
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
"""Main test function"""
|
||
|
|
tester = PerformanceReliabilityTest()
|
||
|
|
|
||
|
|
try:
|
||
|
|
results = tester.run_all_tests()
|
||
|
|
report = tester.generate_report()
|
||
|
|
|
||
|
|
print(report)
|
||
|
|
|
||
|
|
# Return appropriate exit code
|
||
|
|
failed_count = sum(1 for _, success, _, _ in results if not success)
|
||
|
|
return 0 if failed_count == 0 else 1
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Test execution failed: {str(e)}")
|
||
|
|
traceback.print_exc()
|
||
|
|
return 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
exit_code = main()
|
||
|
|
sys.exit(exit_code)
|