#!/usr/bin/env python3 """ Performance and reliability testing script. Tests various input sizes, edge cases, memory usage, and error recovery. """ import asyncio import gc import json import logging import os import sys import tempfile import time import traceback from pathlib import Path from typing import Dict, Any, List, Tuple, Optional # Try to import psutil, fallback to basic memory tracking try: import psutil HAS_PSUTIL = True except ImportError: HAS_PSUTIL = False psutil = None # Configure logging logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" ) logger = logging.getLogger("PerformanceReliability") class PerformanceReliabilityTest: """Test performance and reliability of core components""" def __init__(self): self.test_results: List[Tuple[str, bool, str, Dict[str, Any]]] = [] if HAS_PSUTIL: self.process = psutil.Process() else: self.process = None def get_memory_usage(self) -> float: """Get current memory usage in MB""" if HAS_PSUTIL and self.process: return self.process.memory_info().rss / 1024 / 1024 else: # Fallback to basic memory tracking return 0.0 # Can't measure without psutil def test_dependency_manager_performance(self) -> Tuple[str, bool, str, Dict[str, Any]]: """Test dependency manager performance with multiple calls""" test_name = "Dependency Manager Performance Test" metrics = {} try: import dependency_manager # Measure initial memory initial_memory = self.get_memory_usage() # Test multiple instantiations start_time = time.time() managers = [] for i in range(100): manager = dependency_manager.get_dependency_manager() managers.append(manager) creation_time = time.time() - start_time # Test status report generation performance start_time = time.time() for manager in managers[:10]: # Test first 10 status = manager.get_dependency_status_report() report_time = time.time() - start_time # Measure final memory final_memory = self.get_memory_usage() memory_increase = final_memory - initial_memory if HAS_PSUTIL else 0.0 # Cleanup del managers gc.collect() metrics = { "creation_time_100_instances": f"{creation_time:.3f}s", "report_generation_time_10_calls": f"{report_time:.3f}s", "memory_increase": f"{memory_increase:.2f}MB" if HAS_PSUTIL else "N/A (psutil not available)", "avg_creation_time": f"{creation_time/100*1000:.2f}ms" } # Performance thresholds if creation_time > 5.0: # 5 seconds for 100 instances return test_name, False, "Dependency manager creation too slow", metrics if HAS_PSUTIL and memory_increase > 50: # 50MB increase return test_name, False, "Excessive memory usage", metrics return test_name, True, "Dependency manager performance acceptable", metrics except Exception as e: return test_name, False, f"Performance test failed: {str(e)}", metrics def test_error_handling_reliability(self) -> Tuple[str, bool, str, Dict[str, Any]]: """Test error handling reliability with various error conditions""" test_name = "Error Handling Reliability Test" metrics = {} try: import error_handling import logging # Create error handler logger = logging.getLogger("test_reliability") error_handler = error_handling.ErrorHandler(logger) # Test various error scenarios test_errors = [ Exception("Generic error"), ValueError("Value error"), TypeError("Type error"), RuntimeError("Runtime error"), ConnectionError("Connection error") ] successful_handles = 0 start_time = time.time() for i, error in enumerate(test_errors * 20): # 100 total errors try: result = error_handler.handle_api_error(error, "test_api", f"test_operation_{i}") # ErrorHandler returns a dict with success, message, etc. if isinstance(result, dict) and 'success' in result: successful_handles += 1 elif result is not None: # Any non-None result counts as handled successful_handles += 1 except Exception: pass # Error in error handling - not good but continue handling_time = time.time() - start_time metrics = { "total_errors_processed": len(test_errors) * 20, "successful_handles": successful_handles, "handling_time": f"{handling_time:.3f}s", "avg_handling_time": f"{handling_time/(len(test_errors)*20)*1000:.2f}ms", "success_rate": f"{successful_handles/(len(test_errors)*20)*100:.1f}%" } # Reliability thresholds success_rate = successful_handles / (len(test_errors) * 20) if success_rate < 0.95: # 95% success rate return test_name, False, "Error handling success rate too low", metrics if handling_time > 2.0: # 2 seconds for 100 errors return test_name, False, "Error handling too slow", metrics return test_name, True, "Error handling reliability acceptable", metrics except Exception as e: return test_name, False, f"Reliability test failed: {str(e)}", metrics def test_configuration_edge_cases(self) -> Tuple[str, bool, str, Dict[str, Any]]: """Test configuration handling with edge cases""" test_name = "Configuration Edge Cases Test" metrics = {} try: # Test various configuration scenarios test_configs = [ {}, # Empty config {"invalid": "structure"}, # Invalid structure {"obsidian": {}}, # Partial config {"obsidian": {"vault_path": ""}}, # Empty values {"obsidian": {"vault_path": "/nonexistent/path"}}, # Invalid paths {"claude": {"api_key": ""}}, # Empty API key {"claude": {"api_key": "x" * 1000}}, # Very long API key ] successful_loads = 0 start_time = time.time() for i, config in enumerate(test_configs): try: # Create temporary config file with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: json.dump(config, f) temp_path = f.name # Try to load config (this would normally be done by main.py) with open(temp_path, 'r') as f: loaded_config = json.load(f) # Basic validation that it's a dict if isinstance(loaded_config, dict): successful_loads += 1 # Cleanup os.unlink(temp_path) except Exception: # Expected for some edge cases pass processing_time = time.time() - start_time metrics = { "total_configs_tested": len(test_configs), "successful_loads": successful_loads, "processing_time": f"{processing_time:.3f}s", "avg_processing_time": f"{processing_time/len(test_configs)*1000:.2f}ms" } # Should handle at least basic configs if successful_loads < 3: return test_name, False, "Too many configuration failures", metrics return test_name, True, "Configuration edge cases handled", metrics except Exception as e: return test_name, False, f"Edge case test failed: {str(e)}", metrics def test_memory_usage_patterns(self) -> Tuple[str, bool, str, Dict[str, Any]]: """Test memory usage patterns and potential leaks""" test_name = "Memory Usage Patterns Test" metrics = {} try: # Measure baseline memory gc.collect() baseline_memory = self.get_memory_usage() # Test repeated operations that might cause memory leaks operations_memory = [] for iteration in range(10): # Simulate typical operations temp_data = [] # Create temporary objects for i in range(1000): temp_data.append({ "id": i, "data": "x" * 100, # 100 chars "nested": {"value": i * 2} }) # Process data processed = [item for item in temp_data if item["id"] % 2 == 0] # Measure memory after each iteration current_memory = self.get_memory_usage() operations_memory.append(current_memory) # Cleanup del temp_data, processed gc.collect() # Final memory measurement final_memory = self.get_memory_usage() memory_increase = final_memory - baseline_memory # Calculate memory growth trend if len(operations_memory) > 1: memory_growth = operations_memory[-1] - operations_memory[0] else: memory_growth = 0 metrics = { "baseline_memory": f"{baseline_memory:.2f}MB" if HAS_PSUTIL else "N/A", "final_memory": f"{final_memory:.2f}MB" if HAS_PSUTIL else "N/A", "total_memory_increase": f"{memory_increase:.2f}MB" if HAS_PSUTIL else "N/A", "memory_growth_trend": f"{memory_growth:.2f}MB" if HAS_PSUTIL else "N/A", "max_memory_during_test": f"{max(operations_memory):.2f}MB" if HAS_PSUTIL and operations_memory else "N/A", "iterations_tested": 10 } # Memory usage thresholds (only check if psutil available) if HAS_PSUTIL: if memory_increase > 20: # 20MB increase after cleanup return test_name, False, "Potential memory leak detected", metrics if memory_growth > 15: # 15MB growth trend return test_name, False, "Memory growth trend concerning", metrics return test_name, True, "Memory usage patterns acceptable", metrics except Exception as e: return test_name, False, f"Memory test failed: {str(e)}", metrics def test_graceful_degradation(self) -> Tuple[str, bool, str, Dict[str, Any]]: """Test graceful degradation when dependencies are missing""" test_name = "Graceful Degradation Test" metrics = {} try: import dependency_manager # Test dependency manager with missing modules dep_manager = dependency_manager.get_dependency_manager() # Test getting non-existent modules test_modules = [ 'nonexistent_module', 'fake_dependency', 'missing_package', 'invalid.module.name' ] successful_degradations = 0 start_time = time.time() for module_name in test_modules: try: result = dep_manager.get_module(module_name) # Should return None for missing modules, not crash if result is None: successful_degradations += 1 except Exception: # Should not raise exceptions for missing modules pass degradation_time = time.time() - start_time # Test status report with missing dependencies try: status_report = dep_manager.get_dependency_status_report() status_report_works = isinstance(status_report, str) and len(status_report) > 0 except Exception: status_report_works = False metrics = { "modules_tested": len(test_modules), "successful_degradations": successful_degradations, "degradation_time": f"{degradation_time:.3f}s", "status_report_works": status_report_works, "degradation_rate": f"{successful_degradations/len(test_modules)*100:.1f}%" } # Should gracefully handle all missing modules if successful_degradations < len(test_modules): return test_name, False, "Not all missing modules handled gracefully", metrics if not status_report_works: return test_name, False, "Status report fails with missing dependencies", metrics return test_name, True, "Graceful degradation working correctly", metrics except Exception as e: return test_name, False, f"Degradation test failed: {str(e)}", metrics def test_concurrent_operations(self) -> Tuple[str, bool, str, Dict[str, Any]]: """Test concurrent operations and thread safety""" test_name = "Concurrent Operations Test" metrics = {} try: import dependency_manager import asyncio import concurrent.futures async def async_operation(operation_id: int) -> bool: """Simulate async operation""" try: dep_manager = dependency_manager.get_dependency_manager() status = dep_manager.get_dependency_status_report() missing = dep_manager.get_missing_dependencies() # Simulate some processing time await asyncio.sleep(0.01) return len(status) > 0 and isinstance(missing, list) except Exception: return False # Test concurrent async operations start_time = time.time() async def run_concurrent_test(): tasks = [async_operation(i) for i in range(20)] results = await asyncio.gather(*tasks, return_exceptions=True) return results results = asyncio.run(run_concurrent_test()) concurrent_time = time.time() - start_time successful_operations = sum(1 for r in results if r is True) metrics = { "concurrent_operations": 20, "successful_operations": successful_operations, "concurrent_time": f"{concurrent_time:.3f}s", "avg_operation_time": f"{concurrent_time/20*1000:.2f}ms", "success_rate": f"{successful_operations/20*100:.1f}%" } # Concurrency thresholds if successful_operations < 18: # 90% success rate return test_name, False, "Concurrent operations success rate too low", metrics if concurrent_time > 5.0: # 5 seconds for 20 operations return test_name, False, "Concurrent operations too slow", metrics return test_name, True, "Concurrent operations working correctly", metrics except Exception as e: return test_name, False, f"Concurrency test failed: {str(e)}", metrics def run_all_tests(self) -> List[Tuple[str, bool, str, Dict[str, Any]]]: """Run all performance and reliability tests""" logger.info("🚀 Starting performance and reliability tests...") test_methods = [ self.test_dependency_manager_performance, self.test_error_handling_reliability, self.test_configuration_edge_cases, self.test_memory_usage_patterns, self.test_graceful_degradation, self.test_concurrent_operations ] for test_method in test_methods: try: logger.info(f"Running {test_method.__name__}...") result = test_method() self.test_results.append(result) status = "✅ PASS" if result[1] else "❌ FAIL" logger.info(f"{status} {result[0]}: {result[2]}") except Exception as e: error_result = (test_method.__name__, False, f"Test execution failed: {str(e)}", {}) self.test_results.append(error_result) logger.error(f"❌ FAIL {error_result[0]}: {error_result[2]}") return self.test_results def generate_report(self) -> str: """Generate performance and reliability test report""" total_tests = len(self.test_results) passed_tests = sum(1 for _, success, _, _ in self.test_results if success) failed_tests = total_tests - passed_tests report = f""" {'='*60} PERFORMANCE AND RELIABILITY TEST REPORT {'='*60} Summary: Total Tests: {total_tests} Passed: {passed_tests} Failed: {failed_tests} Success Rate: {(passed_tests/total_tests*100):.1f}% Test Results: """ for test_name, success, message, metrics in self.test_results: status = "✅ PASS" if success else "❌ FAIL" report += f"\n{status} {test_name}: {message}" if metrics: report += "\n Metrics:" for key, value in metrics.items(): report += f"\n - {key}: {value}" report += f"\n\n{'='*60}\n" return report def main(): """Main test function""" tester = PerformanceReliabilityTest() try: results = tester.run_all_tests() report = tester.generate_report() print(report) # Return appropriate exit code failed_count = sum(1 for _, success, _, _ in results if not success) return 0 if failed_count == 0 else 1 except Exception as e: logger.error(f"Test execution failed: {str(e)}") traceback.print_exc() return 1 if __name__ == "__main__": exit_code = main() sys.exit(exit_code)