test-helpers.ts•20.9 kB
/**
* E2E Test Helpers for GEPA System
*
* This module provides utility functions and helpers for:
* - MCP tool execution and validation
* - Memory management operations
* - Component recovery procedures
* - Data corruption detection and repair
* - Performance monitoring and metrics
* - Test scenario execution orchestration
*/
import type { ScenarioResult } from './test-scenarios';
// Global gc function is already declared in @types/node
/**
* Test Helper Configuration
*/
interface TestHelperConfig {
tempDir: string;
timeout: number;
retryOptions: {
maxRetries: number;
baseDelay: number;
};
}
/**
* MCP Tool Call Result
*/
interface MCPToolResult {
success: boolean;
content: Array<{ type: string; text: string }>;
isError?: boolean;
executionTime: number;
metadata?: Record<string, unknown>;
}
/**
* Memory Statistics
*/
interface MemoryStats {
totalTrajectories: number;
totalCandidates: number;
usedSpace: number;
availableSpace: number;
lastUpdateTime: Date;
optimizationCount: number;
fragmentationLevel: number;
}
/**
* Component Recovery Result
*/
interface ComponentRecoveryResult {
success: boolean;
componentName: string;
recoveryMethod: string;
recoveryTime: number;
dataLoss: boolean;
errorDetails?: string;
}
/**
* Cross-System Sync Result
*/
interface CrossSystemSyncResult {
success: boolean;
synchronizedEntities: number;
conflictsResolved: number;
syncTime: number;
syncErrors: string[];
}
/**
* Data Recovery Result
*/
interface DataRecoveryResult {
success: boolean;
recoveredEntities: number;
corruptedEntities: number;
recoveryMethod: string;
dataIntegrityScore: number;
}
/**
* Corruption Detection Result
*/
interface CorruptionDetectionResult {
corruptionDetected: boolean;
corruptedEntities: string[];
corruptionTypes: string[];
severityLevel: 'low' | 'medium' | 'high' | 'critical';
recommendedActions: string[];
}
/**
* Main Test Helpers Class
*/
export class E2ETestHelpers {
private config: TestHelperConfig;
private mcpCallHistory: MCPToolResult[] = [];
private performanceMetrics: Map<string, number[]> = new Map();
constructor(config: TestHelperConfig) {
this.config = config;
}
/**
* Execute MCP tool call with proper error handling and retries
*/
async callMCPTool(toolName: string, params: Record<string, unknown>): Promise<MCPToolResult> {
const startTime = Date.now();
let lastError: Error | null = null;
for (let attempt = 0; attempt < this.config.retryOptions.maxRetries; attempt++) {
try {
const result = await this.executeMCPToolCall(toolName, params);
const executionTime = Date.now() - startTime;
const toolResult: MCPToolResult = {
success: !result.isError,
content: result.content || [{ type: 'text', text: 'No content returned' }],
isError: result.isError || false,
executionTime,
metadata: {
attempt: attempt + 1,
toolName,
params,
},
};
this.mcpCallHistory.push(toolResult);
this.recordPerformanceMetric(`mcp_${toolName}`, executionTime);
return toolResult;
} catch (error) {
lastError = error instanceof Error ? error : new Error('Unknown error');
if (attempt < this.config.retryOptions.maxRetries - 1) {
const delay = this.config.retryOptions.baseDelay * Math.pow(2, attempt);
await this.sleep(delay);
}
}
}
// All retries failed
const executionTime = Date.now() - startTime;
const failureResult: MCPToolResult = {
success: false,
content: [{ type: 'text', text: `Failed to execute ${toolName}: ${lastError?.message}` }],
isError: true,
executionTime,
metadata: {
attempts: this.config.retryOptions.maxRetries,
finalError: lastError?.message,
},
};
this.mcpCallHistory.push(failureResult);
return failureResult;
}
/**
* Execute a test scenario with comprehensive monitoring
*/
async executeScenario(scenarioPromise: Promise<ScenarioResult>): Promise<ScenarioResult> {
const startTime = Date.now();
const initialMemory = process.memoryUsage();
try {
// Set up timeout
const timeoutPromise = this.createTimeoutPromise(this.config.timeout);
// Execute scenario with timeout
const result = await Promise.race([scenarioPromise, timeoutPromise]);
// Measure resource usage
const finalMemory = process.memoryUsage();
const executionTime = Date.now() - startTime;
// Add monitoring metadata
result.metadata = {
...result.metadata,
actualExecutionTime: executionTime,
memoryDelta: {
heapUsed: finalMemory.heapUsed - initialMemory.heapUsed,
heapTotal: finalMemory.heapTotal - initialMemory.heapTotal,
external: finalMemory.external - initialMemory.external,
},
gcSuggestedAfter: finalMemory.heapUsed > 100 * 1024 * 1024, // 100MB threshold
};
this.recordPerformanceMetric('scenario_execution', executionTime);
return result;
} catch (error) {
return {
success: false,
completed: false,
executionTime: Date.now() - startTime,
metadata: {
error: error instanceof Error ? error.message : 'Scenario execution failed',
timeout: error instanceof Error && error.message.includes('timeout'),
},
};
}
}
/**
* Get comprehensive memory statistics
*/
async getMemoryStats(): Promise<MemoryStats> {
try {
// Simulate memory statistics gathering
// In a real implementation, this would query actual storage systems
const currentTime = new Date();
return {
totalTrajectories: Math.floor(Math.random() * 1000) + 100,
totalCandidates: Math.floor(Math.random() * 500) + 50,
usedSpace: Math.floor(Math.random() * 1024 * 1024 * 100), // Up to 100MB
availableSpace: 1024 * 1024 * 500, // 500MB available
lastUpdateTime: currentTime,
optimizationCount: Math.floor(Math.random() * 10),
fragmentationLevel: Math.random() * 0.3, // 0-30% fragmentation
};
} catch (error) {
throw new Error(`Failed to get memory statistics: ${error}`);
}
}
/**
* Trigger memory optimization process
*/
async triggerMemoryOptimization(): Promise<{ success: boolean; spaceSaved: number }> {
const startTime = Date.now();
try {
// Simulate memory optimization
await this.sleep(Math.random() * 1000 + 500); // 500-1500ms
const spaceSaved = Math.floor(Math.random() * 50 * 1024 * 1024); // Up to 50MB saved
this.recordPerformanceMetric('memory_optimization', Date.now() - startTime);
return {
success: true,
spaceSaved,
};
} catch (error) {
throw new Error(`Memory optimization failed: ${error}`);
}
}
/**
* Perform cross-system memory synchronization
*/
async performCrossSystemSync(): Promise<CrossSystemSyncResult> {
const startTime = Date.now();
try {
// Simulate cross-system synchronization
await this.sleep(Math.random() * 2000 + 1000); // 1-3 seconds
const synchronizedEntities = Math.floor(Math.random() * 100) + 10;
const conflictsResolved = Math.floor(Math.random() * 5);
const syncErrors: string[] = [];
// Randomly add some sync errors for testing
if (Math.random() < 0.2) { // 20% chance of errors
syncErrors.push('Transient network timeout during entity sync');
}
if (Math.random() < 0.1) { // 10% chance of errors
syncErrors.push('Version conflict resolved automatically');
}
const result: CrossSystemSyncResult = {
success: syncErrors.length === 0,
synchronizedEntities,
conflictsResolved,
syncTime: Date.now() - startTime,
syncErrors,
};
this.recordPerformanceMetric('cross_system_sync', result.syncTime);
return result;
} catch (error) {
throw new Error(`Cross-system sync failed: ${error}`);
}
}
/**
* Attempt memory recovery after corruption or errors
*/
async attemptMemoryRecovery(): Promise<DataRecoveryResult> {
const startTime = Date.now();
try {
// Simulate memory recovery process
await this.sleep(Math.random() * 3000 + 2000); // 2-5 seconds
const recoveredEntities = Math.floor(Math.random() * 50) + 10;
const corruptedEntities = Math.floor(Math.random() * 5);
const dataIntegrityScore = Math.random() * 0.3 + 0.7; // 0.7-1.0
const recoveryMethods = [
'Backup restoration',
'Integrity checking and repair',
'Redundant data reconstruction',
'Partial recovery with validation',
];
const result: DataRecoveryResult = {
success: dataIntegrityScore > 0.8,
recoveredEntities,
corruptedEntities,
recoveryMethod: recoveryMethods[Math.floor(Math.random() * recoveryMethods.length)] ?? 'Unknown recovery method',
dataIntegrityScore,
};
this.recordPerformanceMetric('memory_recovery', Date.now() - startTime);
return result;
} catch (error) {
throw new Error(`Memory recovery failed: ${error}`);
}
}
/**
* Attempt component recovery after failure
*/
async attemptComponentRecovery(componentName: string): Promise<ComponentRecoveryResult> {
const startTime = Date.now();
try {
// Simulate component recovery
await this.sleep(Math.random() * 2000 + 1000); // 1-3 seconds
const recoveryMethods = [
'Service restart',
'Configuration reload',
'Connection re-establishment',
'Fallback activation',
'Cache invalidation and rebuild',
];
const recoveryMethod = recoveryMethods[Math.floor(Math.random() * recoveryMethods.length)];
const success = Math.random() > 0.1; // 90% success rate
const dataLoss = Math.random() < 0.05; // 5% chance of data loss
const result: ComponentRecoveryResult = {
success,
componentName,
recoveryMethod: recoveryMethod ?? 'Unknown recovery method',
recoveryTime: Date.now() - startTime,
dataLoss,
...(success ? {} : { errorDetails: 'Component failed to respond after recovery attempt' }),
};
this.recordPerformanceMetric('component_recovery', result.recoveryTime);
return result;
} catch (error) {
throw new Error(`Component recovery failed for ${componentName}: ${error}`);
}
}
/**
* Run corruption detection on stored data
*/
async runCorruptionDetection(): Promise<CorruptionDetectionResult> {
const startTime = Date.now();
try {
// Simulate corruption detection
await this.sleep(Math.random() * 1500 + 500); // 500-2000ms
const corruptionDetected = Math.random() < 0.3; // 30% chance for testing
if (!corruptionDetected) {
return {
corruptionDetected: false,
corruptedEntities: [],
corruptionTypes: [],
severityLevel: 'low',
recommendedActions: ['Continue normal operations'],
};
}
const corruptionTypes = [
'Invalid data format',
'Missing required fields',
'Timestamp inconsistencies',
'Reference integrity violations',
'Checksum mismatches',
];
const detectedTypes = corruptionTypes
.filter(() => Math.random() < 0.4)
.slice(0, Math.floor(Math.random() * 3) + 1);
const corruptedEntityCount = Math.floor(Math.random() * 10) + 1;
const corruptedEntities = Array.from({ length: corruptedEntityCount }, (_, i) =>
`entity-${i}-${Date.now()}`
);
const severityLevels: Array<'low' | 'medium' | 'high' | 'critical'> = ['low', 'medium', 'high', 'critical'];
const severityLevel = severityLevels[Math.min(Math.floor(detectedTypes.length), 3)];
const recommendedActions = [
'Run data recovery procedures',
'Validate backup integrity',
'Perform incremental repair',
'Schedule full system verification',
].slice(0, detectedTypes.length);
const result: CorruptionDetectionResult = {
corruptionDetected,
corruptedEntities,
corruptionTypes: detectedTypes,
severityLevel: severityLevel ?? 'low',
recommendedActions,
};
this.recordPerformanceMetric('corruption_detection', Date.now() - startTime);
return result;
} catch (error) {
throw new Error(`Corruption detection failed: ${error}`);
}
}
/**
* Attempt data recovery after corruption detection
*/
async attemptDataRecovery(): Promise<DataRecoveryResult> {
return this.attemptMemoryRecovery(); // Reuse memory recovery logic
}
/**
* Get performance metrics summary
*/
getPerformanceMetrics(): Record<string, { average: number; min: number; max: number; count: number }> {
const summary: Record<string, { average: number; min: number; max: number; count: number }> = {};
for (const [operation, times] of this.performanceMetrics.entries()) {
if (times.length > 0) {
summary[operation] = {
average: times.reduce((sum, time) => sum + time, 0) / times.length,
min: Math.min(...times),
max: Math.max(...times),
count: times.length,
};
}
}
return summary;
}
/**
* Get MCP call history for analysis
*/
getMCPCallHistory(): MCPToolResult[] {
return [...this.mcpCallHistory];
}
/**
* Reset test state for new test runs
*/
reset(): void {
this.mcpCallHistory = [];
this.performanceMetrics.clear();
}
/**
* Validate test environment is ready
*/
async validateEnvironment(): Promise<{ valid: boolean; issues: string[] }> {
const issues: string[] = [];
try {
// Check directory access
const { access, constants } = await import('fs/promises');
await access(this.config.tempDir, constants.R_OK | constants.W_OK);
} catch (error) {
issues.push(`Temp directory not accessible: ${this.config.tempDir}`);
}
// Check memory availability
const memUsage = process.memoryUsage();
if (memUsage.heapUsed > 500 * 1024 * 1024) { // 500MB threshold
issues.push('High memory usage detected before test start');
}
// Check timeout configuration
if (this.config.timeout < 5000) {
issues.push('Timeout too low for E2E tests (minimum 5 seconds recommended)');
}
return {
valid: issues.length === 0,
issues,
};
}
/**
* Generate test execution report
*/
generateExecutionReport(): {
totalCalls: number;
successRate: number;
averageExecutionTime: number;
performanceMetrics: Record<string, any>;
errorSummary: string[];
} {
const totalCalls = this.mcpCallHistory.length;
const successfulCalls = this.mcpCallHistory.filter(call => call.success).length;
const successRate = totalCalls > 0 ? successfulCalls / totalCalls : 0;
const executionTimes = this.mcpCallHistory.map(call => call.executionTime);
const averageExecutionTime = executionTimes.length > 0
? executionTimes.reduce((sum, time) => sum + time, 0) / executionTimes.length
: 0;
const errorSummary = this.mcpCallHistory
.filter(call => !call.success)
.map(call => call.content[0]?.text || 'Unknown error');
return {
totalCalls,
successRate,
averageExecutionTime,
performanceMetrics: this.getPerformanceMetrics(),
errorSummary,
};
}
// Private helper methods
/**
* Execute actual MCP tool call
*/
private async executeMCPToolCall(
toolName: string,
params: Record<string, unknown>
): Promise<{ content: Array<{ type: string; text: string }>; isError?: boolean }> {
// In a real implementation, this would make actual MCP calls
// For testing, we simulate the calls with realistic responses
await this.sleep(Math.random() * 500 + 100); // 100-600ms simulation
// Simulate random failures for error handling testing
if (Math.random() < 0.05) { // 5% failure rate
throw new Error(`Simulated MCP tool failure for ${toolName}`);
}
// Generate realistic mock response based on tool name
return this.generateMockMCPResponse(toolName, params);
}
/**
* Generate mock MCP response for testing
*/
private generateMockMCPResponse(
toolName: string,
params: Record<string, unknown>
): { content: Array<{ type: string; text: string }>; isError?: boolean } {
switch (toolName) {
case 'gepa_start_evolution':
return {
content: [{
type: 'text',
text: `# Evolution Process Started
## Evolution Details
- **Evolution ID**: mock-evolution-${Date.now()}
- **Task**: ${params.taskDescription || 'Mock task'}
- **Population Size**: ${(params.config as any)?.populationSize || 20}
Evolution process initialized successfully.`
}],
};
case 'gepa_record_trajectory':
return {
content: [{
type: 'text',
text: `# Trajectory Recorded Successfully
## Trajectory Details
- **Trajectory ID**: mock-trajectory-${Date.now()}
- **Prompt ID**: ${params.promptId}
- **Task ID**: ${params.taskId}
- **Success**: ✅
- **Performance Score**: ${Math.random().toFixed(3)}`
}],
};
case 'gepa_evaluate_prompt':
return {
content: [{
type: 'text',
text: `# Prompt Evaluation Complete
## Evaluation Details
- **Prompt ID**: ${params.promptId}
- **Tasks Evaluated**: ${Array.isArray(params.taskIds) ? params.taskIds.length : 1}
- **Success Rate**: ${(Math.random() * 0.3 + 0.7).toFixed(1)}%
- **Average Score**: ${(Math.random() * 0.5 + 0.5).toFixed(3)}`
}],
};
case 'gepa_reflect':
return {
content: [{
type: 'text',
text: `# Reflection Analysis Complete
## Analysis Details
- **Trajectories Analyzed**: ${Array.isArray(params.trajectoryIds) ? params.trajectoryIds.length : 1}
- **Patterns Detected**: ${Math.floor(Math.random() * 5) + 1}
- **Confidence**: ${(Math.random() * 0.3 + 0.7).toFixed(1)}%`
}],
};
case 'gepa_get_pareto_frontier':
return {
content: [{
type: 'text',
text: `# Pareto Frontier Results
## Query Results
- **Total Candidates**: ${Math.floor(Math.random() * 50) + 10}
- **Returned**: ${Math.min(params.limit as number || 10, 25)}
- **Average Fitness**: ${(Math.random() * 0.5 + 0.5).toFixed(3)}`
}],
};
case 'gepa_select_optimal':
return {
content: [{
type: 'text',
text: `# Optimal Candidate Selected
## Selected Candidate
- **Candidate ID**: mock-optimal-candidate-${Date.now()}
- **Fitness Score**: ${(Math.random() * 0.3 + 0.7).toFixed(3)}
- **Performance Weight**: ${params.performanceWeight || 0.7}
- **Diversity Weight**: ${params.diversityWeight || 0.3}`
}],
};
default:
return {
content: [{
type: 'text',
text: `Mock response for ${toolName} with parameters: ${JSON.stringify(params, null, 2)}`
}],
};
}
}
/**
* Create timeout promise for scenario execution
*/
private createTimeoutPromise(timeout: number): Promise<ScenarioResult> {
return new Promise((_, reject) => {
setTimeout(() => {
reject(new Error(`Scenario execution timeout after ${timeout}ms`));
}, timeout);
});
}
/**
* Record performance metric
*/
private recordPerformanceMetric(operation: string, time: number): void {
if (!this.performanceMetrics.has(operation)) {
this.performanceMetrics.set(operation, []);
}
const metrics = this.performanceMetrics.get(operation);
if (metrics) {
metrics.push(time);
}
}
/**
* Sleep utility for timing control
*/
private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
}
// Note: E2ETestHelpers is already exported as a class above
// Additional exports for convenience
export type {
MCPToolResult,
MemoryStats,
ComponentRecoveryResult,
CrossSystemSyncResult,
DataRecoveryResult,
CorruptionDetectionResult
};