memory-leak-benchmarks.tsā¢19.6 kB
/**
* Memory Leak Detection Performance Benchmarks
*
* Comprehensive benchmarks to validate the performance impact
* and effectiveness of memory leak detection across GEPA components.
*/
import { performance } from 'perf_hooks';
import { MemoryLeakDetector, MemoryLeakIntegration } from './memory-leak-detector';
import { CacheManager } from './cache/cache-manager';
import { ParetoFrontier } from './pareto-frontier';
import { LLMAdapter } from '../services/llm-adapter';
import { PromptCandidate } from '../types/gepa';
/**
* Benchmark result interface
*/
interface BenchmarkResult {
name: string;
duration: number;
memoryUsageBefore: NodeJS.MemoryUsage;
memoryUsageAfter: NodeJS.MemoryUsage;
operationsPerSecond: number;
detectionsFound: number;
autoFixesApplied: number;
memoryLeaksPrevented: number;
}
/**
* Memory Leak Detection Benchmark Suite
*/
export class MemoryLeakBenchmarks {
private detector: MemoryLeakDetector;
private results: BenchmarkResult[] = [];
constructor() {
this.detector = MemoryLeakIntegration.initialize({
heapGrowthRate: 5,
maxObjectCount: 1000,
maxHeapSize: 100,
monitoringWindow: 10000,
});
}
/**
* Run all benchmarks
*/
async runAllBenchmarks(): Promise<BenchmarkResult[]> {
// eslint-disable-next-line no-console
console.log('š¬ Starting Memory Leak Detection Benchmarks...\n');
// Core detection benchmarks
await this.benchmarkBasicDetection();
await this.benchmarkCacheIntegration();
await this.benchmarkParetoIntegration();
await this.benchmarkLLMIntegration();
// Performance benchmarks
await this.benchmarkHighLoadDetection();
await this.benchmarkConcurrentDetection();
await this.benchmarkMemoryPressure();
// Real-world scenario benchmarks
await this.benchmarkLongRunningSession();
await this.benchmarkAutoFixEffectiveness();
// eslint-disable-next-line no-console
console.log('\nš Benchmark Suite Complete!');
this.printSummary();
return this.results;
}
/**
* Benchmark basic memory leak detection
*/
private async benchmarkBasicDetection(): Promise<void> {
const name = 'Basic Memory Leak Detection';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
let detections = 0;
const iterations = 1000;
// Register test component
const tracker = this.detector.registerComponent('benchmark-basic', {
maxObjectCount: 100,
});
// Allocate objects to trigger detection
for (let i = 0; i < iterations; i++) {
this.detector.trackObjectAllocation('benchmark-basic', { id: i });
if (i % 100 === 0) {
const found = await this.detector.detectMemoryLeaks();
detections += found.length;
}
}
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: iterations / ((endTime - startTime) / 1000),
detectionsFound: detections,
autoFixesApplied: 0,
memoryLeaksPrevented: Math.max(0, iterations - tracker.objectCount),
});
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Benchmark cache manager integration
*/
private async benchmarkCacheIntegration(): Promise<void> {
const name = 'Cache Manager Integration';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const cacheManager = new CacheManager({
l1MaxSize: 1024 * 1024,
l1MaxEntries: 500,
l2Enabled: false,
});
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
let detections = 0;
const iterations = 500;
// Perform cache operations
for (let i = 0; i < iterations; i++) {
await cacheManager.set(`bench-cache-${i}`, {
data: `cache-data-${i}`.repeat(10),
timestamp: Date.now(),
});
if (i % 50 === 0) {
const found = await this.detector.detectMemoryLeaks();
detections += found.length;
}
}
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: iterations / ((endTime - startTime) / 1000),
detectionsFound: detections,
autoFixesApplied: 0,
memoryLeaksPrevented: 0,
});
await cacheManager.shutdown();
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Benchmark Pareto frontier integration
*/
private async benchmarkParetoIntegration(): Promise<void> {
const name = 'Pareto Frontier Integration';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const paretoFrontier = new ParetoFrontier({
objectives: [
{
name: 'score',
weight: 1,
direction: 'maximize',
extractor: (candidate: PromptCandidate) => candidate.averageScore,
},
],
maxSize: 100,
});
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
let detections = 0;
const iterations = 300;
// Add candidates to frontier
for (let i = 0; i < iterations; i++) {
const candidate: PromptCandidate = {
id: `bench-pareto-${i}`,
generation: 1,
content: `Benchmark candidate ${i} with longer content for memory testing`,
averageScore: Math.random(),
rolloutCount: i + 1,
taskPerformance: new Map<string, number>(),
createdAt: new Date(),
lastEvaluated: new Date(),
};
await paretoFrontier.addCandidate(candidate);
if (i % 30 === 0) {
const found = await this.detector.detectMemoryLeaks();
detections += found.length;
}
}
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: iterations / ((endTime - startTime) / 1000),
detectionsFound: detections,
autoFixesApplied: 0,
memoryLeaksPrevented: Math.max(0, iterations - paretoFrontier.size()),
});
paretoFrontier.clear();
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Benchmark LLM adapter integration
*/
private async benchmarkLLMIntegration(): Promise<void> {
const name = 'LLM Adapter Integration';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const llmAdapter = new LLMAdapter({
maxConcurrentProcesses: 2,
processTimeout: 1000,
executable: 'echo', // Use echo for testing
});
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
let detections = 0;
const iterations = 50; // Lower for process-based operations
// Simulate LLM calls
const promises = Array.from({ length: iterations }, async (_, i) => {
try {
await llmAdapter.generateResponse(`benchmark prompt ${i}`);
} catch {
// Ignore errors for benchmark
}
if (i % 10 === 0) {
const found = await this.detector.detectMemoryLeaks();
detections += found.length;
}
});
await Promise.allSettled(promises);
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: iterations / ((endTime - startTime) / 1000),
detectionsFound: detections,
autoFixesApplied: 0,
memoryLeaksPrevented: 0,
});
llmAdapter.shutdown();
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Benchmark high load detection performance
*/
private async benchmarkHighLoadDetection(): Promise<void> {
const name = 'High Load Detection Performance';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
let detections = 0;
const iterations = 5000; // High load
// Register multiple components
for (let comp = 0; comp < 5; comp++) {
this.detector.registerComponent(`high-load-${comp}`, {
maxObjectCount: 200,
});
}
// High frequency allocations
for (let i = 0; i < iterations; i++) {
const componentName = `high-load-${i % 5}`;
this.detector.trackObjectAllocation(componentName, { id: i });
if (i % 500 === 0) {
const found = await this.detector.detectMemoryLeaks();
detections += found.length;
}
}
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: iterations / ((endTime - startTime) / 1000),
detectionsFound: detections,
autoFixesApplied: 0,
memoryLeaksPrevented: 0,
});
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Benchmark concurrent detection
*/
private async benchmarkConcurrentDetection(): Promise<void> {
const name = 'Concurrent Detection Performance';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
let totalDetections = 0;
const concurrency = 10;
const iterationsPerWorker = 100;
// Concurrent workers
const workers = Array.from({ length: concurrency }, async (_, workerId) => {
const componentName = `concurrent-${workerId}`;
this.detector.registerComponent(componentName, {
maxObjectCount: 50,
});
let detections = 0;
for (let i = 0; i < iterationsPerWorker; i++) {
this.detector.trackObjectAllocation(componentName, { id: `${workerId}-${i}` });
if (i % 20 === 0) {
const found = await this.detector.detectMemoryLeaks();
detections += found.length;
}
}
return detections;
});
const results = await Promise.all(workers);
totalDetections = results.reduce((sum, count) => sum + count, 0);
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: (concurrency * iterationsPerWorker) / ((endTime - startTime) / 1000),
detectionsFound: totalDetections,
autoFixesApplied: 0,
memoryLeaksPrevented: 0,
});
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Benchmark memory pressure simulation
*/
private async benchmarkMemoryPressure(): Promise<void> {
const name = 'Memory Pressure Simulation';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
// Simulate memory pressure
await this.detector.simulateMemoryPressure({
enabled: true,
targetMemoryMB: 20,
duration: 2000, // 2 seconds
escalationSteps: 4,
});
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: 1000 / ((endTime - startTime) / 1000), // Arbitrary for pressure test
detectionsFound: 0,
autoFixesApplied: 0,
memoryLeaksPrevented: 0,
});
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Benchmark long-running session
*/
private async benchmarkLongRunningSession(): Promise<void> {
const name = 'Long-Running Session Simulation';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
let detections = 0;
const sessionDuration = 3000; // 3 seconds
const operationInterval = 50; // 50ms
this.detector.registerComponent('long-session', {
maxObjectCount: 100,
});
let operationCount = 0;
const sessionStart = Date.now();
while (Date.now() - sessionStart < sessionDuration) {
// Simulate various operations
this.detector.trackObjectAllocation('long-session', {
id: operationCount++,
timestamp: Date.now(),
});
// Periodic cleanup
if (operationCount % 50 === 0) {
await this.detector.forceCleanup();
}
// Periodic detection
if (operationCount % 25 === 0) {
const found = await this.detector.detectMemoryLeaks();
detections += found.length;
}
await new Promise(resolve => setTimeout(resolve, operationInterval));
}
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: operationCount / ((endTime - startTime) / 1000),
detectionsFound: detections,
autoFixesApplied: 0,
memoryLeaksPrevented: 0,
});
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Benchmark auto-fix effectiveness
*/
private async benchmarkAutoFixEffectiveness(): Promise<void> {
const name = 'Auto-Fix Effectiveness';
// eslint-disable-next-line no-console
console.log(`š Running: ${name}`);
const memoryBefore = process.memoryUsage();
const startTime = performance.now();
let autoFixesApplied = 0;
this.detector.on('autoFixApplied', () => autoFixesApplied++);
// Create conditions that trigger auto-fixes
this.detector.registerComponent('autofix-test', {
maxObjectCount: 10, // Low threshold
});
// Trigger auto-fixes
for (let i = 0; i < 50; i++) {
this.detector.trackObjectAllocation('autofix-test', { id: i });
if (i % 5 === 0) {
await this.detector.detectMemoryLeaks();
}
}
// Force cleanup to trigger more auto-fixes
await this.detector.forceCleanup();
const endTime = performance.now();
const memoryAfter = process.memoryUsage();
this.results.push({
name,
duration: endTime - startTime,
memoryUsageBefore: memoryBefore,
memoryUsageAfter: memoryAfter,
operationsPerSecond: 50 / ((endTime - startTime) / 1000),
detectionsFound: 0,
autoFixesApplied,
memoryLeaksPrevented: autoFixesApplied,
});
// eslint-disable-next-line no-console
console.log(`ā
${name} completed`);
}
/**
* Print benchmark summary
*/
private printSummary(): void {
// eslint-disable-next-line no-console
console.log('\nš BENCHMARK SUMMARY');
// eslint-disable-next-line no-console
console.log('='.repeat(80));
let totalOperations = 0;
let totalDetections = 0;
let totalAutoFixes = 0;
let totalDuration = 0;
// eslint-disable-next-line no-console
console.log('\nš Individual Results:');
this.results.forEach(result => {
const memoryDelta = result.memoryUsageAfter.heapUsed - result.memoryUsageBefore.heapUsed;
// eslint-disable-next-line no-console
console.log(`\nš ${result.name}`);
// eslint-disable-next-line no-console
console.log(` ā±ļø Duration: ${result.duration.toFixed(2)}ms`);
// eslint-disable-next-line no-console
console.log(` š Ops/sec: ${result.operationsPerSecond.toFixed(2)}`);
// eslint-disable-next-line no-console
console.log(` š Detections: ${result.detectionsFound}`);
// eslint-disable-next-line no-console
console.log(` š§ Auto-fixes: ${result.autoFixesApplied}`);
// eslint-disable-next-line no-console
console.log(` š¾ Memory Ī: ${(memoryDelta / 1024 / 1024).toFixed(2)}MB`);
// eslint-disable-next-line no-console
console.log(` š”ļø Prevented: ${result.memoryLeaksPrevented}`);
totalOperations += result.operationsPerSecond * (result.duration / 1000);
totalDetections += result.detectionsFound;
totalAutoFixes += result.autoFixesApplied;
totalDuration += result.duration;
});
// eslint-disable-next-line no-console
console.log('\nšÆ Overall Performance:');
// eslint-disable-next-line no-console
console.log(` š Total benchmarks: ${this.results.length}`);
// eslint-disable-next-line no-console
console.log(` ā±ļø Total duration: ${totalDuration.toFixed(2)}ms`);
// eslint-disable-next-line no-console
console.log(` š Avg ops/sec: ${(totalOperations / this.results.length).toFixed(2)}`);
// eslint-disable-next-line no-console
console.log(` š Total detections: ${totalDetections}`);
// eslint-disable-next-line no-console
console.log(` š§ Total auto-fixes: ${totalAutoFixes}`);
// eslint-disable-next-line no-console
console.log(` ā
Detection rate: ${((totalDetections / this.results.length) * 100).toFixed(1)}%`);
// Performance scoring
const avgOpsPerSec = totalOperations / this.results.length;
const avgDuration = totalDuration / this.results.length;
let performanceGrade = 'F';
if (avgOpsPerSec > 1000 && avgDuration < 1000) performanceGrade = 'A';
else if (avgOpsPerSec > 500 && avgDuration < 2000) performanceGrade = 'B';
else if (avgOpsPerSec > 100 && avgDuration < 5000) performanceGrade = 'C';
else if (avgOpsPerSec > 50) performanceGrade = 'D';
// eslint-disable-next-line no-console
console.log(` š Performance Grade: ${performanceGrade}`);
// eslint-disable-next-line no-console
console.log('\n' + '='.repeat(80));
}
/**
* Cleanup resources
*/
async cleanup(): Promise<void> {
this.detector.shutdown();
MemoryLeakIntegration.shutdown();
}
}
/**
* Export function to run benchmarks from CLI
*/
export async function runMemoryLeakBenchmarks(): Promise<BenchmarkResult[]> {
const benchmarks = new MemoryLeakBenchmarks();
try {
const results = await benchmarks.runAllBenchmarks();
return results;
} finally {
await benchmarks.cleanup();
}
}
// Run benchmarks if called directly
if (require.main === module) {
runMemoryLeakBenchmarks().then(() => {
// eslint-disable-next-line no-console
console.log('\nš Benchmarks completed successfully!');
process.exit(0);
}).catch((error) => {
// eslint-disable-next-line no-console
console.error('\nā Benchmark failed:', error);
process.exit(1);
});
}