memory-leak-detector.ts•23.4 kB
/**
* Comprehensive Memory Leak Detection System for GEPA
*
* Provides real-time monitoring, automatic leak detection, and prevention
* mechanisms for all GEPA components including evolution engine, cache system,
* Pareto frontier, and LLM adapter.
*/
import { EventEmitter } from 'events';
import { getHeapSnapshot } from 'v8';
import * as fs from 'fs/promises';
import * as path from 'path';
/**
* Memory leak threshold configuration
*/
export interface MemoryLeakThresholds {
/** Maximum heap growth rate (MB/second) before triggering alert */
heapGrowthRate: number;
/** Maximum heap size (MB) before triggering alert */
maxHeapSize: number;
/** Maximum number of objects per component */
maxObjectCount: number;
/** Memory usage increase percentage threshold */
memoryIncreaseThreshold: number;
/** Time window for monitoring (milliseconds) */
monitoringWindow: number;
/** Minimum time between heap snapshots (milliseconds) */
snapshotInterval: number;
}
/**
* Component-specific memory tracking
*/
export interface ComponentMemoryTracker {
name: string;
objectCount: number;
memoryUsage: number;
lastCleanup: number;
growthRate: number;
weakRefs: Set<any>; // Changed from WeakRef for compatibility
thresholds: Partial<MemoryLeakThresholds>;
}
/**
* Memory leak detection result
*/
export interface MemoryLeakDetection {
timestamp: number;
component: string;
leakType: 'heap_growth' | 'object_accumulation' | 'weak_ref_buildup' | 'cache_overflow' | 'process_leak';
severity: 'low' | 'medium' | 'high' | 'critical';
currentUsage: number;
growthRate: number;
recommendation: string;
autoFixAvailable: boolean;
}
/**
* Heap snapshot comparison result
*/
export interface HeapComparison {
added: number;
removed: number;
sizeIncrement: number;
leakyConstructors: Array<{
name: string;
count: number;
size: number;
}>;
}
/**
* Memory pressure simulation configuration
*/
export interface MemoryPressureConfig {
enabled: boolean;
targetMemoryMB: number;
duration: number;
escalationSteps: number;
}
/**
* Main Memory Leak Detection System
*/
export class MemoryLeakDetector extends EventEmitter {
private isEnabled = true;
private monitoringInterval?: ReturnType<typeof setInterval>;
private cleanupInterval?: ReturnType<typeof setInterval>;
private lastHeapSnapshot?: any;
private snapshotCount = 0;
private readonly thresholds: MemoryLeakThresholds = {
heapGrowthRate: 10, // 10 MB/second
maxHeapSize: 512, // 512 MB
maxObjectCount: 100000,
memoryIncreaseThreshold: 50, // 50%
monitoringWindow: 60000, // 1 minute
snapshotInterval: 30000, // 30 seconds
};
private readonly componentTrackers = new Map<string, ComponentMemoryTracker>();
private readonly detectionHistory: MemoryLeakDetection[] = [];
private readonly memoryHistory: Array<{ timestamp: number; usage: number }> = [];
// Leak detection statistics
private readonly stats = {
totalDetections: 0,
autoFixesApplied: 0,
falsePositives: 0,
preventedLeaks: 0,
};
constructor(thresholds?: Partial<MemoryLeakThresholds>) {
super();
if (thresholds) {
Object.assign(this.thresholds, thresholds);
}
this.initializeComponentTrackers();
this.startMonitoring();
this.setupCleanupInterval();
this.setupProcessHandlers();
}
/**
* Register a component for memory leak monitoring
*/
registerComponent(
name: string,
thresholds?: Partial<MemoryLeakThresholds>
): ComponentMemoryTracker {
const tracker: ComponentMemoryTracker = {
name,
objectCount: 0,
memoryUsage: 0,
lastCleanup: Date.now(),
growthRate: 0,
weakRefs: new Set(),
thresholds: thresholds || {},
};
this.componentTrackers.set(name, tracker);
this.emit('componentRegistered', { name, tracker });
return tracker;
}
/**
* Track object allocation for a component
*/
trackObjectAllocation(componentName: string, object: any, size?: number): void {
const tracker = this.componentTrackers.get(componentName);
if (!tracker) {
// eslint-disable-next-line no-console
console.warn(`Component ${componentName} not registered for memory tracking`);
return;
}
// Track object for lifecycle monitoring (without WeakRef for compatibility)
if (typeof object === 'object' && object !== null) {
try {
// Add object reference directly
tracker.weakRefs.add(object);
} catch (error) {
// eslint-disable-next-line no-console
console.warn(`Failed to track object for ${componentName}:`, error);
}
}
tracker.objectCount++;
if (size) {
tracker.memoryUsage += size;
}
// Check for immediate threshold violations
this.checkComponentThresholds(tracker);
}
/**
* Track object deallocation for a component
*/
trackObjectDeallocation(componentName: string, size?: number): void {
const tracker = this.componentTrackers.get(componentName);
if (!tracker) return;
tracker.objectCount = Math.max(0, tracker.objectCount - 1);
if (size) {
tracker.memoryUsage = Math.max(0, tracker.memoryUsage - size);
}
}
/**
* Create heap snapshot for comparison
*/
async createHeapSnapshot(): Promise<any> {
try {
const snapshot = getHeapSnapshot();
const snapshotPath = path.join(process.cwd(), `.heap-snapshots/snapshot-${Date.now()}-${this.snapshotCount++}.heapsnapshot`);
await fs.mkdir(path.dirname(snapshotPath), { recursive: true });
await fs.writeFile(snapshotPath, snapshot);
return {
path: snapshotPath,
timestamp: Date.now(),
memoryUsage: process.memoryUsage(),
};
} catch (error) {
// eslint-disable-next-line no-console
console.error('Failed to create heap snapshot:', error);
return null;
}
}
/**
* Compare heap snapshots to detect memory leaks
*/
async compareHeapSnapshots(
snapshot1: any,
snapshot2: any
): Promise<HeapComparison> {
try {
// This is a simplified comparison - in practice would use V8 heap profiler
const usage1 = snapshot1.memoryUsage;
const usage2 = snapshot2.memoryUsage;
const sizeIncrement = usage2.heapUsed - usage1.heapUsed;
const added = Math.max(0, sizeIncrement / 1024); // Approximate object count
const removed = 0; // Would need more sophisticated analysis
// Detect potentially leaky constructors (simplified)
const leakyConstructors = this.detectLeakyConstructors(sizeIncrement);
return {
added,
removed,
sizeIncrement,
leakyConstructors,
};
} catch (error) {
// eslint-disable-next-line no-console
console.error('Failed to compare heap snapshots:', error);
return {
added: 0,
removed: 0,
sizeIncrement: 0,
leakyConstructors: [],
};
}
}
/**
* Force garbage collection and cleanup weak references
*/
async forceCleanup(): Promise<{ cleaned: number; memoryFreed: number }> {
const beforeMemory = process.memoryUsage();
let totalCleaned = 0;
// Clean up weak references
for (const [componentName, tracker] of Array.from(this.componentTrackers)) {
const beforeSize = tracker.weakRefs.size;
// Remove dead references
for (const ref of Array.from(tracker.weakRefs)) {
// Since we're not using WeakRef, check if object is still valid
if (!ref || typeof ref !== 'object') {
tracker.weakRefs.delete(ref);
totalCleaned++;
}
}
const cleaned = beforeSize - tracker.weakRefs.size;
if (cleaned > 0) {
this.emit('weakRefsCleanup', { component: componentName, cleaned });
}
}
// Force garbage collection if available
if (global.gc) {
global.gc();
}
const afterMemory = process.memoryUsage();
const memoryFreed = beforeMemory.heapUsed - afterMemory.heapUsed;
this.emit('forceCleanup', { cleaned: totalCleaned, memoryFreed });
return {
cleaned: totalCleaned,
memoryFreed,
};
}
/**
* Detect memory leaks across all components
*/
async detectMemoryLeaks(): Promise<MemoryLeakDetection[]> {
const detections: MemoryLeakDetection[] = [];
const currentMemory = process.memoryUsage();
// Track memory history
this.memoryHistory.push({
timestamp: Date.now(),
usage: currentMemory.heapUsed,
});
// Clean old history
const cutoff = Date.now() - this.thresholds.monitoringWindow;
while (this.memoryHistory.length > 0) {
const oldestEntry = this.memoryHistory[0];
if (oldestEntry?.timestamp && oldestEntry.timestamp < cutoff) {
this.memoryHistory.shift();
} else {
break;
}
}
// 1. Check overall heap growth
const heapGrowthDetection = this.detectHeapGrowth(currentMemory);
if (heapGrowthDetection) {
detections.push(heapGrowthDetection);
}
// 2. Check component-specific leaks
for (const [_componentName, tracker] of Array.from(this.componentTrackers)) {
const componentDetections = this.detectComponentLeaks(tracker);
detections.push(...componentDetections);
}
// 3. Check for process leaks (file handles, event listeners, etc.)
const processLeak = this.detectProcessLeaks();
if (processLeak) {
detections.push(processLeak);
}
// Store detections and apply auto-fixes
for (const detection of detections) {
this.detectionHistory.push(detection);
this.stats.totalDetections++;
if (detection.autoFixAvailable) {
await this.applyAutoFix(detection);
}
this.emit('memoryLeakDetected', detection);
}
return detections;
}
/**
* Simulate memory pressure for testing
*/
async simulateMemoryPressure(config: MemoryPressureConfig): Promise<void> {
if (!config.enabled) return;
const allocations: Buffer[] = [];
const stepSize = Math.floor(config.targetMemoryMB / config.escalationSteps);
const stepDuration = Math.floor(config.duration / config.escalationSteps);
this.emit('memoryPressureStart', config);
for (let step = 0; step < config.escalationSteps; step++) {
// Allocate memory
const buffer = Buffer.alloc(stepSize * 1024 * 1024);
allocations.push(buffer);
this.emit('memoryPressureStep', {
step: step + 1,
totalSteps: config.escalationSteps,
allocatedMB: (step + 1) * stepSize,
currentMemory: process.memoryUsage(),
});
// Wait for step duration
await new Promise(resolve => setTimeout(resolve, stepDuration));
// Check for leaks during pressure
await this.detectMemoryLeaks();
}
// Cleanup allocations
allocations.length = 0;
// Force garbage collection
if (global.gc) {
global.gc();
}
this.emit('memoryPressureEnd', {
finalMemory: process.memoryUsage(),
duration: config.duration,
});
}
/**
* Get memory leak detection statistics
*/
getStatistics() {
const components = Array.from(this.componentTrackers.values()).map(tracker => ({
name: tracker.name,
objectCount: tracker.objectCount,
memoryUsage: tracker.memoryUsage,
growthRate: tracker.growthRate,
weakRefCount: tracker.weakRefs.size,
}));
const recentDetections = this.detectionHistory.slice(-10);
return {
detections: { ...this.stats },
components,
recentDetections,
memoryTrend: [...this.memoryHistory],
};
}
/**
* Shutdown memory leak detector
*/
shutdown(): void {
this.isEnabled = false;
if (this.monitoringInterval) {
clearInterval(this.monitoringInterval);
}
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
}
this.emit('shutdown');
}
// Private Methods
private initializeComponentTrackers(): void {
// Register core GEPA components
this.registerComponent('evolution-engine', {
maxObjectCount: 10000,
maxHeapSize: 100,
});
this.registerComponent('pareto-frontier', {
maxObjectCount: 5000,
maxHeapSize: 50,
});
this.registerComponent('cache-manager', {
maxObjectCount: 50000,
maxHeapSize: 200,
});
this.registerComponent('llm-adapter', {
maxObjectCount: 1000,
maxHeapSize: 50,
});
this.registerComponent('trajectory-store', {
maxObjectCount: 20000,
maxHeapSize: 150,
});
}
private startMonitoring(): void {
this.monitoringInterval = setInterval(async () => {
if (this.isEnabled) {
try {
await this.detectMemoryLeaks();
// Create periodic heap snapshots
if (Date.now() % this.thresholds.snapshotInterval < 1000) {
const snapshot = await this.createHeapSnapshot();
if (snapshot && this.lastHeapSnapshot) {
const comparison = await this.compareHeapSnapshots(
this.lastHeapSnapshot,
snapshot
);
this.emit('heapComparison', comparison);
}
this.lastHeapSnapshot = snapshot;
}
} catch (error) {
this.emit('monitoringError', error);
}
}
}, 5000); // Check every 5 seconds
}
private setupCleanupInterval(): void {
this.cleanupInterval = setInterval(async () => {
if (this.isEnabled) {
await this.forceCleanup();
}
}, 60000); // Cleanup every minute
}
private setupProcessHandlers(): void {
// Monitor for process exit to cleanup
process.on('exit', () => this.shutdown());
process.on('SIGINT', () => this.shutdown());
process.on('SIGTERM', () => this.shutdown());
}
private checkComponentThresholds(tracker: ComponentMemoryTracker): void {
const thresholds = { ...this.thresholds, ...tracker.thresholds };
if (tracker.objectCount > thresholds.maxObjectCount) {
this.emit('thresholdViolation', {
component: tracker.name,
type: 'objectCount',
current: tracker.objectCount,
threshold: thresholds.maxObjectCount,
});
}
if (tracker.memoryUsage > (thresholds.maxHeapSize * 1024 * 1024)) {
this.emit('thresholdViolation', {
component: tracker.name,
type: 'memoryUsage',
current: tracker.memoryUsage,
threshold: thresholds.maxHeapSize * 1024 * 1024,
});
}
}
private detectHeapGrowth(currentMemory: NodeJS.MemoryUsage): MemoryLeakDetection | null {
if (this.memoryHistory.length < 2) return null;
const oldestEntry = this.memoryHistory[0];
if (!oldestEntry?.timestamp || !oldestEntry?.usage) return null;
const timeDiff = (Date.now() - oldestEntry.timestamp) / 1000; // seconds
const memoryDiff = (currentMemory.heapUsed - oldestEntry.usage) / (1024 * 1024); // MB
const growthRate = timeDiff > 0 ? memoryDiff / timeDiff : 0;
if (growthRate > this.thresholds.heapGrowthRate) {
return {
timestamp: Date.now(),
component: 'system',
leakType: 'heap_growth',
severity: growthRate > this.thresholds.heapGrowthRate * 2 ? 'critical' : 'high',
currentUsage: currentMemory.heapUsed,
growthRate,
recommendation: 'Consider forcing garbage collection or reducing object allocation rate',
autoFixAvailable: true,
};
}
return null;
}
private detectComponentLeaks(tracker: ComponentMemoryTracker): MemoryLeakDetection[] {
const detections: MemoryLeakDetection[] = [];
const thresholds = { ...this.thresholds, ...tracker.thresholds };
// Check object accumulation
if (tracker.objectCount > thresholds.maxObjectCount) {
detections.push({
timestamp: Date.now(),
component: tracker.name,
leakType: 'object_accumulation',
severity: this.calculateSeverity(tracker.objectCount, thresholds.maxObjectCount),
currentUsage: tracker.objectCount,
growthRate: tracker.growthRate,
recommendation: `Consider implementing object pooling or reducing ${tracker.name} object creation`,
autoFixAvailable: tracker.name === 'cache-manager',
});
}
// Check weak reference buildup
if (tracker.weakRefs.size > thresholds.maxObjectCount * 0.5) {
detections.push({
timestamp: Date.now(),
component: tracker.name,
leakType: 'weak_ref_buildup',
severity: 'medium',
currentUsage: tracker.weakRefs.size,
growthRate: 0,
recommendation: 'Force cleanup of weak references',
autoFixAvailable: true,
});
}
return detections;
}
private detectProcessLeaks(): MemoryLeakDetection | null {
// Check for excessive file descriptors, event listeners, etc.
const processUsage = process.memoryUsage();
// Simplified check - in practice would examine more process metrics
if (processUsage.external > 100 * 1024 * 1024) { // 100MB external memory
return {
timestamp: Date.now(),
component: 'process',
leakType: 'process_leak',
severity: 'medium',
currentUsage: processUsage.external,
growthRate: 0,
recommendation: 'Check for file handle leaks or excessive buffer usage',
autoFixAvailable: false,
};
}
return null;
}
private async applyAutoFix(detection: MemoryLeakDetection): Promise<void> {
try {
switch (detection.leakType) {
case 'heap_growth':
if (global.gc) {
global.gc();
this.stats.autoFixesApplied++;
}
break;
case 'weak_ref_buildup':
await this.forceCleanup();
this.stats.autoFixesApplied++;
break;
case 'cache_overflow':
if (detection.component === 'cache-manager') {
// Would integrate with cache manager to force eviction
this.emit('cacheEvictionRequested', { severity: detection.severity });
this.stats.autoFixesApplied++;
}
break;
}
this.emit('autoFixApplied', detection);
} catch (error) {
this.emit('autoFixFailed', { detection, error });
}
}
private calculateSeverity(current: number, threshold: number): 'low' | 'medium' | 'high' | 'critical' {
const ratio = current / threshold;
if (ratio > 2) return 'critical';
if (ratio > 1.5) return 'high';
if (ratio > 1.2) return 'medium';
return 'low';
}
private detectLeakyConstructors(sizeIncrement: number): Array<{
name: string;
count: number;
size: number;
}> {
// Simplified leak detection - in practice would use V8 heap profiler
if (sizeIncrement > 10 * 1024 * 1024) { // 10MB increase
return [
{
name: 'Buffer',
count: Math.floor(sizeIncrement / 1024),
size: sizeIncrement,
},
];
}
return [];
}
}
/**
* Memory Leak Detection Integration Hooks
* Provides easy integration with existing GEPA components
*/
export class MemoryLeakIntegration {
private static detector: MemoryLeakDetector | null = null;
/**
* Initialize memory leak detection for GEPA
*/
static initialize(thresholds?: Partial<MemoryLeakThresholds>): MemoryLeakDetector {
if (!this.detector) {
this.detector = new MemoryLeakDetector(thresholds);
}
return this.detector;
}
/**
* Get current detector instance
*/
static getDetector(): MemoryLeakDetector | null {
return this.detector;
}
/**
* Track cache operations for memory leaks
*/
static trackCacheOperation(
operation: 'set' | 'get' | 'delete' | 'clear',
key: string,
size?: number
): void {
if (this.detector) {
if (operation === 'set') {
this.detector.trackObjectAllocation('cache-manager', key, size);
} else if (operation === 'delete' || operation === 'clear') {
this.detector.trackObjectDeallocation('cache-manager', size);
}
}
}
/**
* Track Pareto frontier operations
*/
static trackParetoOperation(
operation: 'add' | 'remove' | 'clear',
candidateId: string,
size?: number
): void {
if (this.detector) {
if (operation === 'add') {
this.detector.trackObjectAllocation('pareto-frontier', candidateId, size);
} else if (operation === 'remove' || operation === 'clear') {
this.detector.trackObjectDeallocation('pareto-frontier', size);
}
}
}
/**
* Track LLM process operations
*/
static trackLLMProcess(
operation: 'spawn' | 'exit',
processId: string,
memoryUsage?: number
): void {
if (this.detector) {
if (operation === 'spawn') {
this.detector.trackObjectAllocation('llm-adapter', processId, memoryUsage);
} else if (operation === 'exit') {
this.detector.trackObjectDeallocation('llm-adapter', memoryUsage);
}
}
}
/**
* Track circuit breaker operations
*/
static trackCircuitBreaker(operation: string, name: string, memoryUsage: number): void {
if (!this.detector) {
this.initialize();
}
if (this.detector) {
this.detector.trackObjectAllocation(`circuit-breaker-${name}`, {
operation,
name,
memoryUsage,
timestamp: new Date().toISOString()
});
}
}
/**
* Shutdown memory leak detection
*/
/**
* Track incident response system memory usage
*/
static trackIncidentResponse(
operation: string,
memoryUsage: number
): void {
if (this.detector) {
this.detector.trackObjectAllocation('incident-response', `${operation}-${Date.now()}`, memoryUsage);
}
}
/**
* Track monitoring system memory usage
*/
static trackMonitoringSystem(
operation: string,
memoryUsage: number
): void {
if (this.detector) {
this.detector.trackObjectAllocation('monitoring-system', `${operation}-${Date.now()}`, memoryUsage);
}
}
/**
* Track anomaly detection memory usage
*/
static trackAnomalyDetection(
operation: string,
memoryUsage: number
): void {
if (this.detector) {
this.detector.trackObjectAllocation('anomaly-detection', `${operation}-${Date.now()}`, memoryUsage);
}
}
/**
* Track error monitoring memory usage
*/
static trackErrorMonitoring(
operation: string,
memoryUsage: number
): void {
if (this.detector) {
this.detector.trackObjectAllocation('error-monitoring', `${operation}-${Date.now()}`, memoryUsage);
}
}
/**
* Track observability integration memory usage
*/
static trackObservability(
operation: string,
memoryUsage: number
): void {
if (this.detector) {
this.detector.trackObjectAllocation('observability', `${operation}-${Date.now()}`, memoryUsage);
}
}
/**
* Shutdown memory leak detector
*/
static shutdown(): void {
if (this.detector) {
this.detector.shutdown();
this.detector = null;
}
}
}