memory-monitoring-integration.ts•29.1 kB
/**
* Memory Monitoring System Integration
*
* Unified integration layer that combines all memory monitoring components
* into a cohesive real-time monitoring system for GEPA.
*/
import { EventEmitter } from 'events';
import { PerformanceTracker } from '../services/performance-tracker';
import { MemoryLeakDetector, MemoryLeakIntegration } from './memory-leak-detector';
import { GarbageCollectionOptimizer } from './gc-optimizer';
import { MemoryMonitoringSystem, MemoryMonitoringConfig, DashboardData } from './memory-monitoring';
import { MemoryAlertSystem, AlertInstance } from './memory-alerts';
import { MemoryAnalyticsEngine, AnalyticsConfig, SystemAnalytics } from './memory-analytics';
/**
* Complete monitoring system configuration
*/
export interface CompleteMonitoringConfig {
monitoring?: Partial<MemoryMonitoringConfig>;
analytics?: Partial<AnalyticsConfig>;
integration?: {
enableAutomatedResponses: boolean;
enableCrossComponentCorrelation: boolean;
enablePredictiveAlerts: boolean;
autoOptimizationEnabled: boolean;
emergencyThresholds: {
memoryUsagePercent: number;
gcFrequencyPerMinute: number;
anomalyCountPerHour: number;
};
};
}
/**
* Monitoring system status
*/
export interface MonitoringSystemStatus {
timestamp: number;
overall: 'healthy' | 'warning' | 'critical' | 'offline';
components: {
leakDetector: 'active' | 'inactive' | 'error';
gcOptimizer: 'active' | 'inactive' | 'error';
monitoring: 'active' | 'inactive' | 'error';
alertSystem: 'active' | 'inactive' | 'error';
analytics: 'active' | 'inactive' | 'error';
};
metrics: {
totalAlerts: number;
activeAnomalies: number;
memoryEfficiency: number;
systemStability: number;
lastUpdateTime: number;
};
health: {
uptime: number;
errorCount: number;
lastError?: string;
performanceImpact: number;
};
}
/**
* Emergency response action
*/
export interface EmergencyResponse {
id: string;
timestamp: number;
trigger: string;
severity: 'high' | 'critical';
actions: {
description: string;
executed: boolean;
success?: boolean;
error?: string;
}[];
result: {
memoryFreed: number;
stabilityImproved: boolean;
alertsResolved: number;
};
}
/**
* Main Memory Monitoring Integration System
*/
export class MemoryMonitoringIntegration extends EventEmitter {
private performanceTracker!: PerformanceTracker;
private leakDetector!: MemoryLeakDetector;
private gcOptimizer!: GarbageCollectionOptimizer;
private monitoringSystem!: MemoryMonitoringSystem;
private alertSystem!: MemoryAlertSystem;
private analyticsEngine!: MemoryAnalyticsEngine;
private isInitialized = false;
private isRunning = false;
private startTime = 0;
private errorCount = 0;
private lastError?: string;
private config: Required<CompleteMonitoringConfig>;
private emergencyResponses: EmergencyResponse[] = [];
// Integration coordination
private coordinationInterval: NodeJS.Timeout | undefined;
private healthCheckInterval: NodeJS.Timeout | undefined;
constructor(config: CompleteMonitoringConfig = {}) {
super();
this.config = {
monitoring: {
updateInterval: 5000,
historyWindow: 3600000,
maxDataPoints: 720,
alertThresholds: {
memoryUsagePercent: 80,
heapGrowthRate: 10 * 1024 * 1024,
gcFrequency: 10,
leakSeverity: 'medium'
},
dashboard: {
enabled: true,
updateRate: 2000,
components: ['evolution-engine', 'pareto-frontier', 'cache-manager', 'llm-adapter']
},
notifications: {
enabled: true,
channels: ['console'],
escalationPolicy: {
enabled: true,
levels: 3,
delayBetweenLevels: 300000
}
},
...(config.monitoring || {})
},
analytics: {
realTimeAnalytics: true,
analysisInterval: 30000,
dataRetention: 7 * 24 * 60 * 60 * 1000,
anomalySensitivity: 0.7,
baselineWindow: 24 * 60 * 60 * 1000,
prediction: {
enabled: true,
horizon: 60 * 60 * 1000,
confidence: 0.6
},
reporting: {
autoGenerate: true,
schedule: '0 8 * * *',
formats: ['json', 'html']
},
...(config.analytics || {})
},
integration: {
enableAutomatedResponses: true,
enableCrossComponentCorrelation: true,
enablePredictiveAlerts: true,
autoOptimizationEnabled: true,
emergencyThresholds: {
memoryUsagePercent: 95,
gcFrequencyPerMinute: 30,
anomalyCountPerHour: 20
},
...(config.integration || {})
}
};
this.initializeComponents();
this.setupIntegrationHandlers();
}
/**
* Initialize all monitoring components
*/
private initializeComponents(): void {
try {
// Initialize core components
this.performanceTracker = new PerformanceTracker({
enableRealTimeMonitoring: true,
memoryTrackingEnabled: true,
tokenUsageTrackingEnabled: true
});
this.leakDetector = MemoryLeakIntegration.initialize({
heapGrowthRate: (this.config.monitoring.alertThresholds?.heapGrowthRate || 10 * 1024 * 1024) / (60 * 1000), // per ms
maxHeapSize: 1024, // 1GB
monitoringWindow: this.config.monitoring.historyWindow || 3600000,
maxObjectCount: 100000,
memoryIncreaseThreshold: 50,
snapshotInterval: 30000
});
this.gcOptimizer = new GarbageCollectionOptimizer(
this.performanceTracker,
this.leakDetector
);
// Initialize monitoring system
this.monitoringSystem = new MemoryMonitoringSystem(
this.config.monitoring,
this.performanceTracker,
this.leakDetector,
this.gcOptimizer
);
// Initialize alert system
this.alertSystem = new MemoryAlertSystem();
// Initialize analytics engine
this.analyticsEngine = new MemoryAnalyticsEngine(this.config.analytics);
this.isInitialized = true;
this.emit('initialized');
} catch (error) {
this.handleError('Component initialization failed', error);
throw error;
}
}
/**
* Setup integration event handlers
*/
private setupIntegrationHandlers(): void {
// Monitor memory snapshots for analytics
this.monitoringSystem.on('snapshot', (snapshot) => {
// Convert snapshot to analytics format
for (const [componentName, componentData] of Object.entries(snapshot.components)) {
const data = componentData as {
memoryUsage: number;
objectCount: number;
growthRate: number;
poolUtilization: number;
};
this.analyticsEngine.addMemoryData({
timestamp: snapshot.timestamp,
component: componentName,
metrics: {
memoryUsage: data.memoryUsage,
objectCount: data.objectCount,
growthRate: data.growthRate,
poolUtilization: data.poolUtilization,
heapUsagePercent: (snapshot.system.heapUsed / snapshot.system.heapTotal) * 100
}
});
}
// System-level metrics
this.analyticsEngine.addMemoryData({
timestamp: snapshot.timestamp,
component: 'system',
metrics: {
heapUsed: snapshot.system.heapUsed,
heapTotal: snapshot.system.heapTotal,
heapUsagePercent: (snapshot.system.heapUsed / snapshot.system.heapTotal) * 100,
external: snapshot.system.external,
arrayBuffers: snapshot.system.arrayBuffers
}
});
});
// Handle memory alerts
this.monitoringSystem.on('alert', (alert) => {
this.alertSystem.processMemoryData({
timestamp: alert.timestamp,
component: alert.component,
metrics: {
[alert.type]: alert.currentValue,
value: alert.currentValue
},
metadata: {
severity: alert.severity,
type: alert.type,
recommendation: alert.recommendation
}
});
});
// Handle analytics insights
this.analyticsEngine.on('analyticsUpdate', (analytics: SystemAnalytics) => {
this.emit('analyticsUpdate', analytics);
// Check for emergency conditions
if (this.config.integration.enableAutomatedResponses) {
this.checkEmergencyConditions(analytics);
}
});
// Handle anomalies
this.analyticsEngine.on('anomalyDetected', (anomaly) => {
this.alertSystem.processMemoryData({
timestamp: anomaly.timestamp,
component: anomaly.component,
metrics: {
[anomaly.metric]: anomaly.value,
anomaly: 1,
deviation: anomaly.deviation
},
metadata: {
type: 'anomaly',
severity: anomaly.severity,
confidence: anomaly.confidence
}
});
});
// Handle critical anomalies
this.analyticsEngine.on('criticalAnomaly', (anomaly) => {
if (this.config.integration.enableAutomatedResponses) {
this.handleCriticalAnomaly(anomaly);
}
});
// Handle alert auto-fix attempts
this.alertSystem.on('autoFixAttempt', async ({ alert, resolve }) => {
try {
const success = await this.executeAutoFix(alert);
resolve(success);
} catch (error) {
// eslint-disable-next-line no-console
console.error('Auto-fix execution failed:', error);
resolve(false);
}
});
// Handle GC optimizer events
this.gcOptimizer.on('gcMetrics', (metrics) => {
this.alertSystem.processMemoryData({
timestamp: metrics.timestamp,
component: 'gc-optimizer',
metrics: {
gcDuration: metrics.duration,
gcFrequency: 1, // Would calculate actual frequency
heapReclaimed: metrics.heapReclaimed,
gcEfficiency: metrics.efficiency
}
});
});
// Handle memory leak detections
this.leakDetector.on('memoryLeakDetected', (detection) => {
this.alertSystem.processMemoryData({
timestamp: detection.timestamp,
component: detection.component,
metrics: {
leakDetected: 1,
leakSeverity: this.severityToNumber(detection.severity),
currentUsage: detection.currentUsage,
growthRate: detection.growthRate
},
metadata: {
leakType: detection.leakType,
severity: detection.severity,
recommendation: detection.recommendation,
autoFixAvailable: detection.autoFixAvailable
}
});
});
}
/**
* Start the integrated monitoring system
*/
async startMonitoring(): Promise<void> {
if (!this.isInitialized) {
throw new Error('System not initialized');
}
if (this.isRunning) {
// eslint-disable-next-line no-console
console.warn('Monitoring system is already running');
return;
}
try {
this.startTime = Date.now();
this.isRunning = true;
// Start individual components
this.monitoringSystem.startMonitoring();
this.analyticsEngine.startAnalytics();
// Start integration coordination
this.startCoordination();
// Start health monitoring
this.startHealthMonitoring();
this.emit('started');
// eslint-disable-next-line no-console
console.log('Integrated memory monitoring system started successfully');
} catch (error) {
this.isRunning = false;
this.handleError('Failed to start monitoring system', error);
throw error;
}
}
/**
* Stop the monitoring system
*/
async stopMonitoring(): Promise<void> {
if (!this.isRunning) {
// eslint-disable-next-line no-console
console.warn('Monitoring system is not running');
return;
}
try {
this.isRunning = false;
// Stop coordination
if (this.coordinationInterval) {
clearInterval(this.coordinationInterval);
this.coordinationInterval = undefined;
}
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval);
this.healthCheckInterval = undefined;
}
// Stop individual components
this.monitoringSystem.stopMonitoring();
this.analyticsEngine.stopAnalytics();
this.alertSystem.setEnabled(false);
this.emit('stopped');
// eslint-disable-next-line no-console
console.log('Integrated memory monitoring system stopped');
} catch (error) {
this.handleError('Failed to stop monitoring system', error);
throw error;
}
}
/**
* Start integration coordination
*/
private startCoordination(): void {
this.coordinationInterval = setInterval(() => {
this.performCoordination();
}, 10000); // Every 10 seconds
}
/**
* Start health monitoring
*/
private startHealthMonitoring(): void {
this.healthCheckInterval = setInterval(() => {
this.performHealthCheck();
}, 30000); // Every 30 seconds
}
/**
* Perform system coordination
*/
private performCoordination(): void {
try {
// Cross-component correlation
if (this.config.integration.enableCrossComponentCorrelation) {
this.performCrossComponentCorrelation();
}
// Predictive alerts
if (this.config.integration.enablePredictiveAlerts) {
this.checkPredictiveAlerts();
}
// Auto-optimization
if (this.config.integration.autoOptimizationEnabled) {
this.performAutoOptimization();
}
} catch (error) {
this.handleError('Coordination failed', error);
}
}
/**
* Perform health check
*/
private performHealthCheck(): void {
try {
const status = this.getSystemStatus();
this.emit('healthCheck', status);
// Log health status
if (status.overall === 'critical') {
// eslint-disable-next-line no-console
console.error('🔴 System health: CRITICAL');
} else if (status.overall === 'warning') {
// eslint-disable-next-line no-console
console.warn('🟡 System health: WARNING');
}
} catch (error) {
this.handleError('Health check failed', error);
}
}
/**
* Perform cross-component correlation
*/
private performCrossComponentCorrelation(): void {
// Get recent alerts from all components
const recentAlerts = this.alertSystem.getActiveAlerts();
const recentAnomalies = this.analyticsEngine.getSystemAnalytics().anomalies.slice(-10);
// Look for patterns across components
const componentIssues = new Map<string, number>();
for (const alert of recentAlerts) {
const count = componentIssues.get(alert.component) || 0;
componentIssues.set(alert.component, count + 1);
}
for (const anomaly of recentAnomalies) {
const count = componentIssues.get(anomaly.component) || 0;
componentIssues.set(anomaly.component, count + 1);
}
// Check for system-wide issues
if (componentIssues.size >= 3) { // Multiple components affected
this.emit('systemWideIssue', {
timestamp: Date.now(),
affectedComponents: Array.from(componentIssues.keys()),
severity: 'high',
description: 'Multiple components showing issues simultaneously'
});
}
}
/**
* Check for predictive alerts
*/
private checkPredictiveAlerts(): void {
// This would use prediction models to forecast issues
// For now, we'll check trends
const analytics = this.analyticsEngine.getSystemAnalytics();
for (const health of analytics.componentHealth) {
if (health.overallScore < 70 && health.historicalTrend.trend === 'degrading') {
this.emit('predictiveAlert', {
component: health.component,
predictedIssue: 'health_degradation',
confidence: 0.8,
timeToIssue: 3600000, // 1 hour estimate
recommendations: health.recommendations.immediate
});
}
}
}
/**
* Perform auto-optimization
*/
private performAutoOptimization(): void {
const optimizations = this.analyticsEngine.getSystemAnalytics().optimizations;
for (const optimization of optimizations) {
if (optimization.priority === 'critical' &&
optimization.implementation.effort === 'low' &&
optimization.riskAssessment.level === 'low') {
// Auto-execute low-risk, high-impact optimizations
this.executeOptimization(optimization);
}
}
}
/**
* Check for emergency conditions
*/
private checkEmergencyConditions(analytics: SystemAnalytics): void {
const thresholds = this.config.integration.emergencyThresholds;
// Check memory usage
if (analytics.overview.memoryUtilization > thresholds.memoryUsagePercent / 100) {
this.triggerEmergencyResponse('high_memory_usage', 'critical');
}
// Check anomaly count
const recentAnomalies = analytics.anomalies.filter(a =>
Date.now() - a.timestamp < 3600000 // Last hour
);
if (recentAnomalies.length > thresholds.anomalyCountPerHour) {
this.triggerEmergencyResponse('high_anomaly_rate', 'high');
}
// Check critical components
if (analytics.overview.criticalComponents > 0) {
this.triggerEmergencyResponse('critical_components', 'critical');
}
}
/**
* Handle critical anomaly
*/
private async handleCriticalAnomaly(anomaly: any): Promise<void> {
// eslint-disable-next-line no-console
console.error(`🚨 Critical anomaly detected: ${anomaly.component} ${anomaly.metric}`);
// Immediate response actions
const actions = [];
if (anomaly.metric === 'memoryUsage') {
actions.push('Force garbage collection');
actions.push('Clear memory caches');
}
if (anomaly.component !== 'system') {
actions.push(`Restart ${anomaly.component} component`);
}
// Execute emergency response
await this.triggerEmergencyResponse('critical_anomaly', 'critical', actions);
}
/**
* Trigger emergency response
*/
private async triggerEmergencyResponse(
trigger: string,
severity: 'high' | 'critical',
customActions?: string[]
): Promise<void> {
const responseId = `emergency-${Date.now()}`;
const timestamp = Date.now();
// Determine actions based on trigger
const actions = customActions || this.getEmergencyActions(trigger, severity);
const response: EmergencyResponse = {
id: responseId,
timestamp,
trigger,
severity,
actions: actions.map(action => ({ description: action, executed: false })),
result: {
memoryFreed: 0,
stabilityImproved: false,
alertsResolved: 0
}
};
this.emergencyResponses.push(response);
this.emit('emergencyResponse', response);
// eslint-disable-next-line no-console
console.error(`🚨 Emergency response triggered: ${trigger} (${severity})`);
// Execute actions
for (const actionItem of response.actions) {
try {
const result = await this.executeEmergencyAction(actionItem.description);
actionItem.executed = true;
actionItem.success = result.success;
if (result.success) {
response.result.memoryFreed += result.memoryFreed || 0;
response.result.alertsResolved += result.alertsResolved || 0;
}
} catch (error) {
actionItem.executed = true;
actionItem.success = false;
actionItem.error = error instanceof Error ? error.message : String(error);
}
}
// Check if response was successful
const successfulActions = response.actions.filter(a => a.success).length;
response.result.stabilityImproved = successfulActions > response.actions.length / 2;
this.emit('emergencyResponseComplete', response);
}
/**
* Get emergency actions for trigger
*/
private getEmergencyActions(trigger: string, severity: 'high' | 'critical'): string[] {
const actions = [];
switch (trigger) {
case 'high_memory_usage':
actions.push('Force garbage collection');
actions.push('Clear object pools');
actions.push('Reduce cache sizes');
if (severity === 'critical') {
actions.push('Emergency memory cleanup');
}
break;
case 'high_anomaly_rate':
actions.push('Increase monitoring frequency');
actions.push('Force component health checks');
actions.push('Enable enhanced logging');
break;
case 'critical_components':
actions.push('Attempt component recovery');
actions.push('Isolate failing components');
actions.push('Enable failover mechanisms');
break;
default:
actions.push('Perform system health check');
actions.push('Enable enhanced monitoring');
}
return actions;
}
/**
* Execute emergency action
*/
private async executeEmergencyAction(action: string): Promise<{ success: boolean; memoryFreed?: number; alertsResolved?: number }> {
switch (action) {
case 'Force garbage collection':
try {
const result = await this.gcOptimizer.forceGarbageCollection('emergency');
return { success: true, memoryFreed: result.heapReclaimed };
} catch (error) {
return { success: false };
}
case 'Clear object pools':
// Would interact with GC optimizer to clear pools
return { success: true, memoryFreed: 50 * 1024 * 1024 }; // Simulated
case 'Emergency memory cleanup':
try {
await this.leakDetector.forceCleanup();
return { success: true, memoryFreed: 100 * 1024 * 1024 }; // Simulated
} catch (error) {
return { success: false };
}
default:
// eslint-disable-next-line no-console
console.log(`Executing emergency action: ${action}`);
return { success: true };
}
}
/**
* Execute auto-fix for alert
*/
private async executeAutoFix(alert: AlertInstance): Promise<boolean> {
try {
switch (alert.type) {
case 'threshold':
if (alert.component === 'system') {
await this.gcOptimizer.forceGarbageCollection('auto-fix');
return true;
}
break;
case 'leak':
await this.leakDetector.forceCleanup();
return true;
case 'gc':
// Optimize GC settings
return true;
default:
return false;
}
return false;
} catch (error) {
// eslint-disable-next-line no-console
console.error('Auto-fix execution failed:', error);
return false;
}
}
/**
* Execute optimization
*/
private async executeOptimization(optimization: any): Promise<void> {
// eslint-disable-next-line no-console
console.log(`Auto-executing optimization: ${optimization.title}`);
// This would implement actual optimization execution
// For now, just log the attempt
this.emit('optimizationExecuted', {
optimization,
timestamp: Date.now(),
success: true
});
}
/**
* Get comprehensive dashboard data
*/
getDashboardData(): DashboardData & { analytics: SystemAnalytics; status: MonitoringSystemStatus } {
const dashboardData = this.monitoringSystem.getDashboardData();
const analytics = this.analyticsEngine.getSystemAnalytics();
const status = this.getSystemStatus();
return {
...dashboardData,
analytics,
status
};
}
/**
* Get system status
*/
getSystemStatus(): MonitoringSystemStatus {
const now = Date.now();
const analytics = this.analyticsEngine.getSystemAnalytics();
const activeAlerts = this.alertSystem.getActiveAlerts();
// Determine overall status
let overall: 'healthy' | 'warning' | 'critical' | 'offline' = 'healthy';
if (!this.isRunning) {
overall = 'offline';
} else if (analytics.overview.criticalComponents > 0 || activeAlerts.some(a => a.severity === 'critical')) {
overall = 'critical';
} else if (analytics.overview.warningComponents > 0 || activeAlerts.length > 5) {
overall = 'warning';
}
return {
timestamp: now,
overall,
components: {
leakDetector: this.isRunning ? 'active' : 'inactive',
gcOptimizer: this.isRunning ? 'active' : 'inactive',
monitoring: this.isRunning ? 'active' : 'inactive',
alertSystem: this.isRunning ? 'active' : 'inactive',
analytics: this.isRunning ? 'active' : 'inactive'
},
metrics: {
totalAlerts: activeAlerts.length,
activeAnomalies: analytics.anomalies.filter(a => Date.now() - a.timestamp < 3600000).length,
memoryEfficiency: analytics.overview.overallEfficiency,
systemStability: analytics.overview.stabilityIndex,
lastUpdateTime: now
},
health: {
uptime: this.startTime > 0 ? now - this.startTime : 0,
errorCount: this.errorCount,
...(this.lastError ? { lastError: this.lastError } : {}),
performanceImpact: 0.1 // Low impact from monitoring
}
};
}
/**
* Generate comprehensive report
*/
async generateReport(type: 'daily' | 'weekly' | 'monthly' | 'custom' = 'daily'): Promise<any> {
const analyticsReport = await this.analyticsEngine.generateReport(type);
const systemStatus = this.getSystemStatus();
const alertStatistics = this.alertSystem.getStatistics();
return {
...analyticsReport,
systemStatus,
alertStatistics,
emergencyResponses: this.emergencyResponses.slice(-10), // Last 10 responses
integrationMetrics: {
coordinationCycles: this.isRunning ? Math.floor((Date.now() - this.startTime) / 10000) : 0,
autoFixAttempts: alertStatistics.autoFixSuccess + alertStatistics.autoFixFailure,
autoFixSuccessRate: alertStatistics.autoFixSuccess / Math.max(1, alertStatistics.autoFixSuccess + alertStatistics.autoFixFailure),
emergencyResponseCount: this.emergencyResponses.length
}
};
}
/**
* Handle system errors
*/
private handleError(message: string, error: any): void {
this.errorCount++;
this.lastError = `${message}: ${error instanceof Error ? error.message : String(error)}`;
// eslint-disable-next-line no-console
console.error(`Memory Monitoring Integration Error: ${this.lastError}`);
this.emit('error', { message, error, timestamp: Date.now() });
}
/**
* Convert severity to number for processing
*/
private severityToNumber(severity: string): number {
const severityMap: Record<string, number> = {
'low': 1,
'medium': 2,
'high': 3,
'critical': 4
};
return severityMap[severity] || 1;
}
/**
* Get emergency response history
*/
getEmergencyResponses(limit?: number): EmergencyResponse[] {
const responses = [...this.emergencyResponses].reverse(); // Most recent first
return limit ? responses.slice(0, limit) : responses;
}
/**
* Shutdown the entire monitoring system
*/
async shutdown(): Promise<void> {
try {
await this.stopMonitoring();
// Shutdown individual components
this.monitoringSystem.shutdown();
this.analyticsEngine.shutdown();
this.alertSystem.shutdown();
this.gcOptimizer.shutdown();
this.leakDetector.shutdown();
this.removeAllListeners();
// eslint-disable-next-line no-console
console.log('Memory monitoring integration system shut down completely');
} catch (error) {
this.handleError('Shutdown failed', error);
throw error;
}
}
}
/**
* Global integration instance for easy access
*/
export class GlobalMemoryMonitoring {
private static instance: MemoryMonitoringIntegration | null = null;
/**
* Initialize global monitoring instance
*/
static initialize(config?: CompleteMonitoringConfig): MemoryMonitoringIntegration {
if (!this.instance) {
this.instance = new MemoryMonitoringIntegration(config);
}
return this.instance;
}
/**
* Get current monitoring instance
*/
static getInstance(): MemoryMonitoringIntegration | null {
return this.instance;
}
/**
* Shutdown global monitoring
*/
static async shutdown(): Promise<void> {
if (this.instance) {
await this.instance.shutdown();
this.instance = null;
}
}
}