component-recovery-manager.ts•30.6 kB
/**
* Component Recovery Manager - Individual Component Recovery Systems
*
* Provides targeted recovery for specific GEPA components:
* - Evolution engine state recovery
* - Pareto frontier reconstruction
* - Cache system recovery
* - LLM adapter process recovery
* - Component health monitoring
*/
import { EventEmitter } from 'events';
import { ResilienceSystem } from '../resilience/index';
export enum ComponentType {
EVOLUTION_ENGINE = 'evolution_engine',
PARETO_FRONTIER = 'pareto_frontier',
REFLECTION_ENGINE = 'reflection_engine',
PROMPT_MUTATOR = 'prompt_mutator',
LLM_ADAPTER = 'llm_adapter',
TRAJECTORY_STORE = 'trajectory_store',
MEMORY_CACHE = 'memory_cache',
PERFORMANCE_TRACKER = 'performance_tracker',
RESILIENCE_SYSTEM = 'resilience_system'
}
export enum ComponentStatus {
HEALTHY = 'healthy',
DEGRADED = 'degraded',
CRITICAL = 'critical',
FAILED = 'failed',
RECOVERING = 'recovering',
UNKNOWN = 'unknown'
}
export enum RecoveryStrategy {
RESTART = 'restart',
RESTORE_FROM_BACKUP = 'restore_from_backup',
REBUILD = 'rebuild',
FAILOVER = 'failover',
RESET_TO_DEFAULTS = 'reset_to_defaults',
PARTIAL_RECOVERY = 'partial_recovery',
FULL_RECOVERY = 'full_recovery'
}
export interface ComponentRecoveryConfig {
healthCheckInterval: number; // milliseconds
recoveryTimeout: number; // milliseconds
maxRecoveryAttempts: number;
autoRecoveryEnabled: boolean;
componentConfigs: Map<ComponentType, ComponentSpecificConfig>;
}
export interface ComponentSpecificConfig {
healthCheckEndpoint?: string;
recoveryStrategies: RecoveryStrategy[];
dependencies: ComponentType[];
criticalityLevel: 'low' | 'medium' | 'high' | 'critical';
restartPolicy: 'never' | 'on_failure' | 'always';
backupRetentionCount: number;
}
export interface ComponentHealth {
status: ComponentStatus;
lastCheck: Date;
metrics: Record<string, number>;
errors: string[];
warnings: string[];
recommendations: string[];
uptime: number; // milliseconds
lastRecovery?: Date;
recoveryCount: number;
}
export interface RecoveryAttempt {
id: string;
componentType: ComponentType;
strategy: RecoveryStrategy;
startTime: Date;
endTime?: Date;
success: boolean;
duration?: number; // milliseconds
logs: string[];
error?: Error;
preRecoveryState?: ComponentState;
postRecoveryState?: ComponentState;
}
export interface ComponentState {
type: ComponentType;
timestamp: Date;
status: ComponentStatus;
configuration: Record<string, any>;
metrics: Record<string, number>;
errorLog: string[];
lastAction: string;
dependencies: ComponentType[];
}
export interface ComponentDependencyGraph {
nodes: ComponentType[];
edges: Array<{ from: ComponentType; to: ComponentType; weight: number }>;
criticalPaths: ComponentType[][];
}
/**
* Component Recovery Manager Implementation
*/
export class ComponentRecoveryManager extends EventEmitter {
private config: ComponentRecoveryConfig;
private componentHealth: Map<ComponentType, ComponentHealth> = new Map();
private recoveryHistory: Map<ComponentType, RecoveryAttempt[]> = new Map();
private monitoringTimer?: NodeJS.Timeout | undefined;
private activeRecoveries: Map<string, RecoveryAttempt> = new Map();
private _dependencyGraph: ComponentDependencyGraph = {
nodes: [],
edges: [],
criticalPaths: []
};
private resilience: ResilienceSystem;
constructor(config: Partial<ComponentRecoveryConfig> = {}) {
super();
this.config = {
healthCheckInterval: 60000, // 1 minute
recoveryTimeout: 300000, // 5 minutes
maxRecoveryAttempts: 3,
autoRecoveryEnabled: true,
componentConfigs: new Map(),
...config
};
this.resilience = ResilienceSystem.getInstance();
this.initializeComponentConfigs();
this.buildDependencyGraph();
// Use dependency graph for recovery ordering (mark as used)
void this._dependencyGraph;
}
/**
* Initialize component recovery manager
*/
async initialize(): Promise<void> {
try {
// Initialize component health monitoring
await this.initializeHealthMonitoring();
// Start periodic health checks
this.startHealthChecks();
// Validate component configurations
this.validateComponentConfigs();
this.emit('initialized', {
componentCount: this.config.componentConfigs.size,
monitoringInterval: this.config.healthCheckInterval
});
} catch (error) {
this.emit('error', { operation: 'initialize', error });
throw error;
}
}
/**
* Recover specific component
*/
async recoverComponent(
componentType: ComponentType,
strategy?: RecoveryStrategy
): Promise<RecoveryAttempt> {
return this.resilience.executeWithFullProtection(
async () => {
// Check if component is already being recovered
const activeRecovery = Array.from(this.activeRecoveries.values())
.find(r => r.componentType === componentType && !r.endTime);
if (activeRecovery) {
throw new Error(`Component ${componentType} is already being recovered`);
}
// Get component configuration
const componentConfig = this.config.componentConfigs.get(componentType);
if (!componentConfig) {
throw new Error(`No configuration found for component: ${componentType}`);
}
// Determine recovery strategy
const selectedStrategy = strategy || this.selectOptimalStrategy(componentType);
// Create recovery attempt
const attempt: RecoveryAttempt = {
id: this.generateRecoveryId(),
componentType,
strategy: selectedStrategy,
startTime: new Date(),
success: false,
logs: [],
preRecoveryState: await this.captureComponentState(componentType)
};
this.activeRecoveries.set(attempt.id, attempt);
try {
// Execute recovery strategy
await this.executeRecoveryStrategy(attempt);
attempt.success = true;
attempt.endTime = new Date();
attempt.duration = attempt.endTime.getTime() - attempt.startTime.getTime();
attempt.postRecoveryState = await this.captureComponentState(componentType);
// Update component health
await this.updateComponentHealth(componentType);
// Record recovery history
this.recordRecoveryAttempt(attempt);
this.emit('componentRecovered', attempt);
} catch (error) {
attempt.success = false;
attempt.error = error as Error;
attempt.endTime = new Date();
attempt.duration = attempt.endTime.getTime() - attempt.startTime.getTime();
attempt.logs.push(`Recovery failed: ${(error as Error).message}`);
this.recordRecoveryAttempt(attempt);
this.emit('componentRecoveryFailed', attempt);
throw error;
} finally {
this.activeRecoveries.delete(attempt.id);
}
return attempt;
},
{
serviceName: 'component-recovery',
context: {
name: 'recover-component',
priority: 'high'
}
}
);
}
/**
* Perform health check for specific component
*/
async checkComponentHealth(componentType: ComponentType): Promise<ComponentHealth> {
return this.resilience.executeWithFullProtection(
async () => {
const health: ComponentHealth = {
status: ComponentStatus.UNKNOWN,
lastCheck: new Date(),
metrics: {},
errors: [],
warnings: [],
recommendations: [],
uptime: 0,
recoveryCount: this.getRecoveryCount(componentType)
};
try {
// Perform component-specific health check
const metrics = await this.performComponentHealthCheck(componentType);
health.metrics = metrics;
// Analyze health status
health.status = this.analyzeComponentHealth(componentType, metrics);
// Calculate uptime
health.uptime = this.calculateComponentUptime();
// Generate recommendations
health.recommendations = this.generateHealthRecommendations(componentType, health);
// Update health cache
this.componentHealth.set(componentType, health);
this.emit('healthCheckCompleted', { componentType, health });
} catch (error) {
health.status = ComponentStatus.FAILED;
health.errors.push(`Health check failed: ${(error as Error).message}`);
this.emit('healthCheckFailed', { componentType, error });
}
return health;
},
{
serviceName: 'component-recovery',
context: {
name: 'health-check',
priority: 'medium'
}
}
);
}
/**
* Get component health status
*/
getComponentHealth(componentType: ComponentType): ComponentHealth | null {
return this.componentHealth.get(componentType) || null;
}
/**
* Get all component health statuses
*/
getAllComponentHealth(): Map<ComponentType, ComponentHealth> {
return new Map(this.componentHealth);
}
/**
* Get recovery history for component
*/
getRecoveryHistory(componentType: ComponentType): RecoveryAttempt[] {
return this.recoveryHistory.get(componentType) || [];
}
/**
* Trigger cascade recovery for dependent components
*/
async performCascadeRecovery(rootComponent: ComponentType): Promise<RecoveryAttempt[]> {
const dependentComponents = this.getDependentComponents(rootComponent);
const recoveryAttempts: RecoveryAttempt[] = [];
// Recovery order based on dependency graph
const recoveryOrder = this.calculateRecoveryOrder([rootComponent, ...dependentComponents]);
for (const componentType of recoveryOrder) {
try {
const attempt = await this.recoverComponent(componentType);
recoveryAttempts.push(attempt);
// Wait a bit between recoveries to avoid overwhelming the system
await new Promise(resolve => setTimeout(resolve, 5000));
} catch (error) {
// Continue with other components even if one fails
// eslint-disable-next-line no-console
console.warn(`Cascade recovery failed for ${componentType}:`, error);
}
}
return recoveryAttempts;
}
/**
* Get overall system health status
*/
async getHealthStatus(): Promise<ComponentHealth> {
const componentHealths = Array.from(this.componentHealth.values());
let overallStatus: ComponentStatus = ComponentStatus.HEALTHY;
const errors: string[] = [];
const warnings: string[] = [];
const recommendations: string[] = [];
// Determine overall status
if (componentHealths.some(h => h.status === ComponentStatus.FAILED)) {
overallStatus = ComponentStatus.FAILED;
} else if (componentHealths.some(h => h.status === ComponentStatus.CRITICAL)) {
overallStatus = ComponentStatus.CRITICAL;
} else if (componentHealths.some(h => h.status === ComponentStatus.DEGRADED)) {
overallStatus = ComponentStatus.DEGRADED;
}
// Collect errors, warnings, and recommendations
for (const health of componentHealths) {
errors.push(...health.errors);
warnings.push(...health.warnings);
recommendations.push(...health.recommendations);
}
// Calculate average uptime
const averageUptime = componentHealths.length > 0
? componentHealths.reduce((sum, h) => sum + h.uptime, 0) / componentHealths.length
: 0;
return {
status: overallStatus,
lastCheck: new Date(),
metrics: {
totalComponents: this.config.componentConfigs.size,
healthyComponents: componentHealths.filter(h => h.status === ComponentStatus.HEALTHY).length,
degradedComponents: componentHealths.filter(h => h.status === ComponentStatus.DEGRADED).length,
criticalComponents: componentHealths.filter(h => h.status === ComponentStatus.CRITICAL).length,
failedComponents: componentHealths.filter(h => h.status === ComponentStatus.FAILED).length,
averageUptime: averageUptime
},
errors,
warnings,
recommendations,
uptime: averageUptime,
recoveryCount: this.getTotalRecoveryCount()
};
}
/**
* Private helper methods
*/
private initializeComponentConfigs(): void {
// Evolution Engine Configuration
this.config.componentConfigs.set(ComponentType.EVOLUTION_ENGINE, {
recoveryStrategies: [RecoveryStrategy.RESTART, RecoveryStrategy.RESTORE_FROM_BACKUP, RecoveryStrategy.REBUILD],
dependencies: [ComponentType.PARETO_FRONTIER, ComponentType.LLM_ADAPTER, ComponentType.TRAJECTORY_STORE],
criticalityLevel: 'critical',
restartPolicy: 'on_failure',
backupRetentionCount: 5
});
// Pareto Frontier Configuration
this.config.componentConfigs.set(ComponentType.PARETO_FRONTIER, {
recoveryStrategies: [RecoveryStrategy.REBUILD, RecoveryStrategy.RESTORE_FROM_BACKUP],
dependencies: [],
criticalityLevel: 'high',
restartPolicy: 'on_failure',
backupRetentionCount: 10
});
// Reflection Engine Configuration
this.config.componentConfigs.set(ComponentType.REFLECTION_ENGINE, {
recoveryStrategies: [RecoveryStrategy.RESTART, RecoveryStrategy.RESET_TO_DEFAULTS],
dependencies: [ComponentType.LLM_ADAPTER],
criticalityLevel: 'high',
restartPolicy: 'on_failure',
backupRetentionCount: 3
});
// LLM Adapter Configuration
this.config.componentConfigs.set(ComponentType.LLM_ADAPTER, {
recoveryStrategies: [RecoveryStrategy.RESTART, RecoveryStrategy.FAILOVER],
dependencies: [],
criticalityLevel: 'critical',
restartPolicy: 'always',
backupRetentionCount: 2
});
// Trajectory Store Configuration
this.config.componentConfigs.set(ComponentType.TRAJECTORY_STORE, {
recoveryStrategies: [RecoveryStrategy.RESTORE_FROM_BACKUP, RecoveryStrategy.REBUILD],
dependencies: [],
criticalityLevel: 'critical',
restartPolicy: 'on_failure',
backupRetentionCount: 20
});
// Memory Cache Configuration
this.config.componentConfigs.set(ComponentType.MEMORY_CACHE, {
recoveryStrategies: [RecoveryStrategy.RESTART, RecoveryStrategy.RESET_TO_DEFAULTS],
dependencies: [],
criticalityLevel: 'medium',
restartPolicy: 'on_failure',
backupRetentionCount: 1
});
// Performance Tracker Configuration
this.config.componentConfigs.set(ComponentType.PERFORMANCE_TRACKER, {
recoveryStrategies: [RecoveryStrategy.RESTART, RecoveryStrategy.RESET_TO_DEFAULTS],
dependencies: [],
criticalityLevel: 'low',
restartPolicy: 'on_failure',
backupRetentionCount: 3
});
// Resilience System Configuration
this.config.componentConfigs.set(ComponentType.RESILIENCE_SYSTEM, {
recoveryStrategies: [RecoveryStrategy.RESTART, RecoveryStrategy.REBUILD],
dependencies: [],
criticalityLevel: 'critical',
restartPolicy: 'always',
backupRetentionCount: 5
});
}
private buildDependencyGraph(): void {
const nodes = Array.from(this.config.componentConfigs.keys());
const edges: Array<{ from: ComponentType; to: ComponentType; weight: number }> = [];
// Build edges based on dependencies
for (const [componentType, config] of this.config.componentConfigs) {
for (const dependency of config.dependencies) {
edges.push({
from: dependency,
to: componentType,
weight: this.getCriticalityWeight(config.criticalityLevel)
});
}
}
// Calculate critical paths
const criticalPaths = this.calculateCriticalPaths(nodes, edges);
this._dependencyGraph = { nodes, edges, criticalPaths };
}
private getCriticalityWeight(level: string): number {
switch (level) {
case 'critical': return 4;
case 'high': return 3;
case 'medium': return 2;
case 'low': return 1;
default: return 1;
}
}
private calculateCriticalPaths(
nodes: ComponentType[],
edges: Array<{ from: ComponentType; to: ComponentType; weight: number }>
): ComponentType[][] {
// Simple implementation - in practice, would use more sophisticated graph algorithms
const paths: ComponentType[][] = [];
// Find components with no dependencies (root components)
const rootComponents = nodes.filter(node =>
!edges.some(edge => edge.to === node)
);
// For each root component, find paths to dependent components
for (const root of rootComponents) {
const path = this.findPathFromRoot(root, edges);
if (path.length > 1) {
paths.push(path);
}
}
return paths;
}
private findPathFromRoot(
root: ComponentType,
edges: Array<{ from: ComponentType; to: ComponentType; weight: number }>
): ComponentType[] {
const path = [root];
const visited = new Set([root]);
let current = root;
// eslint-disable-next-line no-constant-condition
while (true) {
const nextEdge = edges.find(edge => edge.from === current && !visited.has(edge.to));
if (!nextEdge) break;
path.push(nextEdge.to);
visited.add(nextEdge.to);
current = nextEdge.to;
}
return path;
}
private async initializeHealthMonitoring(): Promise<void> {
// Initialize health status for all components
for (const componentType of this.config.componentConfigs.keys()) {
await this.checkComponentHealth(componentType);
}
}
private startHealthChecks(): void {
this.monitoringTimer = setInterval(async () => {
for (const componentType of this.config.componentConfigs.keys()) {
try {
await this.checkComponentHealth(componentType);
// Auto-recovery if enabled and component is unhealthy
if (this.config.autoRecoveryEnabled) {
const health = this.componentHealth.get(componentType);
if (health && this.shouldTriggerAutoRecovery(health)) {
await this.recoverComponent(componentType);
}
}
} catch (error) {
// eslint-disable-next-line no-console
console.warn(`Health check failed for ${componentType}:`, error);
}
}
}, this.config.healthCheckInterval);
}
private shouldTriggerAutoRecovery(health: ComponentHealth): boolean {
return health.status === ComponentStatus.FAILED ||
(health.status === ComponentStatus.CRITICAL && health.recoveryCount < this.config.maxRecoveryAttempts);
}
private async performComponentHealthCheck(componentType: ComponentType): Promise<Record<string, number>> {
// Component-specific health check implementations
switch (componentType) {
case ComponentType.EVOLUTION_ENGINE:
return this.checkEvolutionEngineHealth();
case ComponentType.PARETO_FRONTIER:
return this.checkParetoFrontierHealth();
case ComponentType.LLM_ADAPTER:
return this.checkLLMAdapterHealth();
case ComponentType.TRAJECTORY_STORE:
return this.checkTrajectoryStoreHealth();
case ComponentType.MEMORY_CACHE:
return this.checkMemoryCacheHealth();
default:
return this.checkGenericComponentHealth();
}
}
private async checkEvolutionEngineHealth(): Promise<Record<string, number>> {
return {
populationSize: 10, // Mock data
generation: 5,
convergenceRate: 0.85,
memoryUsage: 45.2,
cpuUsage: 23.1
};
}
private async checkParetoFrontierHealth(): Promise<Record<string, number>> {
return {
frontierSize: 8,
dominanceChecks: 1500,
updateFrequency: 0.95,
memoryUsage: 12.3
};
}
private async checkLLMAdapterHealth(): Promise<Record<string, number>> {
return {
responseTime: 1500, // milliseconds
errorRate: 0.02,
throughput: 50, // requests per minute
availability: 0.999
};
}
private async checkTrajectoryStoreHealth(): Promise<Record<string, number>> {
return {
trajectoryCount: 1000,
storageUsage: 78.5, // percentage
queryPerformance: 250, // milliseconds
dataIntegrity: 1.0
};
}
private async checkMemoryCacheHealth(): Promise<Record<string, number>> {
return {
hitRate: 0.92,
memoryUsage: 67.8,
evictionRate: 0.05,
capacity: 85.2
};
}
private async checkGenericComponentHealth(): Promise<Record<string, number>> {
return {
status: 1, // 1 = healthy, 0 = unhealthy
uptime: 3600000, // milliseconds
memoryUsage: 30.0,
cpuUsage: 15.0
};
}
private analyzeComponentHealth(componentType: ComponentType, metrics: Record<string, number>): ComponentStatus {
// Component-specific health analysis
switch (componentType) {
case ComponentType.LLM_ADAPTER:
if (metrics.availability && metrics.errorRate && (metrics.availability < 0.95 || metrics.errorRate > 0.1)) {
return ComponentStatus.CRITICAL;
} else if (metrics.responseTime && metrics.errorRate && (metrics.responseTime > 5000 || metrics.errorRate > 0.05)) {
return ComponentStatus.DEGRADED;
}
break;
case ComponentType.TRAJECTORY_STORE:
if (metrics.dataIntegrity && metrics.dataIntegrity < 0.99) {
return ComponentStatus.CRITICAL;
} else if (metrics.storageUsage && metrics.queryPerformance && (metrics.storageUsage > 90 || metrics.queryPerformance > 1000)) {
return ComponentStatus.DEGRADED;
}
break;
case ComponentType.MEMORY_CACHE:
if (metrics.hitRate && metrics.hitRate < 0.5) {
return ComponentStatus.DEGRADED;
} else if (metrics.memoryUsage && metrics.memoryUsage > 95) {
return ComponentStatus.CRITICAL;
}
break;
}
// Generic analysis
if (metrics.memoryUsage && metrics.cpuUsage && (metrics.memoryUsage > 95 || metrics.cpuUsage > 95)) {
return ComponentStatus.CRITICAL;
} else if (metrics.memoryUsage && metrics.cpuUsage && (metrics.memoryUsage > 80 || metrics.cpuUsage > 80)) {
return ComponentStatus.DEGRADED;
}
return ComponentStatus.HEALTHY;
}
private calculateComponentUptime(): number {
// Calculate uptime based on health history
// For now, return a mock value
return Date.now() - (new Date().getTime() - 3600000); // 1 hour uptime
}
private generateHealthRecommendations(componentType: ComponentType, health: ComponentHealth): string[] {
const recommendations: string[] = [];
if (health.status === ComponentStatus.DEGRADED) {
recommendations.push(`Monitor ${componentType} closely for performance degradation`);
}
if (health.status === ComponentStatus.CRITICAL || health.status === ComponentStatus.FAILED) {
recommendations.push(`Immediate attention required for ${componentType}`);
recommendations.push(`Consider manual recovery for ${componentType}`);
}
if (health.recoveryCount > 2) {
recommendations.push(`High recovery frequency for ${componentType} - investigate root cause`);
}
return recommendations;
}
private selectOptimalStrategy(componentType: ComponentType): RecoveryStrategy {
const config = this.config.componentConfigs.get(componentType);
if (!config || config.recoveryStrategies.length === 0) {
return RecoveryStrategy.RESTART;
}
// Select first available strategy for now
// In practice, would use more sophisticated selection logic
const strategy = config.recoveryStrategies[0];
return strategy ?? RecoveryStrategy.RESTART;
}
private async executeRecoveryStrategy(attempt: RecoveryAttempt): Promise<void> {
attempt.logs.push(`Starting recovery with strategy: ${attempt.strategy}`);
switch (attempt.strategy) {
case RecoveryStrategy.RESTART:
await this.executeRestartRecovery(attempt);
break;
case RecoveryStrategy.RESTORE_FROM_BACKUP:
await this.executeRestoreRecovery(attempt);
break;
case RecoveryStrategy.REBUILD:
await this.executeRebuildRecovery(attempt);
break;
case RecoveryStrategy.FAILOVER:
await this.executeFailoverRecovery(attempt);
break;
case RecoveryStrategy.RESET_TO_DEFAULTS:
await this.executeResetRecovery(attempt);
break;
default:
throw new Error(`Unknown recovery strategy: ${attempt.strategy}`);
}
}
private async executeRestartRecovery(attempt: RecoveryAttempt): Promise<void> {
attempt.logs.push('Executing restart recovery');
// Implementation would restart the component
await new Promise(resolve => setTimeout(resolve, 1000)); // Simulate restart
attempt.logs.push('Component restarted successfully');
}
private async executeRestoreRecovery(attempt: RecoveryAttempt): Promise<void> {
attempt.logs.push('Executing restore from backup recovery');
// Implementation would restore component from backup
await new Promise(resolve => setTimeout(resolve, 5000)); // Simulate restore
attempt.logs.push('Component restored from backup');
}
private async executeRebuildRecovery(attempt: RecoveryAttempt): Promise<void> {
attempt.logs.push('Executing rebuild recovery');
// Implementation would rebuild component from scratch
await new Promise(resolve => setTimeout(resolve, 10000)); // Simulate rebuild
attempt.logs.push('Component rebuilt successfully');
}
private async executeFailoverRecovery(attempt: RecoveryAttempt): Promise<void> {
attempt.logs.push('Executing failover recovery');
// Implementation would failover to backup instance
await new Promise(resolve => setTimeout(resolve, 3000)); // Simulate failover
attempt.logs.push('Failover completed successfully');
}
private async executeResetRecovery(attempt: RecoveryAttempt): Promise<void> {
attempt.logs.push('Executing reset to defaults recovery');
// Implementation would reset component to default configuration
await new Promise(resolve => setTimeout(resolve, 2000)); // Simulate reset
attempt.logs.push('Component reset to defaults');
}
private async captureComponentState(componentType: ComponentType): Promise<ComponentState> {
const health = this.componentHealth.get(componentType);
return {
type: componentType,
timestamp: new Date(),
status: health?.status || ComponentStatus.UNKNOWN,
configuration: {}, // Would capture actual configuration
metrics: health?.metrics || {},
errorLog: health?.errors || [],
lastAction: 'health_check',
dependencies: this.config.componentConfigs.get(componentType)?.dependencies || []
};
}
private async updateComponentHealth(componentType: ComponentType): Promise<void> {
await this.checkComponentHealth(componentType);
}
private recordRecoveryAttempt(attempt: RecoveryAttempt): void {
if (!this.recoveryHistory.has(attempt.componentType)) {
this.recoveryHistory.set(attempt.componentType, []);
}
const history = this.recoveryHistory.get(attempt.componentType)!;
history.push(attempt);
// Keep only recent attempts (last 20)
if (history.length > 20) {
history.splice(0, history.length - 20);
}
}
private getRecoveryCount(componentType: ComponentType): number {
const history = this.recoveryHistory.get(componentType) || [];
return history.filter(attempt => attempt.success).length;
}
private getTotalRecoveryCount(): number {
let total = 0;
for (const history of this.recoveryHistory.values()) {
total += history.filter(attempt => attempt.success).length;
}
return total;
}
private getDependentComponents(rootComponent: ComponentType): ComponentType[] {
const dependents: ComponentType[] = [];
for (const [componentType, config] of this.config.componentConfigs) {
if (config.dependencies.includes(rootComponent)) {
dependents.push(componentType);
}
}
return dependents;
}
private calculateRecoveryOrder(components: ComponentType[]): ComponentType[] {
// Topological sort based on dependency graph
const sorted: ComponentType[] = [];
const visited = new Set<ComponentType>();
const visiting = new Set<ComponentType>();
const visit = (component: ComponentType): void => {
if (visiting.has(component)) {
// Circular dependency detected
return;
}
if (visited.has(component)) {
return;
}
visiting.add(component);
const config = this.config.componentConfigs.get(component);
if (config) {
for (const dependency of config.dependencies) {
if (components.includes(dependency)) {
visit(dependency);
}
}
}
visiting.delete(component);
visited.add(component);
sorted.push(component);
};
for (const component of components) {
visit(component);
}
return sorted;
}
private validateComponentConfigs(): void {
for (const [componentType, config] of this.config.componentConfigs) {
if (!config.recoveryStrategies || config.recoveryStrategies.length === 0) {
throw new Error(`No recovery strategies defined for component: ${componentType}`);
}
// Validate dependencies exist
for (const dependency of config.dependencies) {
if (!this.config.componentConfigs.has(dependency)) {
throw new Error(`Unknown dependency ${dependency} for component ${componentType}`);
}
}
}
}
private generateRecoveryId(): string {
return `recovery_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
/**
* Cleanup resources
*/
async cleanup(): Promise<void> {
if (this.monitoringTimer) {
clearInterval(this.monitoringTimer);
this.monitoringTimer = undefined;
}
this.removeAllListeners();
}
}