MCP Console Automation Server

import { EventEmitter } from 'events'; import { Logger } from '../utils/logger.js'; import { ConsoleSession, SessionState, SSHConnectionOptions, } from '../types/index.js'; import { PersistentSessionStorage, PersistentSessionData, } from './PersistentSessionStorage.js'; export interface ReconnectionConfig { enabled: boolean; maxRetries: number; baseDelay: number; maxDelay: number; backoffMultiplier: number; healthCheckInterval: number; connectionTimeout: number; keepAliveInterval: number; autoReconnect: boolean; preserveState: boolean; } export interface ReconnectionAttempt { sessionId: string; attemptNumber: number; startTime: Date; endTime?: Date; success: boolean; error?: string; strategy: ReconnectionStrategy; duration: number; metadata?: Record<string, any>; } export type ReconnectionStrategy = | 'simple-reconnect' | 'session-restore' | 'process-restart' | 'ssh-reconnect' | 'state-recovery' | 'force-reconnect'; export interface ReconnectionState { sessionId: string; isReconnecting: boolean; attempts: ReconnectionAttempt[]; lastSuccessfulConnection: Date | null; lastFailure: Date | null; nextRetryTime: Date | null; strategy: ReconnectionStrategy; preservedState?: { workingDirectory: string; environment: Record<string, string>; lastCommand?: string; }; } /** * Automatic Session Reconnection System * Handles network disconnections and provides seamless session restoration */ export class SessionReconnection extends EventEmitter { private config: ReconnectionConfig; private logger: Logger; private storage: PersistentSessionStorage; private reconnectionStates: Map<string, ReconnectionState> = new Map(); private healthCheckTimer: NodeJS.Timeout | null = null; private activeReconnections: Set<string> = new Set(); // Statistics private stats = { totalAttempts: 0, successfulReconnections: 0, failedReconnections: 0, averageReconnectionTime: 0, networkDisconnections: 0, processRestarts: 0, lastReconnection: null as Date | null, }; constructor( storage: PersistentSessionStorage, config?: Partial<ReconnectionConfig> ) { super(); this.logger = new Logger('SessionReconnection'); this.storage = storage; this.config = { enabled: config?.enabled ?? true, maxRetries: config?.maxRetries || 5, baseDelay: config?.baseDelay || 2000, maxDelay: config?.maxDelay || 30000, backoffMultiplier: config?.backoffMultiplier || 1.5, healthCheckInterval: config?.healthCheckInterval || 30000, connectionTimeout: config?.connectionTimeout || 15000, keepAliveInterval: config?.keepAliveInterval || 60000, autoReconnect: config?.autoReconnect ?? true, preserveState: config?.preserveState ?? true, }; this.logger.info( 'SessionReconnection initialized with config:', this.config ); } /** * Start the reconnection monitoring system */ async start(): Promise<void> { if (!this.config.enabled) { this.logger.info('Session reconnection is disabled'); return; } this.logger.info('Starting session reconnection monitoring...'); // Start health check monitoring this.startHealthCheck(); // Load existing reconnection states await this.loadReconnectionStates(); this.emit('started'); this.logger.info('Session reconnection monitoring started'); } /** * Stop the reconnection system */ async stop(): Promise<void> { this.logger.info('Stopping session reconnection...'); if (this.healthCheckTimer) { clearInterval(this.healthCheckTimer); this.healthCheckTimer = null; } // Wait for active reconnections to complete if (this.activeReconnections.size > 0) { this.logger.info( `Waiting for ${this.activeReconnections.size} active reconnections to complete...` ); const timeout = setTimeout(() => { this.logger.warn('Timeout waiting for active reconnections'); }, 30000); while (this.activeReconnections.size > 0) { await this.delay(1000); } clearTimeout(timeout); } this.emit('stopped'); this.logger.info('Session reconnection stopped'); } /** * Register a session for reconnection monitoring */ async registerSession( sessionId: string, session: ConsoleSession ): Promise<void> { if (!this.config.enabled) { return; } this.logger.info( `Registering session ${sessionId} for reconnection monitoring` ); const reconnectionState: ReconnectionState = { sessionId, isReconnecting: false, attempts: [], lastSuccessfulConnection: new Date(), lastFailure: null, nextRetryTime: null, strategy: 'simple-reconnect', preservedState: this.config.preserveState ? { workingDirectory: session.cwd, environment: session.env || {}, lastCommand: session.command, } : undefined, }; this.reconnectionStates.set(sessionId, reconnectionState); this.emit('session-registered', { sessionId, state: reconnectionState }); } /** * Unregister a session from reconnection monitoring */ async unregisterSession(sessionId: string): Promise<void> { this.logger.info( `Unregistering session ${sessionId} from reconnection monitoring` ); const state = this.reconnectionStates.get(sessionId); if (state?.isReconnecting) { this.logger.warn( `Session ${sessionId} is currently reconnecting - marking for cleanup` ); } this.reconnectionStates.delete(sessionId); this.activeReconnections.delete(sessionId); this.emit('session-unregistered', { sessionId }); } /** * Handle session disconnection */ async handleDisconnection( sessionId: string, reason: string, error?: Error ): Promise<void> { if (!this.config.enabled || !this.config.autoReconnect) { return; } const state = this.reconnectionStates.get(sessionId); if (!state) { this.logger.warn(`No reconnection state found for session ${sessionId}`); return; } if (state.isReconnecting) { this.logger.debug(`Session ${sessionId} is already reconnecting`); return; } this.logger.warn(`Session ${sessionId} disconnected: ${reason}`); state.lastFailure = new Date(); this.stats.networkDisconnections++; // Determine reconnection strategy based on disconnect reason const strategy = this.determineReconnectionStrategy(reason, error); state.strategy = strategy; this.emit('session-disconnected', { sessionId, reason, error: error?.message, strategy, }); // Start reconnection process await this.attemptReconnection(sessionId); } /** * Manually trigger reconnection for a session */ async forceReconnection( sessionId: string, strategy?: ReconnectionStrategy ): Promise<boolean> { const state = this.reconnectionStates.get(sessionId); if (!state) { throw new Error(`No reconnection state found for session ${sessionId}`); } if (strategy) { state.strategy = strategy; } this.logger.info( `Forcing reconnection for session ${sessionId} using strategy: ${state.strategy}` ); return await this.attemptReconnection(sessionId); } /** * Get reconnection state for a session */ getReconnectionState(sessionId: string): ReconnectionState | undefined { return this.reconnectionStates.get(sessionId); } /** * Get reconnection statistics */ getReconnectionStats() { return { ...this.stats, activeReconnections: this.activeReconnections.size, registeredSessions: this.reconnectionStates.size, successRate: this.stats.totalAttempts > 0 ? (this.stats.successfulReconnections / this.stats.totalAttempts) * 100 : 0, }; } /** * Get all reconnection states */ getAllReconnectionStates(): Record<string, ReconnectionState> { const result: Record<string, ReconnectionState> = {}; for (const [sessionId, state] of this.reconnectionStates) { result[sessionId] = { ...state }; } return result; } // Private methods private async attemptReconnection(sessionId: string): Promise<boolean> { const state = this.reconnectionStates.get(sessionId); if (!state || state.isReconnecting) { return false; } // Check if max retries exceeded if (state.attempts.length >= this.config.maxRetries) { this.logger.warn( `Max reconnection attempts (${this.config.maxRetries}) exceeded for session ${sessionId}` ); this.emit('reconnection-max-attempts', { sessionId, attempts: state.attempts.length, }); return false; } state.isReconnecting = true; this.activeReconnections.add(sessionId); const attemptNumber = state.attempts.length + 1; this.logger.info( `Starting reconnection attempt ${attemptNumber}/${this.config.maxRetries} for session ${sessionId}` ); const attempt: ReconnectionAttempt = { sessionId, attemptNumber, startTime: new Date(), success: false, strategy: state.strategy, duration: 0, }; try { this.emit('reconnection-started', { sessionId, attemptNumber, strategy: state.strategy, }); // Execute reconnection strategy const success = await this.executeReconnectionStrategy( sessionId, state.strategy ); attempt.success = success; attempt.endTime = new Date(); attempt.duration = attempt.endTime.getTime() - attempt.startTime.getTime(); if (success) { state.lastSuccessfulConnection = new Date(); state.nextRetryTime = null; this.stats.successfulReconnections++; this.stats.lastReconnection = new Date(); this.updateAverageReconnectionTime(attempt.duration); this.logger.info( `Successfully reconnected session ${sessionId} on attempt ${attemptNumber}` ); this.emit('reconnection-success', { sessionId, attemptNumber, duration: attempt.duration, strategy: state.strategy, }); } else { this.stats.failedReconnections++; this.logger.warn( `Failed to reconnect session ${sessionId} on attempt ${attemptNumber}` ); // Schedule next retry with exponential backoff if (attemptNumber < this.config.maxRetries) { const delay = Math.min( this.config.baseDelay * Math.pow(this.config.backoffMultiplier, attemptNumber - 1), this.config.maxDelay ); state.nextRetryTime = new Date(Date.now() + delay); this.logger.info( `Scheduling next reconnection attempt for session ${sessionId} in ${delay}ms` ); setTimeout(() => { this.attemptReconnection(sessionId); }, delay); } } state.attempts.push(attempt); this.stats.totalAttempts++; return success; } catch (error) { attempt.success = false; attempt.error = error instanceof Error ? error.message : String(error); attempt.endTime = new Date(); attempt.duration = attempt.endTime.getTime() - attempt.startTime.getTime(); state.attempts.push(attempt); this.stats.totalAttempts++; this.stats.failedReconnections++; this.logger.error( `Reconnection attempt ${attemptNumber} failed for session ${sessionId}:`, error ); this.emit('reconnection-error', { sessionId, attemptNumber, error: error instanceof Error ? error.message : String(error), }); return false; } finally { state.isReconnecting = false; this.activeReconnections.delete(sessionId); } } private async executeReconnectionStrategy( sessionId: string, strategy: ReconnectionStrategy ): Promise<boolean> { switch (strategy) { case 'simple-reconnect': return await this.executeSimpleReconnect(sessionId); case 'session-restore': return await this.executeSessionRestore(sessionId); case 'process-restart': return await this.executeProcessRestart(sessionId); case 'ssh-reconnect': return await this.executeSSHReconnect(sessionId); case 'state-recovery': return await this.executeStateRecovery(sessionId); case 'force-reconnect': return await this.executeForceReconnect(sessionId); default: this.logger.warn(`Unknown reconnection strategy: ${strategy}`); return false; } } private async executeSimpleReconnect(sessionId: string): Promise<boolean> { try { this.emit('reconnection-strategy-attempt', { sessionId, strategy: 'simple-reconnect', message: 'Attempting simple reconnection', }); // Request simple reconnection from session manager this.emit('reconnection-request', { sessionId, type: 'simple-reconnect', }); return true; // Assume success - actual verification would come from session manager } catch (error) { this.logger.error( `Simple reconnect failed for session ${sessionId}:`, error ); return false; } } private async executeSessionRestore(sessionId: string): Promise<boolean> { try { this.emit('reconnection-strategy-attempt', { sessionId, strategy: 'session-restore', message: 'Restoring session from persistent storage', }); // Get persistent session data const sessionData = await this.storage.retrieveSession(sessionId); if (!sessionData) { this.logger.warn(`No persistent data found for session ${sessionId}`); return false; } // Request session restoration this.emit('session-restore-request', { sessionId, sessionData, preserveState: this.config.preserveState, }); return true; } catch (error) { this.logger.error( `Session restore failed for session ${sessionId}:`, error ); return false; } } private async executeProcessRestart(sessionId: string): Promise<boolean> { try { this.emit('reconnection-strategy-attempt', { sessionId, strategy: 'process-restart', message: 'Restarting process with preserved state', }); const state = this.reconnectionStates.get(sessionId); const sessionData = await this.storage.retrieveSession(sessionId); // Request process restart this.emit('process-restart-request', { sessionId, sessionData, preservedState: state?.preservedState, }); this.stats.processRestarts++; return true; } catch (error) { this.logger.error( `Process restart failed for session ${sessionId}:`, error ); return false; } } private async executeSSHReconnect(sessionId: string): Promise<boolean> { try { this.emit('reconnection-strategy-attempt', { sessionId, strategy: 'ssh-reconnect', message: 'Reconnecting SSH session', }); const sessionData = await this.storage.retrieveSession(sessionId); if (!sessionData?.originalSession.sshOptions) { return false; // Not an SSH session } // Request SSH reconnection this.emit('ssh-reconnect-request', { sessionId, sshOptions: sessionData.originalSession.sshOptions, preservedState: this.reconnectionStates.get(sessionId)?.preservedState, }); return true; } catch (error) { this.logger.error( `SSH reconnect failed for session ${sessionId}:`, error ); return false; } } private async executeStateRecovery(sessionId: string): Promise<boolean> { try { this.emit('reconnection-strategy-attempt', { sessionId, strategy: 'state-recovery', message: 'Recovering session state and context', }); const sessionData = await this.storage.retrieveSession(sessionId); const state = this.reconnectionStates.get(sessionId); if (!sessionData) { return false; } // Request comprehensive state recovery this.emit('state-recovery-request', { sessionId, sessionData, preservedState: state?.preservedState, outputHistory: this.storage.getSessionHistory(sessionId), }); return true; } catch (error) { this.logger.error( `State recovery failed for session ${sessionId}:`, error ); return false; } } private async executeForceReconnect(sessionId: string): Promise<boolean> { try { this.emit('reconnection-strategy-attempt', { sessionId, strategy: 'force-reconnect', message: 'Forcing reconnection with aggressive recovery', }); // Force reconnection by trying multiple strategies const strategies: ReconnectionStrategy[] = [ 'simple-reconnect', 'session-restore', 'process-restart', ]; for (const strategy of strategies) { const success = await this.executeReconnectionStrategy( sessionId, strategy ); if (success) { return true; } // Wait between attempts await this.delay(1000); } return false; } catch (error) { this.logger.error( `Force reconnect failed for session ${sessionId}:`, error ); return false; } } private determineReconnectionStrategy( reason: string, error?: Error ): ReconnectionStrategy { const reasonLower = reason.toLowerCase(); const errorMessage = error?.message?.toLowerCase() || ''; // Network-related disconnections if ( reasonLower.includes('network') || reasonLower.includes('connection') || errorMessage.includes('econnreset') || errorMessage.includes('enotfound') ) { return 'simple-reconnect'; } // SSH-specific issues if (reasonLower.includes('ssh') || errorMessage.includes('ssh')) { return 'ssh-reconnect'; } // Process-related issues if ( reasonLower.includes('process') || reasonLower.includes('killed') || errorMessage.includes('spawn') || errorMessage.includes('exit') ) { return 'process-restart'; } // Timeout issues if (reasonLower.includes('timeout') || errorMessage.includes('timeout')) { return 'session-restore'; } // State corruption if (reasonLower.includes('state') || reasonLower.includes('corrupt')) { return 'state-recovery'; } // Default strategy return 'simple-reconnect'; } private startHealthCheck(): void { if (this.config.healthCheckInterval <= 0) { return; } this.healthCheckTimer = setInterval(async () => { await this.performHealthCheck(); }, this.config.healthCheckInterval); } private async performHealthCheck(): Promise<void> { const now = Date.now(); for (const [sessionId, state] of this.reconnectionStates.entries()) { // Check for pending retries if ( state.nextRetryTime && now >= state.nextRetryTime.getTime() && !state.isReconnecting ) { this.logger.debug( `Triggering scheduled reconnection for session ${sessionId}` ); await this.attemptReconnection(sessionId); } // Request health check from external systems this.emit('health-check-request', { sessionId, state }); } } private async loadReconnectionStates(): Promise<void> { // Load any persistent reconnection states if needed // This could be implemented to restore reconnection states across process restarts this.logger.debug('Loading reconnection states...'); } private updateAverageReconnectionTime(duration: number): void { if (this.stats.successfulReconnections === 1) { this.stats.averageReconnectionTime = duration; } else { this.stats.averageReconnectionTime = (this.stats.averageReconnectionTime * (this.stats.successfulReconnections - 1) + duration) / this.stats.successfulReconnections; } } private delay(ms: number): Promise<void> { return new Promise((resolve) => setTimeout(resolve, ms)); } }