import { EventEmitter } from 'events';
import { Logger } from '../utils/logger.js';
import { SessionState, SessionOptions } from '../types/index.js';
import * as fs from 'fs/promises';
import * as path from 'path';
import { v4 as uuidv4 } from 'uuid';
export interface RecoveryConfig {
enabled: boolean;
maxRecoveryAttempts: number;
recoveryDelay: number;
backoffMultiplier: number;
maxBackoffDelay: number;
persistenceEnabled: boolean;
persistencePath: string;
enableSmartRecovery: boolean;
enablePersistentSession: boolean;
snapshotInterval: number;
recoveryTimeout: number;
enableRecoveryMetrics: boolean;
}
export interface RecoverySnapshot {
sessionId: string;
timestamp: Date;
sessionState: SessionState;
sessionOptions: SessionOptions;
environment: Record<string, string>;
workingDirectory: string;
commandHistory: string[];
outputBuffer: string[];
errorContext?: {
lastError: string;
errorTimestamp: Date;
errorCount: number;
};
interactiveState?: {
isInteractive: boolean;
promptType?: string;
lastPromptDetected?: Date;
pendingCommands: string[];
sessionUnresponsive: boolean;
timeoutCount: number;
lastSuccessfulCommand?: Date;
};
}
export interface RecoveryAttempt {
sessionId: string;
attemptNumber: number;
startTime: Date;
endTime?: Date;
success: boolean;
error?: string;
strategy: RecoveryStrategy;
metrics: {
duration: number;
resourcesUsed: number;
dataRestored: number;
};
}
export type RecoveryStrategy =
| 'restart'
| 'reconnect'
| 'restore'
| 'replicate'
| 'migrate'
| 'fallback'
| 'prompt-interrupt'
| 'prompt-reset'
| 'session-refresh'
| 'command-retry';
export interface RecoveryPlan {
sessionId: string;
strategies: RecoveryStrategy[];
priority: 'high' | 'medium' | 'low';
estimatedTime: number;
requiredResources: string[];
fallbackOptions: string[];
interactiveContext?: {
promptTimeout: boolean;
commandInProgress: boolean;
lastKnownPrompt?: string;
unresponsiveTime?: number;
};
}
/**
* Automatic Session Recovery System
* Provides intelligent session restoration with multiple recovery strategies
*/
export class SessionRecovery extends EventEmitter {
private logger: Logger;
private config: RecoveryConfig;
private recoveryAttempts: Map<string, RecoveryAttempt[]> = new Map();
private sessionSnapshots: Map<string, RecoverySnapshot> = new Map();
private activeRecoveries: Set<string> = new Set();
private snapshotTimers: Map<string, NodeJS.Timeout> = new Map();
private isRunning = false;
// Recovery statistics
private stats = {
totalRecoveryAttempts: 0,
successfulRecoveries: 0,
failedRecoveries: 0,
averageRecoveryTime: 0,
sessionsSaved: 0,
sessionsRestored: 0,
bytesRestored: 0,
strategiesUsed: new Map<RecoveryStrategy, number>(),
lastRecovery: null as Date | null
};
constructor(config?: Partial<RecoveryConfig>) {
super();
this.logger = new Logger('SessionRecovery');
this.config = {
enabled: config?.enabled ?? true,
maxRecoveryAttempts: config?.maxRecoveryAttempts || 3,
recoveryDelay: config?.recoveryDelay || 5000,
backoffMultiplier: config?.backoffMultiplier || 2,
maxBackoffDelay: config?.maxBackoffDelay || 60000,
persistenceEnabled: config?.persistenceEnabled ?? true,
persistencePath: config?.persistencePath || './data/session-snapshots',
enableSmartRecovery: config?.enableSmartRecovery ?? true,
enablePersistentSession: config?.enablePersistentSession ?? true,
snapshotInterval: config?.snapshotInterval || 30000,
recoveryTimeout: config?.recoveryTimeout || 120000,
enableRecoveryMetrics: config?.enableRecoveryMetrics ?? true
};
this.logger.info('SessionRecovery initialized with config:', this.config);
}
/**
* Start the session recovery system
*/
async start(): Promise<void> {
if (this.isRunning) {
this.logger.warn('SessionRecovery is already running');
return;
}
this.logger.info('Starting SessionRecovery...');
this.isRunning = true;
// Ensure persistence directory exists
if (this.config.persistenceEnabled) {
try {
await fs.mkdir(this.config.persistencePath, { recursive: true });
} catch (error) {
this.logger.error('Failed to create persistence directory:', error);
}
}
// Load existing snapshots from disk
await this.loadPersistedSnapshots();
this.emit('started');
this.logger.info('SessionRecovery started successfully');
}
/**
* Stop the session recovery system
*/
async stop(): Promise<void> {
if (!this.isRunning) {
this.logger.warn('SessionRecovery is not running');
return;
}
this.logger.info('Stopping SessionRecovery...');
this.isRunning = false;
// Clear snapshot timers
for (const [sessionId, timer] of this.snapshotTimers) {
clearInterval(timer);
}
this.snapshotTimers.clear();
// Wait for active recoveries to complete
if (this.activeRecoveries.size > 0) {
this.logger.info(`Waiting for ${this.activeRecoveries.size} active recoveries to complete...`);
const timeout = setTimeout(() => {
this.logger.warn('Timeout waiting for active recoveries');
}, 30000);
while (this.activeRecoveries.size > 0) {
await this.delay(1000);
}
clearTimeout(timeout);
}
// Persist current snapshots
if (this.config.persistenceEnabled) {
await this.persistAllSnapshots();
}
this.emit('stopped');
this.logger.info('SessionRecovery stopped');
}
/**
* Register a session for recovery monitoring
*/
async registerSession(sessionId: string, sessionState: SessionState, sessionOptions: SessionOptions): Promise<void> {
if (!this.config.enabled) {
return;
}
this.logger.info(`Registering session ${sessionId} for recovery monitoring`);
// Create initial snapshot
const snapshot: RecoverySnapshot = {
sessionId,
timestamp: new Date(),
sessionState: { ...sessionState },
sessionOptions: { ...sessionOptions },
environment: { ...sessionOptions.env || {} },
workingDirectory: sessionOptions.cwd || process.cwd(),
commandHistory: [],
outputBuffer: []
};
this.sessionSnapshots.set(sessionId, snapshot);
this.stats.sessionsSaved++;
// Start periodic snapshotting
if (this.config.snapshotInterval > 0) {
this.startPeriodicSnapshots(sessionId);
}
// Persist snapshot if enabled
if (this.config.persistenceEnabled) {
await this.persistSnapshot(snapshot);
}
this.emit('session-registered', { sessionId, snapshot });
}
/**
* Unregister a session from recovery monitoring
*/
async unregisterSession(sessionId: string): Promise<void> {
this.logger.info(`Unregistering session ${sessionId} from recovery monitoring`);
// Clear snapshot timer
const timer = this.snapshotTimers.get(sessionId);
if (timer) {
clearInterval(timer);
this.snapshotTimers.delete(sessionId);
}
// Remove snapshot
this.sessionSnapshots.delete(sessionId);
// Remove persisted snapshot
if (this.config.persistenceEnabled) {
try {
const snapshotPath = path.join(this.config.persistencePath, `${sessionId}.snapshot.json`);
await fs.unlink(snapshotPath);
} catch (error) {
// Ignore if file doesn't exist
}
}
this.emit('session-unregistered', { sessionId });
}
/**
* Update session snapshot with new state
*/
async updateSessionSnapshot(
sessionId: string,
updates: Partial<Pick<RecoverySnapshot, 'sessionState' | 'environment' | 'workingDirectory' | 'commandHistory' | 'outputBuffer' | 'errorContext' | 'interactiveState'>>
): Promise<void> {
const snapshot = this.sessionSnapshots.get(sessionId);
if (!snapshot) {
return;
}
// Update snapshot
Object.assign(snapshot, updates);
snapshot.timestamp = new Date();
// Persist if enabled
if (this.config.persistenceEnabled) {
await this.persistSnapshot(snapshot);
}
this.emit('snapshot-updated', { sessionId, snapshot });
}
/**
* Attempt to recover a failed session
*/
async recoverSession(sessionId: string, failureReason?: string): Promise<boolean> {
if (!this.config.enabled || this.activeRecoveries.has(sessionId)) {
return false;
}
const snapshot = this.sessionSnapshots.get(sessionId);
if (!snapshot) {
this.logger.warn(`No snapshot found for session ${sessionId} - cannot recover`);
return false;
}
const existingAttempts = this.recoveryAttempts.get(sessionId) || [];
if (existingAttempts.length >= this.config.maxRecoveryAttempts) {
this.logger.warn(`Max recovery attempts (${this.config.maxRecoveryAttempts}) reached for session ${sessionId}`);
this.emit('recovery-max-attempts', { sessionId, attempts: existingAttempts.length });
return false;
}
this.activeRecoveries.add(sessionId);
const attemptNumber = existingAttempts.length + 1;
this.logger.info(`Starting recovery attempt ${attemptNumber}/${this.config.maxRecoveryAttempts} for session ${sessionId}`);
try {
// Generate recovery plan
const recoveryPlan = await this.generateRecoveryPlan(sessionId, failureReason);
this.emit('recovery-started', {
sessionId,
attemptNumber,
plan: recoveryPlan,
failureReason
});
// Execute recovery strategies
const success = await this.executeRecoveryPlan(sessionId, recoveryPlan, attemptNumber);
if (success) {
this.stats.successfulRecoveries++;
this.stats.lastRecovery = new Date();
this.logger.info(`Successfully recovered session ${sessionId} on attempt ${attemptNumber}`);
this.emit('recovery-success', {
sessionId,
attemptNumber,
plan: recoveryPlan
});
} else {
this.stats.failedRecoveries++;
this.logger.warn(`Failed to recover session ${sessionId} on attempt ${attemptNumber}`);
// Schedule next attempt with exponential backoff
if (attemptNumber < this.config.maxRecoveryAttempts) {
const delay = Math.min(
this.config.recoveryDelay * Math.pow(this.config.backoffMultiplier, attemptNumber - 1),
this.config.maxBackoffDelay
);
this.logger.info(`Scheduling next recovery attempt for session ${sessionId} in ${delay}ms`);
setTimeout(() => {
this.recoverSession(sessionId, failureReason);
}, delay);
}
}
return success;
} catch (error) {
this.logger.error(`Recovery attempt ${attemptNumber} failed for session ${sessionId}:`, error);
this.stats.failedRecoveries++;
this.emit('recovery-error', {
sessionId,
attemptNumber,
error: error instanceof Error ? error.message : String(error)
});
return false;
} finally {
this.activeRecoveries.delete(sessionId);
this.stats.totalRecoveryAttempts++;
}
}
/**
* Generate intelligent recovery plan based on failure analysis
*/
private async generateRecoveryPlan(sessionId: string, failureReason?: string): Promise<RecoveryPlan> {
const snapshot = this.sessionSnapshots.get(sessionId)!;
const sessionType = snapshot.sessionOptions.sshOptions ? 'ssh' : 'local';
let strategies: RecoveryStrategy[] = [];
let priority: 'high' | 'medium' | 'low' = 'medium';
let estimatedTime = 30000; // 30 seconds default
// Analyze failure reason to determine best recovery strategy
const reason = failureReason?.toLowerCase();
if (this.config.enableSmartRecovery && failureReason && reason) {
if (reason.includes('network') || reason.includes('connection')) {
strategies = sessionType === 'ssh'
? ['reconnect', 'restart', 'migrate', 'fallback']
: ['restart', 'restore', 'fallback'];
priority = 'high';
estimatedTime = sessionType === 'ssh' ? 45000 : 15000;
} else if (reason.includes('timeout') || reason.includes('unresponsive')) {
// Enhanced timeout handling with interactive prompt awareness
if (reason.includes('prompt') || reason.includes('interactive')) {
strategies = ['prompt-interrupt', 'prompt-reset', 'session-refresh', 'restart', 'fallback'];
priority = 'high';
estimatedTime = 15000;
} else {
strategies = ['restart', 'restore', 'replicate', 'fallback'];
priority = 'high';
estimatedTime = 20000;
}
} else if (reason.includes('memory') || reason.includes('resource')) {
strategies = ['migrate', 'restart', 'fallback'];
priority = 'medium';
estimatedTime = 60000;
} else if (reason.includes('permission') || reason.includes('auth')) {
strategies = ['reconnect', 'restore', 'fallback'];
priority = 'low';
estimatedTime = 30000;
} else {
// Generic failure
strategies = sessionType === 'ssh'
? ['reconnect', 'restart', 'restore', 'fallback']
: ['restart', 'restore', 'replicate', 'fallback'];
}
} else {
// Default recovery strategies
strategies = sessionType === 'ssh'
? ['reconnect', 'restart', 'restore', 'fallback']
: ['restart', 'restore', 'fallback'];
}
const requiredResources = [
'cpu',
sessionType === 'ssh' ? 'network' : 'local-process',
'memory'
];
const fallbackOptions = [
'Create new session with restored state',
'Notify user of session failure',
'Archive session for manual recovery'
];
// Add interactive context if dealing with prompt/timeout issues
let interactiveContext: RecoveryPlan['interactiveContext'];
if (reason && (reason.includes('timeout') || reason.includes('prompt') || reason.includes('interactive'))) {
const interactiveState = snapshot.interactiveState;
interactiveContext = {
promptTimeout: reason.includes('timeout'),
commandInProgress: interactiveState?.pendingCommands.length > 0 || false,
lastKnownPrompt: interactiveState?.promptType,
unresponsiveTime: interactiveState?.sessionUnresponsive ?
(Date.now() - (interactiveState.lastSuccessfulCommand?.getTime() || Date.now())) : undefined
};
}
return {
sessionId,
strategies,
priority,
estimatedTime,
requiredResources,
fallbackOptions,
interactiveContext
};
}
/**
* Execute recovery plan using the specified strategies
*/
private async executeRecoveryPlan(sessionId: string, plan: RecoveryPlan, attemptNumber: number): Promise<boolean> {
const startTime = Date.now();
const attempt: RecoveryAttempt = {
sessionId,
attemptNumber,
startTime: new Date(),
success: false,
strategy: plan.strategies[0], // Will be updated as we try strategies
metrics: {
duration: 0,
resourcesUsed: 0,
dataRestored: 0
}
};
// Store attempt
if (!this.recoveryAttempts.has(sessionId)) {
this.recoveryAttempts.set(sessionId, []);
}
this.recoveryAttempts.get(sessionId)!.push(attempt);
// Try each strategy in order
for (const strategy of plan.strategies) {
attempt.strategy = strategy;
this.stats.strategiesUsed.set(strategy, (this.stats.strategiesUsed.get(strategy) || 0) + 1);
this.logger.info(`Attempting recovery strategy '${strategy}' for session ${sessionId}`);
try {
const success = await this.executeRecoveryStrategy(sessionId, strategy);
if (success) {
attempt.success = true;
attempt.endTime = new Date();
attempt.metrics.duration = Date.now() - startTime;
// Update average recovery time
this.updateAverageRecoveryTime(attempt.metrics.duration);
this.logger.info(`Recovery strategy '${strategy}' succeeded for session ${sessionId}`);
return true;
} else {
this.logger.warn(`Recovery strategy '${strategy}' failed for session ${sessionId}`);
}
} catch (error) {
this.logger.error(`Recovery strategy '${strategy}' error for session ${sessionId}:`, error);
}
}
// All strategies failed
attempt.endTime = new Date();
attempt.metrics.duration = Date.now() - startTime;
attempt.error = 'All recovery strategies failed';
return false;
}
/**
* Execute a specific recovery strategy
*/
private async executeRecoveryStrategy(sessionId: string, strategy: RecoveryStrategy): Promise<boolean> {
const snapshot = this.sessionSnapshots.get(sessionId);
if (!snapshot) return false;
switch (strategy) {
case 'restart':
return await this.executeRestartStrategy(sessionId, snapshot);
case 'reconnect':
return await this.executeReconnectStrategy(sessionId, snapshot);
case 'restore':
return await this.executeRestoreStrategy(sessionId, snapshot);
case 'replicate':
return await this.executeReplicateStrategy(sessionId, snapshot);
case 'migrate':
return await this.executeMigrateStrategy(sessionId, snapshot);
case 'fallback':
return await this.executeFallbackStrategy(sessionId, snapshot);
case 'prompt-interrupt':
return await this.executePromptInterruptStrategy(sessionId, snapshot);
case 'prompt-reset':
return await this.executePromptResetStrategy(sessionId, snapshot);
case 'session-refresh':
return await this.executeSessionRefreshStrategy(sessionId, snapshot);
case 'command-retry':
return await this.executeCommandRetryStrategy(sessionId, snapshot);
default:
this.logger.warn(`Unknown recovery strategy: ${strategy}`);
return false;
}
}
/**
* Restart the session with the same configuration
*/
private async executeRestartStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'restart',
message: 'Restarting session with same configuration'
});
// Request session restart with original options
this.emit('session-restart-request', {
sessionId,
sessionOptions: snapshot.sessionOptions,
restoreState: {
workingDirectory: snapshot.workingDirectory,
environment: snapshot.environment
}
});
return true; // Assume success - actual verification would come from session manager
} catch (error) {
this.logger.error(`Restart strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Reconnect to existing session (SSH only)
*/
private async executeReconnectStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
if (!snapshot.sessionOptions.sshOptions) {
return false; // Not applicable for local sessions
}
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'reconnect',
message: 'Attempting to reconnect to SSH session'
});
// Request SSH reconnection
this.emit('ssh-reconnect-request', {
sessionId,
sshOptions: snapshot.sessionOptions.sshOptions
});
return true;
} catch (error) {
this.logger.error(`Reconnect strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Restore session state from snapshot
*/
private async executeRestoreStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'restore',
message: 'Restoring session from snapshot data'
});
// Restore session state
this.emit('session-restore-request', {
sessionId,
snapshot: {
sessionState: snapshot.sessionState,
environment: snapshot.environment,
workingDirectory: snapshot.workingDirectory,
commandHistory: snapshot.commandHistory
}
});
this.stats.sessionsRestored++;
this.stats.bytesRestored += this.estimateSnapshotSize(snapshot);
return true;
} catch (error) {
this.logger.error(`Restore strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Replicate session on different resources
*/
private async executeReplicateStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'replicate',
message: 'Replicating session on alternative resources'
});
// Create new session with replicated state
const newSessionId = `${sessionId}-replica-${Date.now()}`;
this.emit('session-replicate-request', {
originalSessionId: sessionId,
newSessionId,
sessionOptions: snapshot.sessionOptions,
restoreState: snapshot
});
return true;
} catch (error) {
this.logger.error(`Replicate strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Migrate session to different host/resource
*/
private async executeMigrateStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'migrate',
message: 'Migrating session to alternative host'
});
// Request session migration
this.emit('session-migrate-request', {
sessionId,
currentSnapshot: snapshot,
migrationOptions: {
preferredHosts: [], // Would be populated based on configuration
resourceRequirements: snapshot.sessionOptions
}
});
return true;
} catch (error) {
this.logger.error(`Migrate strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Execute fallback strategy (last resort)
*/
private async executeFallbackStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'fallback',
message: 'Executing fallback recovery - notifying user and archiving session'
});
// Archive session data for manual recovery
if (this.config.persistenceEnabled) {
const archivePath = path.join(this.config.persistencePath, 'failed-sessions', `${sessionId}-${Date.now()}.archive.json`);
await fs.mkdir(path.dirname(archivePath), { recursive: true });
await fs.writeFile(archivePath, JSON.stringify(snapshot, null, 2));
}
// Notify about failed recovery
this.emit('recovery-fallback', {
sessionId,
snapshot,
message: 'Session could not be automatically recovered. Data has been archived for manual recovery.',
archiveLocation: this.config.persistenceEnabled ? 'failed-sessions' : null
});
return true; // Fallback always "succeeds" by definition
} catch (error) {
this.logger.error(`Fallback strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Start periodic snapshots for a session
*/
private startPeriodicSnapshots(sessionId: string): void {
const timer = setInterval(() => {
// Request updated session state
this.emit('snapshot-request', {
sessionId,
callback: (updates?: Partial<RecoverySnapshot>) => {
if (updates) {
this.updateSessionSnapshot(sessionId, updates);
}
}
});
}, this.config.snapshotInterval);
this.snapshotTimers.set(sessionId, timer);
}
/**
* Persist a snapshot to disk
*/
private async persistSnapshot(snapshot: RecoverySnapshot): Promise<void> {
if (!this.config.persistenceEnabled) return;
try {
const snapshotPath = path.join(this.config.persistencePath, `${snapshot.sessionId}.snapshot.json`);
await fs.writeFile(snapshotPath, JSON.stringify(snapshot, null, 2));
} catch (error) {
this.logger.error(`Failed to persist snapshot for session ${snapshot.sessionId}:`, error);
}
}
/**
* Load persisted snapshots from disk
*/
private async loadPersistedSnapshots(): Promise<void> {
if (!this.config.persistenceEnabled) return;
try {
const files = await fs.readdir(this.config.persistencePath);
const snapshotFiles = files.filter(f => f.endsWith('.snapshot.json'));
for (const file of snapshotFiles) {
try {
const filePath = path.join(this.config.persistencePath, file);
const content = await fs.readFile(filePath, 'utf-8');
const snapshot: RecoverySnapshot = JSON.parse(content);
// Convert date strings back to Date objects
snapshot.timestamp = new Date(snapshot.timestamp);
if (snapshot.errorContext) {
snapshot.errorContext.errorTimestamp = new Date(snapshot.errorContext.errorTimestamp);
}
this.sessionSnapshots.set(snapshot.sessionId, snapshot);
this.logger.debug(`Loaded snapshot for session ${snapshot.sessionId}`);
} catch (error) {
this.logger.error(`Failed to load snapshot from ${file}:`, error);
}
}
this.logger.info(`Loaded ${snapshotFiles.length} persisted snapshots`);
} catch (error) {
this.logger.error('Failed to load persisted snapshots:', error);
}
}
/**
* Persist all current snapshots
*/
private async persistAllSnapshots(): Promise<void> {
const snapshots = Array.from(this.sessionSnapshots.values());
await Promise.all(snapshots.map(snapshot => this.persistSnapshot(snapshot)));
this.logger.info(`Persisted ${snapshots.length} snapshots`);
}
/**
* Estimate the size of a snapshot in bytes
*/
private estimateSnapshotSize(snapshot: RecoverySnapshot): number {
return JSON.stringify(snapshot).length * 2; // Rough estimate (UTF-16)
}
/**
* Update average recovery time metric
*/
private updateAverageRecoveryTime(duration: number): void {
if (this.stats.successfulRecoveries === 1) {
this.stats.averageRecoveryTime = duration;
} else {
this.stats.averageRecoveryTime =
((this.stats.averageRecoveryTime * (this.stats.successfulRecoveries - 1)) + duration) /
this.stats.successfulRecoveries;
}
}
/**
* Get recovery statistics
*/
getRecoveryStatistics() {
return {
...this.stats,
activeRecoveries: this.activeRecoveries.size,
registeredSessions: this.sessionSnapshots.size,
strategiesUsed: Object.fromEntries(this.stats.strategiesUsed),
successRate: this.stats.totalRecoveryAttempts > 0 ?
(this.stats.successfulRecoveries / this.stats.totalRecoveryAttempts) * 100 : 0,
config: this.config,
isRunning: this.isRunning
};
}
/**
* Get recovery history for a specific session
*/
getSessionRecoveryHistory(sessionId: string): RecoveryAttempt[] {
return this.recoveryAttempts.get(sessionId) || [];
}
/**
* Get all registered snapshots
*/
getAllSnapshots(): Record<string, RecoverySnapshot> {
const result: Record<string, RecoverySnapshot> = {};
for (const [sessionId, snapshot] of this.sessionSnapshots) {
result[sessionId] = { ...snapshot };
}
return result;
}
/**
* Utility method for delays
*/
private delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Execute prompt interrupt strategy - interrupt stuck interactive prompts
*/
private async executePromptInterruptStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'prompt-interrupt',
message: 'Interrupting stuck interactive prompt'
});
// Send interrupt signals to break out of stuck prompts
this.emit('session-interrupt-request', {
sessionId,
interruptType: 'prompt',
signals: ['SIGINT', 'CTRL_C', 'ESC'],
interactiveState: snapshot.interactiveState
});
return true;
} catch (error) {
this.logger.error(`Prompt interrupt strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Execute prompt reset strategy - reset prompt state and clear buffers
*/
private async executePromptResetStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'prompt-reset',
message: 'Resetting prompt state and clearing buffers'
});
// Clear output buffers and reset prompt detection
this.emit('session-prompt-reset-request', {
sessionId,
actions: [
'clear-output-buffer',
'reset-prompt-detector',
'flush-pending-commands',
'reinitialize-prompt-patterns'
],
preserveState: {
workingDirectory: snapshot.workingDirectory,
environment: snapshot.environment
}
});
return true;
} catch (error) {
this.logger.error(`Prompt reset strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Execute session refresh strategy - refresh session without full restart
*/
private async executeSessionRefreshStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'session-refresh',
message: 'Refreshing session state without full restart'
});
// Send refresh command (like newline or space) to re-establish communication
this.emit('session-refresh-request', {
sessionId,
refreshActions: [
'send-newline',
'check-responsiveness',
'verify-prompt',
'restore-context'
],
timeout: 10000,
fallbackToRestart: true,
preserveState: snapshot.interactiveState
});
return true;
} catch (error) {
this.logger.error(`Session refresh strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Execute command retry strategy - retry failed commands with exponential backoff
*/
private async executeCommandRetryStrategy(sessionId: string, snapshot: RecoverySnapshot): Promise<boolean> {
try {
this.emit('recovery-strategy-attempt', {
sessionId,
strategy: 'command-retry',
message: 'Retrying failed commands with intelligent backoff'
});
const interactiveState = snapshot.interactiveState;
if (!interactiveState?.pendingCommands.length) {
this.logger.info(`No pending commands to retry for session ${sessionId}`);
return true; // Success - nothing to retry
}
// Retry pending commands with exponential backoff
this.emit('session-command-retry-request', {
sessionId,
commands: interactiveState.pendingCommands,
retryConfig: {
maxRetries: 3,
baseDelay: 1000,
backoffMultiplier: 2,
maxDelay: 10000,
jitter: true
},
verification: {
checkPrompt: true,
timeoutPerCommand: 15000,
verifyOutput: true
}
});
return true;
} catch (error) {
this.logger.error(`Command retry strategy failed for session ${sessionId}:`, error);
return false;
}
}
/**
* Update interactive state for a session
*/
async updateInteractiveState(
sessionId: string,
interactiveUpdates: Partial<RecoverySnapshot['interactiveState']>
): Promise<void> {
const snapshot = this.sessionSnapshots.get(sessionId);
if (!snapshot) {
return;
}
// Initialize interactive state if it doesn't exist
if (!snapshot.interactiveState) {
snapshot.interactiveState = {
isInteractive: false,
pendingCommands: [],
sessionUnresponsive: false,
timeoutCount: 0
};
}
// Update interactive state
Object.assign(snapshot.interactiveState, interactiveUpdates);
snapshot.timestamp = new Date();
// Persist if enabled
if (this.config.persistenceEnabled) {
await this.persistSnapshot(snapshot);
}
this.emit('interactive-state-updated', { sessionId, interactiveState: snapshot.interactiveState });
}
/**
* Check if session needs interactive prompt recovery
*/
shouldTriggerInteractiveRecovery(sessionId: string): {
shouldTrigger: boolean;
reason?: string;
urgency: 'low' | 'medium' | 'high';
} {
const snapshot = this.sessionSnapshots.get(sessionId);
if (!snapshot?.interactiveState) {
return { shouldTrigger: false, urgency: 'low' };
}
const state = snapshot.interactiveState;
const now = Date.now();
// Check for timeout conditions
if (state.sessionUnresponsive) {
const unresponsiveTime = state.lastSuccessfulCommand ?
now - state.lastSuccessfulCommand.getTime() : 0;
if (unresponsiveTime > 30000) { // 30 seconds
return {
shouldTrigger: true,
reason: 'Session unresponsive for extended period',
urgency: 'high'
};
}
}
// Check for excessive timeouts
if (state.timeoutCount >= 3) {
return {
shouldTrigger: true,
reason: 'Multiple consecutive timeouts detected',
urgency: 'high'
};
}
// Check for stuck interactive prompts
if (state.isInteractive && state.lastPromptDetected) {
const promptAge = now - state.lastPromptDetected.getTime();
if (promptAge > 60000) { // 1 minute
return {
shouldTrigger: true,
reason: 'Interactive prompt appears stuck',
urgency: 'medium'
};
}
}
// Check for pending commands that haven't been processed
if (state.pendingCommands.length > 0) {
const commandAge = state.lastSuccessfulCommand ?
now - state.lastSuccessfulCommand.getTime() : now;
if (commandAge > 45000) { // 45 seconds
return {
shouldTrigger: true,
reason: 'Pending commands not processing',
urgency: 'medium'
};
}
}
return { shouldTrigger: false, urgency: 'low' };
}
/**
* Get interactive recovery statistics
*/
getInteractiveRecoveryStats(): {
totalInteractiveSessions: number;
sessionsWithTimeouts: number;
unresponsiveSessions: number;
averageTimeoutCount: number;
successfulPromptInterrupts: number;
successfulPromptResets: number;
} {
const snapshots = Array.from(this.sessionSnapshots.values());
const interactiveSessions = snapshots.filter(s => s.interactiveState?.isInteractive);
const sessionsWithTimeouts = interactiveSessions.filter(s => s.interactiveState!.timeoutCount > 0);
const unresponsiveSessions = interactiveSessions.filter(s => s.interactiveState!.sessionUnresponsive);
const totalTimeouts = interactiveSessions.reduce((sum, s) => sum + (s.interactiveState!.timeoutCount || 0), 0);
const averageTimeoutCount = interactiveSessions.length > 0 ? totalTimeouts / interactiveSessions.length : 0;
// Get strategy usage stats
const promptInterrupts = this.stats.strategiesUsed.get('prompt-interrupt') || 0;
const promptResets = this.stats.strategiesUsed.get('prompt-reset') || 0;
return {
totalInteractiveSessions: interactiveSessions.length,
sessionsWithTimeouts: sessionsWithTimeouts.length,
unresponsiveSessions: unresponsiveSessions.length,
averageTimeoutCount,
successfulPromptInterrupts: promptInterrupts,
successfulPromptResets: promptResets
};
}
/**
* Clean up resources
*/
async destroy(): Promise<void> {
await this.stop();
this.sessionSnapshots.clear();
this.recoveryAttempts.clear();
this.removeAllListeners();
this.logger.info('SessionRecovery destroyed');
}
}
