disaster-recovery.test.ts•23 kB
/**
* Comprehensive Test Suite for GEPA Disaster Recovery Systems
*
* Tests all disaster recovery components:
* - State Backup Manager
* - Disaster Recovery Manager
* - Component Recovery Manager
* - Data Integrity Manager
* - Recovery Orchestrator
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import {
DisasterRecoverySystem,
StateBackupManager,
DisasterRecoveryManager,
ComponentRecoveryManager,
DataIntegrityManager,
RecoveryOrchestrator,
DisasterType,
ComponentType,
CorruptionLevel,
ChecksumType,
RecoveryStrategy
} from './index';
// Mock file system operations
vi.mock('fs', () => ({
existsSync: vi.fn(() => true),
mkdirSync: vi.fn(),
writeFileSync: vi.fn(),
readFileSync: vi.fn(() => 'mock file content'),
unlinkSync: vi.fn(),
readdirSync: vi.fn(() => []),
statSync: vi.fn(() => ({ size: 1024 }))
}));
describe('GEPA Disaster Recovery System', () => {
let recoverySystem: DisasterRecoverySystem;
beforeEach(() => {
recoverySystem = DisasterRecoverySystem.getInstance();
});
afterEach(async () => {
await recoverySystem.cleanup();
vi.clearAllMocks();
});
describe('Disaster Recovery System Integration', () => {
it('should initialize all recovery components', async () => {
await expect(recoverySystem.initialize()).resolves.not.toThrow();
});
it('should create system backup successfully', async () => {
const backup = await recoverySystem.createSystemBackup('test-backup');
expect(backup).toMatchObject({
id: expect.any(String),
timestamp: expect.any(Date),
label: 'test-backup',
components: expect.any(Array)
});
});
it('should restore system from backup', async () => {
const backup = await recoverySystem.createSystemBackup('test-backup');
const result = await recoverySystem.restoreSystemFromBackup(backup.id);
expect(result).toMatchObject({
success: expect.any(Boolean),
backupId: backup.id,
restoreTime: expect.any(Number)
});
});
it('should execute disaster recovery procedure', async () => {
const execution = await recoverySystem.executeDisasterRecovery(DisasterType.MEMORY_EXHAUSTION);
expect(execution).toMatchObject({
id: expect.any(String),
disasterEvent: expect.objectContaining({
type: DisasterType.MEMORY_EXHAUSTION
}),
procedure: expect.any(Object)
});
});
it('should perform comprehensive health check', async () => {
const health = await recoverySystem.performHealthCheck();
expect(health).toMatchObject({
overall: expect.stringMatching(/healthy|degraded|critical/),
systems: expect.any(Object),
recommendations: expect.any(Array)
});
});
});
describe('State Backup Manager', () => {
let backupManager: StateBackupManager;
beforeEach(() => {
backupManager = new StateBackupManager({
backupDirectory: './test-backups',
maxBackups: 10,
compressionEnabled: true
});
});
afterEach(async () => {
await backupManager.cleanup();
});
it('should initialize backup manager', async () => {
await expect(backupManager.initialize()).resolves.not.toThrow();
});
it('should create evolution state backup', async () => {
await backupManager.initialize();
const evolutionState = {
config: { populationSize: 10 },
population: [{ id: '1', prompt: 'test' }],
generation: 5,
paretoFrontier: { size: 3 },
metrics: { convergence: 0.8 }
};
const backup = await backupManager.createEvolutionStateBackup(evolutionState, 'test-evolution');
expect(backup).toMatchObject({
id: expect.any(String),
label: 'test-evolution',
type: expect.stringMatching(/full|incremental|differential/),
components: expect.arrayContaining([
expect.objectContaining({
name: 'evolution-state',
type: 'evolution-state'
})
])
});
});
it('should create trajectory data backup', async () => {
await backupManager.initialize();
const trajectories = [
{ id: '1', prompt: 'test prompt', response: 'test response', score: 0.9 },
{ id: '2', prompt: 'another prompt', response: 'another response', score: 0.8 }
];
const backup = await backupManager.createTrajectoryDataBackup(trajectories, 'test-trajectories');
expect(backup).toMatchObject({
id: expect.any(String),
label: 'test-trajectories',
metadata: expect.objectContaining({
totalTrajectories: 2
})
});
});
it('should list available backups with filters', async () => {
await backupManager.initialize();
// Create some test backups
await backupManager.createEvolutionStateBackup({
config: {}, population: [], generation: 0, paretoFrontier: null, metrics: {}
}, 'test-1');
await backupManager.createEvolutionStateBackup({
config: {}, population: [], generation: 0, paretoFrontier: null, metrics: {}
}, 'test-2');
const allBackups = backupManager.getAvailableBackups();
expect(allBackups).toHaveLength(2);
const filteredBackups = backupManager.getAvailableBackups({
label: 'test-1'
});
expect(filteredBackups).toHaveLength(1);
expect(filteredBackups[0].label).toBe('test-1');
});
it('should restore from backup with validation', async () => {
await backupManager.initialize();
const backup = await backupManager.createEvolutionStateBackup({
config: {}, population: [], generation: 0, paretoFrontier: null, metrics: {}
}, 'test-restore');
const result = await backupManager.restoreFromBackup(backup.id, {
validateIntegrity: true,
createPreRestoreBackup: true
});
expect(result).toMatchObject({
success: expect.any(Boolean),
backupId: backup.id,
integrityChecks: expect.any(Array)
});
if (result.preRestoreBackupId) {
expect(result.preRestoreBackupId).toMatch(/^backup_/);
}
});
it('should handle backup deletion', async () => {
await backupManager.initialize();
const backup = await backupManager.createEvolutionStateBackup({
config: {}, population: [], generation: 0, paretoFrontier: null, metrics: {}
}, 'test-delete');
await expect(backupManager.deleteBackup(backup.id)).resolves.not.toThrow();
const backups = backupManager.getAvailableBackups();
expect(backups.find(b => b.id === backup.id)).toBeUndefined();
});
});
describe('Disaster Recovery Manager', () => {
let disasterManager: DisasterRecoveryManager;
beforeEach(() => {
disasterManager = new DisasterRecoveryManager({
monitoringInterval: 1000,
autoRecoveryEnabled: true
});
});
afterEach(async () => {
await disasterManager.cleanup();
});
it('should initialize disaster recovery manager', async () => {
await expect(disasterManager.initialize()).resolves.not.toThrow();
});
it('should execute recovery for memory exhaustion', async () => {
await disasterManager.initialize();
const disasterEvent = {
id: 'test-disaster',
type: DisasterType.MEMORY_EXHAUSTION,
severity: 'critical' as const,
timestamp: new Date(),
source: 'test',
description: 'Test memory exhaustion',
metrics: { memoryUsage: 95 },
affectedComponents: ['evolution-engine']
};
const execution = await disasterManager.executeRecovery(disasterEvent);
expect(execution).toMatchObject({
id: expect.any(String),
disasterEvent,
procedure: expect.objectContaining({
disasterType: DisasterType.MEMORY_EXHAUSTION
}),
status: expect.stringMatching(/executing|completed|failed/)
});
});
it('should perform automatic failover', async () => {
await disasterManager.initialize();
const failoverConfig = {
primaryService: 'llm-primary',
backupServices: ['llm-backup-1', 'llm-backup-2'],
switchoverTime: 5000,
rollbackEnabled: true,
dataConsistencyChecks: true
};
const result = await disasterManager.performFailover(failoverConfig);
expect(result).toMatchObject({
success: true,
newPrimaryService: expect.any(String),
switchoverTime: expect.any(Number),
validationResults: expect.any(Object)
});
});
it('should execute emergency shutdown', async () => {
await disasterManager.initialize();
const shutdownOptions = {
reason: 'Test emergency shutdown',
preserveState: true,
notifyOperators: true,
gracefulTimeout: 10000,
forceKill: false
};
await expect(disasterManager.emergencyShutdown(shutdownOptions)).resolves.not.toThrow();
});
it('should get recovery status', () => {
const status = disasterManager.getRecoveryStatus();
expect(status).toMatchObject({
activeExecutions: expect.any(Number),
recentDisasters: expect.any(Array),
systemHealth: expect.stringMatching(/healthy|degraded|critical/),
nextMonitoringCheck: expect.any(Date)
});
});
});
describe('Component Recovery Manager', () => {
let componentManager: ComponentRecoveryManager;
beforeEach(() => {
componentManager = new ComponentRecoveryManager({
healthCheckInterval: 1000,
autoRecoveryEnabled: true
});
});
afterEach(async () => {
await componentManager.cleanup();
});
it('should initialize component recovery manager', async () => {
await expect(componentManager.initialize()).resolves.not.toThrow();
});
it('should recover evolution engine component', async () => {
await componentManager.initialize();
const attempt = await componentManager.recoverComponent(
ComponentType.EVOLUTION_ENGINE,
RecoveryStrategy.RESTART
);
expect(attempt).toMatchObject({
id: expect.any(String),
componentType: ComponentType.EVOLUTION_ENGINE,
strategy: RecoveryStrategy.RESTART,
success: expect.any(Boolean),
logs: expect.any(Array)
});
});
it('should check component health', async () => {
await componentManager.initialize();
const health = await componentManager.checkComponentHealth(ComponentType.LLM_ADAPTER);
expect(health).toMatchObject({
status: expect.stringMatching(/healthy|degraded|critical|failed/),
lastCheck: expect.any(Date),
metrics: expect.any(Object),
uptime: expect.any(Number)
});
});
it('should perform cascade recovery', async () => {
await componentManager.initialize();
const attempts = await componentManager.performCascadeRecovery(ComponentType.EVOLUTION_ENGINE);
expect(attempts).toBeInstanceOf(Array);
expect(attempts.length).toBeGreaterThan(0);
attempts.forEach(attempt => {
expect(attempt).toMatchObject({
id: expect.any(String),
componentType: expect.any(String),
success: expect.any(Boolean)
});
});
});
it('should get recovery history', async () => {
await componentManager.initialize();
// Create a recovery attempt first
await componentManager.recoverComponent(ComponentType.MEMORY_CACHE);
const history = componentManager.getRecoveryHistory(ComponentType.MEMORY_CACHE);
expect(history).toBeInstanceOf(Array);
if (history.length > 0) {
expect(history[0]).toMatchObject({
componentType: ComponentType.MEMORY_CACHE,
success: expect.any(Boolean)
});
}
});
});
describe('Data Integrity Manager', () => {
let integrityManager: DataIntegrityManager;
beforeEach(() => {
integrityManager = new DataIntegrityManager({
checksumType: ChecksumType.SHA256,
realtimeMonitoring: false, // Disable for tests
autoRepairEnabled: true
});
});
afterEach(async () => {
await integrityManager.cleanup();
});
it('should initialize data integrity manager', async () => {
await expect(integrityManager.initialize()).resolves.not.toThrow();
});
it('should perform comprehensive integrity check', async () => {
await integrityManager.initialize();
const results = await integrityManager.performComprehensiveCheck();
expect(results).toBeInstanceOf(Array);
results.forEach(result => {
expect(result).toMatchObject({
id: expect.any(String),
timestamp: expect.any(Date),
dataType: expect.any(String),
overallValid: expect.any(Boolean),
corruptionLevel: expect.stringMatching(/none|minor|moderate|severe|critical/)
});
});
});
it('should validate data with custom rules', async () => {
await integrityManager.initialize();
const testData = {
generation: 5,
population: [{ id: '1', prompt: 'test' }],
config: { populationSize: 10 }
};
const result = await integrityManager.validateData(testData, 'evolution_state');
expect(result).toMatchObject({
valid: expect.any(Boolean),
errors: expect.any(Array),
warnings: expect.any(Array),
corruptionLevel: expect.stringMatching(/none|minor|moderate|severe|critical/)
});
});
it('should calculate and verify checksums', () => {
const testData = { test: 'data', number: 42 };
const checksum = integrityManager.calculateChecksum(testData, ChecksumType.SHA256);
expect(checksum).toMatch(/^[a-f0-9]{64}$/); // SHA256 hex string
const isValid = integrityManager.verifyChecksum(testData, checksum, ChecksumType.SHA256);
expect(isValid).toBe(true);
const invalidChecksum = 'invalid-checksum';
const isInvalid = integrityManager.verifyChecksum(testData, invalidChecksum, ChecksumType.SHA256);
expect(isInvalid).toBe(false);
});
it('should create data snapshots', async () => {
await integrityManager.initialize();
const testData = { population: [1, 2, 3], generation: 10 };
const snapshot = await integrityManager.createDataSnapshot(
'./test-data.json',
'evolution_state',
testData
);
expect(snapshot).toMatchObject({
id: expect.any(String),
timestamp: expect.any(Date),
dataType: 'evolution_state',
checksum: expect.any(String),
size: expect.any(Number)
});
});
it('should attempt automatic repair', async () => {
await integrityManager.initialize();
const corruptedData = { incomplete: 'data' };
const integrityResult = {
id: 'test-check',
timestamp: new Date(),
dataType: 'evolution_state',
dataPath: './test-corrupted.json',
checksumValid: false,
structureValid: false,
contentValid: true,
crossReferenceValid: true,
overallValid: false,
corruptionLevel: CorruptionLevel.MODERATE,
errors: [],
warnings: [],
repairSuggestions: [],
metrics: {
checkDuration: 100,
dataSize: 1024,
errorCount: 1,
warningCount: 0
}
};
const repairResult = await integrityManager.attemptRepair(corruptedData, integrityResult);
expect(repairResult).toMatchObject({
success: expect.any(Boolean),
strategy: expect.stringMatching(/auto_repair|restore_from_backup|reconstruct_data|isolate_corruption/),
confidence: expect.any(Number),
warnings: expect.any(Array)
});
});
});
describe('Recovery Orchestrator', () => {
let orchestrator: RecoveryOrchestrator;
let stateBackupManager: StateBackupManager;
let disasterRecoveryManager: DisasterRecoveryManager;
let componentRecoveryManager: ComponentRecoveryManager;
let dataIntegrityManager: DataIntegrityManager;
beforeEach(() => {
stateBackupManager = new StateBackupManager();
disasterRecoveryManager = new DisasterRecoveryManager();
componentRecoveryManager = new ComponentRecoveryManager();
dataIntegrityManager = new DataIntegrityManager();
orchestrator = new RecoveryOrchestrator({
stateBackupManager,
disasterRecoveryManager,
componentRecoveryManager,
dataIntegrityManager
}, {
maxConcurrentRecoveries: 2,
retryFailedSteps: true
});
});
afterEach(async () => {
await orchestrator.cleanup();
});
it('should initialize recovery orchestrator', async () => {
await expect(orchestrator.initialize()).resolves.not.toThrow();
});
it('should create system backup through orchestrator', async () => {
await orchestrator.initialize();
const backup = await orchestrator.createSystemBackup('orchestrator-test');
expect(backup).toMatchObject({
id: expect.any(String),
label: 'orchestrator-test'
});
});
it('should execute disaster recovery through orchestrator', async () => {
await orchestrator.initialize();
const execution = await orchestrator.executeDisasterRecovery(DisasterType.SERVICE_FAILURE);
expect(execution).toMatchObject({
id: expect.any(String),
plan: expect.objectContaining({
name: expect.any(String),
steps: expect.any(Array)
}),
status: expect.stringMatching(/pending|running|completed|failed/)
});
});
it('should get comprehensive recovery dashboard', async () => {
await orchestrator.initialize();
const dashboard = await orchestrator.getDashboard();
expect(dashboard).toMatchObject({
systemStatus: expect.objectContaining({
overall: expect.stringMatching(/healthy|degraded|critical|recovering/),
lastUpdate: expect.any(Date)
}),
recoveryHistory: expect.objectContaining({
totalExecutions: expect.any(Number),
successRate: expect.any(Number)
}),
systemHealth: expect.objectContaining({
backup: expect.any(Object),
disaster: expect.any(Object),
component: expect.any(Object),
integrity: expect.any(Object)
}),
recommendations: expect.any(Array),
alerts: expect.any(Array)
});
});
});
describe('Error Handling and Edge Cases', () => {
it('should handle backup creation failures gracefully', async () => {
const backupManager = new StateBackupManager({
backupDirectory: '/invalid/path'
});
await expect(backupManager.initialize()).rejects.toThrow();
});
it('should handle recovery attempts on non-existent backups', async () => {
const backupManager = new StateBackupManager();
await backupManager.initialize();
await expect(
backupManager.restoreFromBackup('non-existent-backup-id')
).rejects.toThrow('Backup not found');
});
it('should handle component recovery with invalid component type', async () => {
const componentManager = new ComponentRecoveryManager();
await componentManager.initialize();
await expect(
componentManager.recoverComponent('invalid-component' as ComponentType)
).rejects.toThrow();
});
it('should handle integrity checks on invalid data', async () => {
const integrityManager = new DataIntegrityManager();
await integrityManager.initialize();
const invalidData = null;
const result = await integrityManager.validateData(invalidData, 'evolution_state');
expect(result.valid).toBe(false);
expect(result.errors.length).toBeGreaterThan(0);
});
it('should handle concurrent recovery limit', async () => {
const stateBackupManager = new StateBackupManager();
const disasterRecoveryManager = new DisasterRecoveryManager();
const componentRecoveryManager = new ComponentRecoveryManager();
const dataIntegrityManager = new DataIntegrityManager();
const orchestrator = new RecoveryOrchestrator({
stateBackupManager,
disasterRecoveryManager,
componentRecoveryManager,
dataIntegrityManager
}, {
maxConcurrentRecoveries: 1
});
await orchestrator.initialize();
// Start first recovery
const firstRecovery = orchestrator.executeDisasterRecovery(DisasterType.MEMORY_EXHAUSTION);
// Try to start second recovery immediately
await expect(
orchestrator.executeDisasterRecovery(DisasterType.SERVICE_FAILURE)
).rejects.toThrow('Maximum concurrent recoveries reached');
// Clean up
await firstRecovery.catch(() => {}); // Ignore errors for cleanup
});
});
describe('Performance and Stress Tests', () => {
it('should handle multiple backup operations efficiently', async () => {
const backupManager = new StateBackupManager({
maxBackups: 100
});
await backupManager.initialize();
const startTime = Date.now();
const backupPromises = [];
for (let i = 0; i < 10; i++) {
const promise = backupManager.createEvolutionStateBackup({
config: {},
population: [],
generation: i,
paretoFrontier: null,
metrics: {}
}, `stress-test-${i}`);
backupPromises.push(promise);
}
const backups = await Promise.all(backupPromises);
const duration = Date.now() - startTime;
expect(backups).toHaveLength(10);
expect(duration).toBeLessThan(10000); // Should complete within 10 seconds
});
it('should maintain performance during integrity checks', async () => {
const integrityManager = new DataIntegrityManager({
realtimeMonitoring: false
});
await integrityManager.initialize();
const startTime = Date.now();
const checks = [];
for (let i = 0; i < 5; i++) {
checks.push(integrityManager.performComprehensiveCheck());
}
const results = await Promise.all(checks);
const duration = Date.now() - startTime;
expect(results).toHaveLength(5);
expect(duration).toBeLessThan(5000); // Should complete within 5 seconds
});
});
});