Skip to main content
Glama

Prompt Auto-Optimizer MCP

by sloth-wq
data-integrity-manager.ts29.2 kB
/** * Data Integrity Manager - Checksums, Validation, and Corruption Detection * * Provides comprehensive data integrity and validation capabilities: * - Checksums and data validation * - Corruption detection algorithms * - Automatic repair mechanisms * - Data consistency checks * - Real-time integrity monitoring */ import { EventEmitter } from 'events'; import { createHash } from 'crypto'; import { readFileSync, writeFileSync, existsSync } from 'fs'; import { ResilienceSystem } from '../resilience/index'; import { ComponentStatus, type ComponentHealth } from './component-recovery-manager'; export enum CorruptionLevel { NONE = 'none', MINOR = 'minor', MODERATE = 'moderate', SEVERE = 'severe', CRITICAL = 'critical' } export enum RepairStrategy { AUTO_REPAIR = 'auto_repair', RESTORE_FROM_BACKUP = 'restore_from_backup', MANUAL_INTERVENTION = 'manual_intervention', ISOLATE_CORRUPTION = 'isolate_corruption', RECONSTRUCT_DATA = 'reconstruct_data', ROLLBACK_CHANGES = 'rollback_changes' } export enum ChecksumType { MD5 = 'md5', SHA256 = 'sha256', SHA512 = 'sha512', BLAKE2B = 'blake2b', CRC32 = 'crc32' } export interface IntegrityCheckConfig { checksumType: ChecksumType; realtimeMonitoring: boolean; checkInterval: number; // milliseconds autoRepairEnabled: boolean; backupValidation: boolean; memoryValidation: boolean; crossReferenceChecks: boolean; quarantineEnabled: boolean; alertThreshold: CorruptionLevel; } export interface ValidationData { [key: string]: unknown; } export interface ValidationRule { id: string; name: string; description: string; dataType: 'evolution_state' | 'trajectory' | 'configuration' | 'cache' | 'metrics'; validationFunction: (data: ValidationData) => ValidationResult; severity: 'low' | 'medium' | 'high' | 'critical'; autoRepair: boolean; } export interface ValidationResult { valid: boolean; errors: string[]; warnings: string[]; corruptionLevel: CorruptionLevel; affectedFields: string[]; repairSuggestions: RepairSuggestion[]; } export interface RepairSuggestion { strategy: RepairStrategy; description: string; confidence: number; // 0-1 estimatedTime: number; // milliseconds riskLevel: 'low' | 'medium' | 'high'; prerequisites: string[]; } export interface IntegrityCheckResult { id: string; timestamp: Date; dataType: string; dataPath: string; checksumValid: boolean; structureValid: boolean; contentValid: boolean; crossReferenceValid: boolean; overallValid: boolean; corruptionLevel: CorruptionLevel; errors: IntegrityError[]; warnings: string[]; repairSuggestions: RepairSuggestion[]; metrics: { checkDuration: number; dataSize: number; errorCount: number; warningCount: number; }; } export interface IntegrityError { type: 'checksum_mismatch' | 'structure_violation' | 'content_corruption' | 'reference_broken' | 'missing_data'; severity: 'low' | 'medium' | 'high' | 'critical'; field: string; expected: unknown; actual: unknown; description: string; repairStrategy?: RepairStrategy; } export interface DataSnapshot { id: string; timestamp: Date; dataType: string; checksum: string; size: number; version: string; metadata: Record<string, unknown>; } export interface QuarantineEntry { id: string; timestamp: Date; dataPath: string; corruptionLevel: CorruptionLevel; originalChecksum: string; corruptedChecksum: string; errors: IntegrityError[]; quarantinePath: string; autoRepairAttempted: boolean; } /** * Data Integrity Manager Implementation */ export class DataIntegrityManager extends EventEmitter { private config: IntegrityCheckConfig; private validationRules: Map<string, ValidationRule> = new Map(); private integrityHistory: IntegrityCheckResult[] = []; private dataSnapshots: Map<string, DataSnapshot> = new Map(); private quarantineEntries: Map<string, QuarantineEntry> = new Map(); private monitoringTimer?: ReturnType<typeof setInterval> | undefined; private checksumCache: Map<string, string> = new Map(); private resilience: ResilienceSystem; constructor(config: Partial<IntegrityCheckConfig> = {}) { super(); this.config = { checksumType: ChecksumType.SHA256, realtimeMonitoring: true, checkInterval: 300000, // 5 minutes autoRepairEnabled: true, backupValidation: true, memoryValidation: true, crossReferenceChecks: true, quarantineEnabled: true, alertThreshold: CorruptionLevel.MODERATE, ...config }; this.resilience = ResilienceSystem.getInstance(); this.initializeValidationRules(); } /** * Initialize data integrity manager */ async initialize(): Promise<void> { try { // Load existing snapshots and quarantine entries await this.loadDataSnapshots(); await this.loadQuarantineEntries(); // Start real-time monitoring if enabled if (this.config.realtimeMonitoring) { this.startRealtimeMonitoring(); } // Perform initial integrity check await this.performComprehensiveCheck(); this.emit('initialized', { validationRules: this.validationRules.size, dataSnapshots: this.dataSnapshots.size, quarantineEntries: this.quarantineEntries.size }); } catch (error) { this.emit('error', { operation: 'initialize', error }); throw error; } } /** * Perform comprehensive integrity check */ async performComprehensiveCheck(): Promise<IntegrityCheckResult[]> { return this.resilience.executeWithFullProtection( async () => { const results: IntegrityCheckResult[] = []; // Check evolution state integrity const evolutionResult = await this.checkEvolutionStateIntegrity(); if (evolutionResult) results.push(evolutionResult); // Check trajectory data integrity const trajectoryResults = await this.checkTrajectoryDataIntegrity(); results.push(...trajectoryResults); // Check configuration integrity const configResult = await this.checkConfigurationIntegrity(); if (configResult) results.push(configResult); // Check cache integrity const cacheResults = await this.checkCacheIntegrity(); results.push(...cacheResults); // Check cross-references if enabled if (this.config.crossReferenceChecks) { const crossRefResults = await this.checkCrossReferences(); results.push(...crossRefResults); } // Process results and trigger repairs if needed await this.processIntegrityResults(results); this.emit('comprehensiveCheckCompleted', { resultCount: results.length, corruptionDetected: results.some(r => !r.overallValid) }); return results; }, { serviceName: 'data-integrity', context: { name: 'comprehensive-check', priority: 'high' } } ); } /** * Validate specific data with custom rules */ async validateData( data: unknown, dataType: string, ruleIds?: string[] ): Promise<ValidationResult> { return this.resilience.executeWithFullProtection( async () => { const result: ValidationResult = { valid: true, errors: [], warnings: [], corruptionLevel: CorruptionLevel.NONE, affectedFields: [], repairSuggestions: [] }; // Get applicable validation rules const rules = this.getApplicableRules(dataType, ruleIds); // Apply validation rules for (const rule of rules) { try { const ruleResult = rule.validationFunction(data as ValidationData); // Merge results if (!ruleResult.valid) { result.valid = false; result.errors.push(...ruleResult.errors); } result.warnings.push(...ruleResult.warnings); result.affectedFields.push(...ruleResult.affectedFields); result.repairSuggestions.push(...ruleResult.repairSuggestions); // Update corruption level to highest found if (this.getCorruptionLevelSeverity(ruleResult.corruptionLevel) > this.getCorruptionLevelSeverity(result.corruptionLevel)) { result.corruptionLevel = ruleResult.corruptionLevel; } } catch (error) { result.valid = false; result.errors.push(`Validation rule ${rule.id} failed: ${(error as Error).message}`); } } this.emit('dataValidated', { dataType, valid: result.valid, corruptionLevel: result.corruptionLevel }); return result; }, { serviceName: 'data-integrity', context: { name: 'validate-data', priority: 'medium' } } ); } /** * Calculate checksum for data */ calculateChecksum(data: unknown, type: ChecksumType = this.config.checksumType): string { const serialized = typeof data === 'string' ? data : JSON.stringify(data, data && typeof data === 'object' ? Object.keys(data as object).sort() : undefined); switch (type) { case ChecksumType.MD5: return createHash('md5').update(serialized).digest('hex'); case ChecksumType.SHA256: return createHash('sha256').update(serialized).digest('hex'); case ChecksumType.SHA512: return createHash('sha512').update(serialized).digest('hex'); case ChecksumType.BLAKE2B: return createHash('blake2b512').update(serialized).digest('hex'); case ChecksumType.CRC32: // Simple CRC32 implementation return this.calculateCRC32(serialized); default: return createHash('sha256').update(serialized).digest('hex'); } } /** * Verify checksum for data */ verifyChecksum(data: unknown, expectedChecksum: string, type: ChecksumType = this.config.checksumType): boolean { const actualChecksum = this.calculateChecksum(data, type); return actualChecksum === expectedChecksum; } /** * Create data snapshot */ async createDataSnapshot(dataPath: string, dataType: string, data: unknown): Promise<DataSnapshot> { const snapshot: DataSnapshot = { id: this.generateSnapshotId(), timestamp: new Date(), dataType, checksum: this.calculateChecksum(data), size: this.calculateDataSize(data), version: this.extractVersion(data), metadata: { path: dataPath, elementCount: this.countElements(data), structure: this.analyzeStructure(data) } }; this.dataSnapshots.set(snapshot.id, snapshot); this.emit('snapshotCreated', snapshot); return snapshot; } /** * Attempt automatic repair of corrupted data */ async attemptRepair( corruptedData: unknown, integrityResult: IntegrityCheckResult ): Promise<{ success: boolean; repairedData?: unknown; strategy: RepairStrategy; confidence: number; warnings: string[]; }> { return this.resilience.executeWithFullProtection( async () => { // Select best repair strategy const strategy = this.selectRepairStrategy(integrityResult); let success = false; let repairedData: unknown; let confidence = 0; const warnings: string[] = []; try { switch (strategy) { case RepairStrategy.AUTO_REPAIR: ({ success, repairedData, confidence } = await this.performAutoRepair(corruptedData, integrityResult)); break; case RepairStrategy.RESTORE_FROM_BACKUP: ({ success, repairedData, confidence } = await this.restoreFromBackup(integrityResult.dataPath)); break; case RepairStrategy.RECONSTRUCT_DATA: ({ success, repairedData, confidence } = await this.reconstructData(corruptedData, integrityResult)); break; case RepairStrategy.ISOLATE_CORRUPTION: ({ success, repairedData, confidence } = await this.isolateCorruption(corruptedData, integrityResult)); break; default: warnings.push(`Repair strategy ${strategy} not supported for auto-repair`); success = false; } // Validate repaired data if (success && repairedData) { const validationResult = await this.validateData(repairedData, integrityResult.dataType); if (!validationResult.valid) { success = false; warnings.push('Repaired data failed validation'); } } } catch (error) { success = false; warnings.push(`Repair failed: ${(error as Error).message}`); } this.emit('repairAttempted', { strategy, success, confidence, dataPath: integrityResult.dataPath }); return { success, repairedData, strategy, confidence, warnings }; }, { serviceName: 'data-integrity', context: { name: 'attempt-repair', priority: 'high' } } ); } /** * Quarantine corrupted data */ async quarantineData( dataPath: string, corruptionLevel: CorruptionLevel, errors: IntegrityError[] ): Promise<QuarantineEntry> { if (!this.config.quarantineEnabled) { throw new Error('Quarantine is disabled'); } const entry: QuarantineEntry = { id: this.generateQuarantineId(), timestamp: new Date(), dataPath, corruptionLevel, originalChecksum: this.checksumCache.get(dataPath) || '', corruptedChecksum: '', errors, quarantinePath: this.generateQuarantinePath(dataPath), autoRepairAttempted: false }; try { // Read original data and calculate corrupted checksum if (existsSync(dataPath)) { const data = readFileSync(dataPath, 'utf8'); entry.corruptedChecksum = this.calculateChecksum(data); // Move to quarantine writeFileSync(entry.quarantinePath, data); } this.quarantineEntries.set(entry.id, entry); this.emit('dataQuarantined', entry); } catch (error) { this.emit('quarantineFailed', { dataPath, error }); throw error; } return entry; } /** * Get health status */ async getHealthStatus(): Promise<ComponentHealth> { const recentChecks = this.integrityHistory .filter(r => Date.now() - r.timestamp.getTime() < 3600000) // Last hour .slice(-20); const corruptionCount = recentChecks.filter(r => !r.overallValid).length; const quarantineCount = this.quarantineEntries.size; let status: ComponentStatus = ComponentStatus.HEALTHY; const recommendations: string[] = []; if (corruptionCount > 10 || quarantineCount > 5) { status = ComponentStatus.CRITICAL; recommendations.push('High level of data corruption detected'); } else if (corruptionCount > 5 || quarantineCount > 2) { status = ComponentStatus.DEGRADED; recommendations.push('Multiple integrity issues detected'); } if (quarantineCount > 0) { recommendations.push(`${quarantineCount} items in quarantine require attention`); } return { status, lastCheck: new Date(), metrics: { recentChecks: recentChecks.length, corruptionCount, quarantineCount, validationRules: this.validationRules.size, dataSnapshots: this.dataSnapshots.size }, errors: [], warnings: [], recommendations, uptime: Date.now(), recoveryCount: 0 }; } /** * Private helper methods */ private initializeValidationRules(): void { // Evolution State Validation Rules this.addValidationRule({ id: 'evolution_state_structure', name: 'Evolution State Structure Validation', description: 'Validates the structure of evolution state data', dataType: 'evolution_state', severity: 'critical', autoRepair: true, validationFunction: (data) => { const result: ValidationResult = { valid: true, errors: [], warnings: [], corruptionLevel: CorruptionLevel.NONE, affectedFields: [], repairSuggestions: [] }; // Check required fields const requiredFields = ['generation', 'population', 'config']; for (const field of requiredFields) { if (!(field in data)) { result.valid = false; result.errors.push(`Missing required field: ${field}`); result.affectedFields.push(field); result.corruptionLevel = CorruptionLevel.SEVERE; } } // Check population structure if (data.population && !Array.isArray(data.population)) { result.valid = false; result.errors.push('Population must be an array'); result.affectedFields.push('population'); result.corruptionLevel = CorruptionLevel.CRITICAL; } return result; } }); // Trajectory Data Validation Rules this.addValidationRule({ id: 'trajectory_data_integrity', name: 'Trajectory Data Integrity', description: 'Validates trajectory data completeness and consistency', dataType: 'trajectory', severity: 'high', autoRepair: true, validationFunction: (data) => { const result: ValidationResult = { valid: true, errors: [], warnings: [], corruptionLevel: CorruptionLevel.NONE, affectedFields: [], repairSuggestions: [] }; if (!Array.isArray(data)) { result.valid = false; result.errors.push('Trajectory data must be an array'); result.corruptionLevel = CorruptionLevel.CRITICAL; return result; } // Validate each trajectory for (let i = 0; i < data.length; i++) { const trajectory = data[i]; if (!trajectory.id || !trajectory.prompt || !trajectory.response) { result.valid = false; result.errors.push(`Incomplete trajectory at index ${i}`); result.affectedFields.push(`[${i}]`); result.corruptionLevel = CorruptionLevel.MODERATE; } } return result; } }); // Configuration Validation Rules this.addValidationRule({ id: 'configuration_schema', name: 'Configuration Schema Validation', description: 'Validates configuration against expected schema', dataType: 'configuration', severity: 'medium', autoRepair: true, validationFunction: (data) => { const result: ValidationResult = { valid: true, errors: [], warnings: [], corruptionLevel: CorruptionLevel.NONE, affectedFields: [], repairSuggestions: [] }; // Check for required configuration sections const requiredSections = ['evolution', 'llm', 'storage']; for (const section of requiredSections) { if (!(section in data)) { result.warnings.push(`Missing configuration section: ${section}`); result.affectedFields.push(section); result.corruptionLevel = CorruptionLevel.MINOR; } } return result; } }); } private addValidationRule(rule: ValidationRule): void { this.validationRules.set(rule.id, rule); } private getApplicableRules(dataType: string, ruleIds?: string[]): ValidationRule[] { const rules = Array.from(this.validationRules.values()); let applicableRules = rules.filter(rule => rule.dataType === dataType); if (ruleIds && ruleIds.length > 0) { applicableRules = applicableRules.filter(rule => ruleIds.includes(rule.id)); } return applicableRules; } private async checkEvolutionStateIntegrity(): Promise<IntegrityCheckResult | null> { // Mock implementation - would check actual evolution state return this.createMockIntegrityResult('evolution_state', './evolution_state.json'); } private async checkTrajectoryDataIntegrity(): Promise<IntegrityCheckResult[]> { // Mock implementation - would check actual trajectory data return [ this.createMockIntegrityResult('trajectory', './trajectories/batch_1.json'), this.createMockIntegrityResult('trajectory', './trajectories/batch_2.json') ]; } private async checkConfigurationIntegrity(): Promise<IntegrityCheckResult | null> { // Mock implementation - would check actual configuration return this.createMockIntegrityResult('configuration', './config.json'); } private async checkCacheIntegrity(): Promise<IntegrityCheckResult[]> { // Mock implementation - would check actual cache data return [ this.createMockIntegrityResult('cache', './cache/evolution_cache.json') ]; } private async checkCrossReferences(): Promise<IntegrityCheckResult[]> { // Mock implementation - would check cross-references between data return []; } private createMockIntegrityResult(dataType: string, dataPath: string): IntegrityCheckResult { // Simulate some processing time const checkDuration = Math.random() * 100 + 50; return { id: this.generateCheckId(), timestamp: new Date(), dataType, dataPath, checksumValid: Math.random() > 0.1, // 90% chance of valid checksum structureValid: Math.random() > 0.05, // 95% chance of valid structure contentValid: Math.random() > 0.02, // 98% chance of valid content crossReferenceValid: Math.random() > 0.03, // 97% chance of valid cross-references overallValid: true, // Will be calculated corruptionLevel: CorruptionLevel.NONE, // Will be calculated errors: [], warnings: [], repairSuggestions: [], metrics: { checkDuration, dataSize: Math.floor(Math.random() * 1000000) + 10000, errorCount: 0, warningCount: 0 } }; } private async processIntegrityResults(results: IntegrityCheckResult[]): Promise<void> { for (const result of results) { // Calculate overall validity result.overallValid = result.checksumValid && result.structureValid && result.contentValid && result.crossReferenceValid; // Determine corruption level if (!result.overallValid) { if (!result.checksumValid || !result.structureValid) { result.corruptionLevel = CorruptionLevel.CRITICAL; } else if (!result.contentValid) { result.corruptionLevel = CorruptionLevel.MODERATE; } else { result.corruptionLevel = CorruptionLevel.MINOR; } } // Store in history this.integrityHistory.push(result); // Trigger alerts if threshold exceeded if (this.getCorruptionLevelSeverity(result.corruptionLevel) >= this.getCorruptionLevelSeverity(this.config.alertThreshold)) { this.emit('corruptionAlert', result); // Auto-repair if enabled and corruption detected if (this.config.autoRepairEnabled && !result.overallValid) { try { await this.attemptRepair({}, result); } catch (error) { // eslint-disable-next-line no-console console.warn(`Auto-repair failed for ${result.dataPath}:`, error); } } } } // Cleanup old history (keep last 1000 entries) if (this.integrityHistory.length > 1000) { this.integrityHistory = this.integrityHistory.slice(-1000); } } private getCorruptionLevelSeverity(level: CorruptionLevel): number { switch (level) { case CorruptionLevel.NONE: return 0; case CorruptionLevel.MINOR: return 1; case CorruptionLevel.MODERATE: return 2; case CorruptionLevel.SEVERE: return 3; case CorruptionLevel.CRITICAL: return 4; default: return 0; } } private selectRepairStrategy(integrityResult: IntegrityCheckResult): RepairStrategy { // Select repair strategy based on corruption level and type if (integrityResult.corruptionLevel === CorruptionLevel.CRITICAL) { return RepairStrategy.RESTORE_FROM_BACKUP; } else if (integrityResult.corruptionLevel === CorruptionLevel.SEVERE) { return RepairStrategy.RECONSTRUCT_DATA; } else if (integrityResult.corruptionLevel === CorruptionLevel.MODERATE) { return RepairStrategy.AUTO_REPAIR; } else { return RepairStrategy.ISOLATE_CORRUPTION; } } private async performAutoRepair(data: unknown, _result: IntegrityCheckResult): Promise<{ success: boolean; repairedData?: unknown; confidence: number; }> { // Simple auto-repair implementation // In practice, would implement sophisticated repair algorithms return { success: true, repairedData: data, confidence: 0.8 }; } private async restoreFromBackup(_dataPath: string): Promise<{ success: boolean; repairedData?: unknown; confidence: number; }> { // Implementation would restore from actual backup return { success: true, repairedData: {}, confidence: 0.95 }; } private async reconstructData(data: unknown, _result: IntegrityCheckResult): Promise<{ success: boolean; repairedData?: unknown; confidence: number; }> { // Implementation would reconstruct data from partial information return { success: true, repairedData: data, confidence: 0.7 }; } private async isolateCorruption(data: unknown, _result: IntegrityCheckResult): Promise<{ success: boolean; repairedData?: unknown; confidence: number; }> { // Implementation would isolate corrupted parts return { success: true, repairedData: data, confidence: 0.9 }; } private calculateCRC32(data: string): string { // Simple CRC32 implementation let crc = 0xFFFFFFFF; for (let i = 0; i < data.length; i++) { crc ^= data.charCodeAt(i); for (let j = 0; j < 8; j++) { if (crc & 1) { crc = (crc >>> 1) ^ 0xEDB88320; } else { crc >>>= 1; } } } return ((crc ^ 0xFFFFFFFF) >>> 0).toString(16).padStart(8, '0'); } private calculateDataSize(data: unknown): number { return JSON.stringify(data).length; } private extractVersion(data: unknown): string { if (data && typeof data === 'object') { const obj = data as Record<string, unknown>; return (obj.version || obj.v || '1.0.0') as string; } return '1.0.0'; } private countElements(data: unknown): number { if (Array.isArray(data)) return data.length; if (typeof data === 'object' && data !== null) return Object.keys(data).length; return 1; } private analyzeStructure(data: unknown): string { if (Array.isArray(data)) return 'array'; if (typeof data === 'object' && data !== null) return 'object'; return typeof data; } private startRealtimeMonitoring(): void { this.monitoringTimer = setInterval(async () => { try { // Perform periodic integrity checks await this.performComprehensiveCheck(); } catch (error) { this.emit('monitoringError', error); } }, this.config.checkInterval); } private async loadDataSnapshots(): Promise<void> { // Implementation would load existing snapshots from storage // For now, create empty map } private async loadQuarantineEntries(): Promise<void> { // Implementation would load existing quarantine entries from storage // For now, create empty map } private generateSnapshotId(): string { return `snapshot_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; } private generateQuarantineId(): string { return `quarantine_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; } private generateCheckId(): string { return `check_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; } private generateQuarantinePath(originalPath: string): string { const timestamp = Date.now(); return `./quarantine/${timestamp}_${originalPath.replace(/[^a-zA-Z0-9]/g, '_')}`; } /** * Cleanup resources */ async cleanup(): Promise<void> { if (this.monitoringTimer) { clearInterval(this.monitoringTimer); this.monitoringTimer = undefined; } this.removeAllListeners(); } }

MCP directory API

We provide all the information about MCP servers via our MCP API.

curl -X GET 'https://glama.ai/api/mcp/v1/servers/sloth-wq/prompt-auto-optimizer-mcp'

If you have feedback or need assistance with the MCP directory API, please join our Discord server