documcp

/** * Advanced unit tests for Memory Export/Import System * Tests data portability, backup, and migration capabilities * Part of Issue #55 - Advanced Memory Components Unit Tests */ import { promises as fs } from 'fs'; import path from 'path'; import os from 'os'; import { MemoryManager } from '../../src/memory/manager.js'; import { JSONLStorage } from '../../src/memory/storage.js'; import { IncrementalLearningSystem } from '../../src/memory/learning.js'; import { KnowledgeGraph } from '../../src/memory/knowledge-graph.js'; import { MemoryExportImportSystem, ExportOptions, ImportOptions, ExportResult, ImportResult, } from '../../src/memory/export-import.js'; describe('MemoryExportImportSystem', () => { let tempDir: string; let exportDir: string; let memoryManager: MemoryManager; let storage: JSONLStorage; let learningSystem: IncrementalLearningSystem; let knowledgeGraph: KnowledgeGraph; let exportImportSystem: MemoryExportImportSystem; beforeEach(async () => { // Create unique temp directories for each test tempDir = path.join( os.tmpdir(), `export-import-test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, ); exportDir = path.join(tempDir, 'exports'); await fs.mkdir(tempDir, { recursive: true }); await fs.mkdir(exportDir, { recursive: true }); memoryManager = new MemoryManager(tempDir); await memoryManager.initialize(); // Use the memory manager's storage for consistency storage = memoryManager.getStorage(); learningSystem = new IncrementalLearningSystem(memoryManager); await learningSystem.initialize(); knowledgeGraph = new KnowledgeGraph(memoryManager); await knowledgeGraph.initialize(); exportImportSystem = new MemoryExportImportSystem( storage, memoryManager, learningSystem, knowledgeGraph, ); }); afterEach(async () => { // Cleanup temp directories try { await fs.rm(tempDir, { recursive: true, force: true }); } catch (error) { // Ignore cleanup errors } }); describe('Export System', () => { beforeEach(async () => { // Set up test data for export tests memoryManager.setContext({ projectId: 'export-test-project' }); await memoryManager.remember( 'analysis', { language: { primary: 'typescript' }, framework: { name: 'react' }, metrics: { complexity: 'medium', performance: 'good' }, }, { tags: ['frontend', 'typescript'], repository: 'github.com/test/repo', }, ); await memoryManager.remember( 'recommendation', { recommended: 'docusaurus', confidence: 0.9, reasoning: ['typescript support', 'react compatibility'], }, { tags: ['documentation', 'ssg'], }, ); await memoryManager.remember( 'deployment', { status: 'success', platform: 'github-pages', duration: 120, url: 'https://test.github.io', }, { tags: ['deployment', 'success'], }, ); }); test('should export memories in JSON format', async () => { const exportOptions: ExportOptions = { format: 'json', includeMetadata: true, includeLearning: false, includeKnowledgeGraph: false, compression: 'none', }; const exportPath = path.join(exportDir, 'test-export.json'); const result = await exportImportSystem.exportMemories(exportPath, exportOptions); expect(result).toBeDefined(); expect(result.success).toBe(true); expect(result.entries).toBeGreaterThan(0); expect(result.filePath).toBe(exportPath); // Verify file was created const fileExists = await fs .access(exportPath) .then(() => true) .catch(() => false); expect(fileExists).toBe(true); // Verify file content const content = await fs.readFile(exportPath, 'utf-8'); const exported = JSON.parse(content); expect(exported).toHaveProperty('metadata'); expect(exported).toHaveProperty('memories'); expect(Array.isArray(exported.memories)).toBe(true); expect(exported.memories.length).toBe(3); }); test('should export memories in JSONL format', async () => { const exportOptions: ExportOptions = { format: 'jsonl', includeMetadata: true, includeLearning: false, includeKnowledgeGraph: false, }; const exportPath = path.join(exportDir, 'test-export.jsonl'); const result = await exportImportSystem.exportMemories(exportPath, exportOptions); expect(result.success).toBe(true); expect(result.entries).toBe(3); // Verify JSONL format const content = await fs.readFile(exportPath, 'utf-8'); const lines = content.trim().split('\n'); expect(lines.length).toBe(4); // 1 metadata + 3 memory entries lines.forEach((line) => { expect(() => JSON.parse(line)).not.toThrow(); }); // First line should be metadata const firstLine = JSON.parse(lines[0]); expect(firstLine).toHaveProperty('version'); expect(firstLine).toHaveProperty('exportedAt'); }); test('should export with filtering options', async () => { const exportOptions: ExportOptions = { format: 'json', includeMetadata: true, includeLearning: false, includeKnowledgeGraph: false, filters: { types: ['analysis', 'recommendation'], tags: ['frontend'], }, }; const exportPath = path.join(exportDir, 'filtered-export.json'); const result = await exportImportSystem.exportMemories(exportPath, exportOptions); expect(result.success).toBe(true); const content = await fs.readFile(exportPath, 'utf-8'); const exported = JSON.parse(content); // Should only include filtered types exported.memories.forEach((memory: any) => { expect(['analysis', 'recommendation']).toContain(memory.type); }); }); test('should handle compression options', async () => { const exportOptions: ExportOptions = { format: 'json', includeMetadata: true, includeLearning: false, includeKnowledgeGraph: false, compression: 'gzip', }; const exportPath = path.join(exportDir, 'compressed-export.json.gz'); const result = await exportImportSystem.exportMemories(exportPath, exportOptions); expect(result.success).toBe(true); expect(result.metadata.compression).toBe('gzip'); // Verify compressed file exists const fileExists = await fs .access(exportPath) .then(() => true) .catch(() => false); expect(fileExists).toBe(true); }); test('should export with anonymization', async () => { const exportOptions: ExportOptions = { format: 'json', includeMetadata: true, includeLearning: false, includeKnowledgeGraph: false, anonymize: { enabled: true, fields: ['repository', 'url'], method: 'hash', }, }; const exportPath = path.join(exportDir, 'anonymized-export.json'); const result = await exportImportSystem.exportMemories(exportPath, exportOptions); expect(result.success).toBe(true); const content = await fs.readFile(exportPath, 'utf-8'); const exported = JSON.parse(content); // Check that specified fields are anonymized exported.memories.forEach((memory: any) => { if (memory.metadata.repository) { // Should be hashed, not original value expect(memory.metadata.repository).not.toBe('github.com/test/repo'); } if (memory.data.url) { expect(memory.data.url).not.toBe('https://test.github.io'); } }); }); }); describe('Import System', () => { let testExportPath: string; beforeEach(async () => { // Create test export file for import tests testExportPath = path.join(exportDir, 'test-import.json'); const testData = { metadata: { exportedAt: new Date().toISOString(), version: '1.0.0', source: 'test', }, memories: [ { id: 'test-import-1', type: 'analysis', timestamp: new Date().toISOString(), data: { language: { primary: 'python' }, framework: { name: 'django' }, }, metadata: { projectId: 'import-test-project', tags: ['backend', 'python'], }, }, { id: 'test-import-2', type: 'recommendation', timestamp: new Date().toISOString(), data: { recommended: 'mkdocs', confidence: 0.8, }, metadata: { projectId: 'import-test-project', tags: ['documentation'], }, }, ], }; await fs.writeFile(testExportPath, JSON.stringify(testData, null, 2)); }); test('should import memories from JSON file', async () => { const importOptions: ImportOptions = { format: 'json', mode: 'append', validation: 'strict', conflictResolution: 'skip', backup: false, dryRun: false, }; const result = await exportImportSystem.importMemories(testExportPath, importOptions); expect(result).toBeDefined(); expect(result.success).toBe(true); expect(result.imported).toBe(2); expect(result.skipped).toBe(0); expect(result.errors).toBe(0); // Verify memories were imported const searchResults = await memoryManager.search('import-test-project'); expect(searchResults.length).toBeGreaterThanOrEqual(2); }); test('should handle import conflicts', async () => { // First import const importOptions: ImportOptions = { format: 'json', mode: 'append', validation: 'loose', conflictResolution: 'skip', backup: false, dryRun: false, }; await exportImportSystem.importMemories(testExportPath, importOptions); // Second import with same data (should skip duplicates) const result2 = await exportImportSystem.importMemories(testExportPath, importOptions); expect(result2.success).toBe(true); expect(result2.skipped).toBeGreaterThan(0); }); test('should validate imported data', async () => { // Create invalid test data const invalidDataPath = path.join(exportDir, 'invalid-import.json'); const invalidData = { memories: [ { // Missing required fields type: 'invalid', data: null, }, ], }; await fs.writeFile(invalidDataPath, JSON.stringify(invalidData)); const importOptions: ImportOptions = { format: 'json', mode: 'append', validation: 'strict', conflictResolution: 'skip', backup: false, dryRun: false, }; const result = await exportImportSystem.importMemories(invalidDataPath, importOptions); expect(result.success).toBe(false); expect(result.errors).toBeGreaterThan(0); expect(Array.isArray(result.errorDetails)).toBe(true); expect(result.errorDetails.length).toBeGreaterThan(0); }); test('should perform dry run import', async () => { const importOptions: ImportOptions = { format: 'json', mode: 'append', validation: 'strict', conflictResolution: 'skip', backup: false, dryRun: true, }; const result = await exportImportSystem.importMemories(testExportPath, importOptions); expect(result.success).toBe(true); // In dry run mode, nothing should be actually imported expect(result.imported).toBe(0); // Nothing actually imported in dry run // Verify no memories were actually imported const searchResults = await memoryManager.search('import-test-project'); expect(searchResults.length).toBe(0); }); test('should create backup before import', async () => { // Add some existing data memoryManager.setContext({ projectId: 'existing-data' }); await memoryManager.remember('analysis', { existing: true }); const importOptions: ImportOptions = { format: 'json', mode: 'replace', validation: 'loose', conflictResolution: 'overwrite', backup: true, dryRun: false, }; const result = await exportImportSystem.importMemories(testExportPath, importOptions); expect(result.success).toBe(true); // Backup creation is handled internally during import process // Verify that the import was successful expect(result.success).toBe(true); }); }); describe('Data Migration and Transformation', () => { test('should transform data during import', async () => { const sourceDataPath = path.join(exportDir, 'source-data.json'); const sourceData = { memories: [ { id: 'transform-test-1', type: 'analysis', timestamp: new Date().toISOString(), data: { // Old format lang: 'typescript', fw: 'react', }, metadata: { project: 'transform-test', }, }, ], }; await fs.writeFile(sourceDataPath, JSON.stringify(sourceData)); const importOptions: ImportOptions = { format: 'json', mode: 'append', validation: 'loose', conflictResolution: 'skip', backup: false, dryRun: false, mapping: { 'data.lang': 'data.language.primary', 'data.fw': 'data.framework.name', 'metadata.project': 'metadata.projectId', }, transformation: { enabled: true, rules: [ { field: 'data.language.primary', operation: 'transform', params: { value: 'typescript' }, }, ], }, }; const result = await exportImportSystem.importMemories(sourceDataPath, importOptions); expect(result.success).toBe(true); // Transformation should result in successful import expect(result.imported).toBeGreaterThan(0); // Verify transformation worked const imported = await memoryManager.search('transform-test'); expect(imported.length).toBe(1); expect(imported[0].data.language?.primary).toBe('typescript'); expect(imported[0].data.framework?.name).toBe('react'); expect(imported[0].metadata.projectId).toBe('transform-test'); }); test('should migrate between different versions', async () => { const oldVersionData = { version: '0.1.0', memories: [ { id: 'migration-test-1', type: 'analysis', timestamp: new Date().toISOString(), // Old schema project: 'migration-test', language: 'python', recommendation: 'mkdocs', }, ], }; const migrationPath = path.join(exportDir, 'migration-data.json'); await fs.writeFile(migrationPath, JSON.stringify(oldVersionData)); // Create a simple migration plan for testing const migrationPlan = await exportImportSystem.createMigrationPlan( { system: 'OldVersion', fields: {} }, { system: 'DocuMCP', fields: {} }, ); const result = await exportImportSystem.executeMigration(migrationPath, migrationPlan); expect(result.success).toBe(true); expect(result.imported).toBeGreaterThan(0); // Verify migration created proper structure const migrated = await memoryManager.search('migration-test'); expect(migrated.length).toBe(1); expect(migrated[0]).toHaveProperty('data'); expect(migrated[0]).toHaveProperty('metadata'); }); }); describe('Bulk Operations and Performance', () => { test('should handle large-scale export efficiently', async () => { memoryManager.setContext({ projectId: 'bulk-export-test' }); // Create many memories const promises = Array.from({ length: 100 }, (_, i) => memoryManager.remember('analysis', { index: i, content: `bulk test content ${i}`, }), ); await Promise.all(promises); const exportOptions: ExportOptions = { format: 'jsonl', includeMetadata: true, includeLearning: false, includeKnowledgeGraph: false, }; const startTime = Date.now(); const exportPath = path.join(exportDir, 'bulk-export.jsonl'); const result = await exportImportSystem.exportMemories(exportPath, exportOptions); const exportTime = Date.now() - startTime; expect(result.success).toBe(true); expect(result.entries).toBe(100); expect(exportTime).toBeLessThan(10000); // Should complete within 10 seconds }); test('should provide progress updates for long operations', async () => { memoryManager.setContext({ projectId: 'progress-test' }); // Add test data await memoryManager.remember('analysis', { progressTest: true }); const progressUpdates: number[] = []; exportImportSystem.on('export-progress', (progress: number) => { progressUpdates.push(progress); }); const exportOptions: ExportOptions = { format: 'json', includeMetadata: true, includeLearning: false, includeKnowledgeGraph: false, }; const exportPath = path.join(exportDir, 'progress-export.json'); await exportImportSystem.exportMemories(exportPath, exportOptions); // Progress updates might not be generated for small datasets expect(Array.isArray(progressUpdates)).toBe(true); }); }); describe('Error Handling and Recovery', () => { test('should handle file system errors gracefully', async () => { const invalidPath = '/invalid/path/that/does/not/exist/export.json'; const exportOptions: ExportOptions = { format: 'json', includeMetadata: true, includeLearning: false, includeKnowledgeGraph: false, }; const result = await exportImportSystem.exportMemories(invalidPath, exportOptions); expect(result.success).toBe(false); expect(result.errors.length).toBeGreaterThan(0); }); test('should recover from partial import failures', async () => { const partialDataPath = path.join(exportDir, 'partial-data.json'); const partialData = { memories: [ { id: 'valid-memory', type: 'analysis', timestamp: new Date().toISOString(), data: { valid: true }, metadata: { projectId: 'partial-test' }, }, { // Invalid memory id: 'invalid-memory', type: null, data: null, }, ], }; await fs.writeFile(partialDataPath, JSON.stringify(partialData)); const importOptions: ImportOptions = { format: 'json', mode: 'append', validation: 'loose', conflictResolution: 'skip', backup: false, dryRun: false, }; const result = await exportImportSystem.importMemories(partialDataPath, importOptions); expect(result.imported).toBe(1); // Only valid memory imported expect(result.errors).toBe(1); // One error for invalid memory expect(Array.isArray(result.errorDetails)).toBe(true); expect(result.errorDetails.length).toBe(1); }); test('should validate data integrity', async () => { const corruptDataPath = path.join(exportDir, 'corrupt-data.json'); await fs.writeFile(corruptDataPath, '{ invalid json'); const importOptions: ImportOptions = { format: 'json', mode: 'append', validation: 'strict', conflictResolution: 'skip', backup: false, dryRun: false, }; const result = await exportImportSystem.importMemories(corruptDataPath, importOptions); expect(result.success).toBe(false); expect(result.errors).toBeGreaterThan(0); }); }); });