documcp

/** * Comprehensive unit tests for Memory Storage System * Tests JSONL storage, indexing, CRUD operations, and performance * Part of Issue #54 - Core Memory System Unit Tests */ import { promises as fs } from 'fs'; import path from 'path'; import os from 'os'; import { JSONLStorage, MemoryEntry } from '../../src/memory/storage.js'; describe('JSONLStorage', () => { let storage: JSONLStorage; let tempDir: string; beforeEach(async () => { // Create unique temp directory for each test tempDir = path.join( os.tmpdir(), `memory-storage-test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, ); await fs.mkdir(tempDir, { recursive: true }); storage = new JSONLStorage(tempDir); await storage.initialize(); }); afterEach(async () => { // Cleanup temp directory try { await fs.rm(tempDir, { recursive: true, force: true }); } catch (error) { // Ignore cleanup errors } }); describe('Basic Storage Operations', () => { test('should create storage instance and initialize', async () => { expect(storage).toBeDefined(); expect(storage).toBeInstanceOf(JSONLStorage); // Verify storage directory was created const stats = await fs.stat(tempDir); expect(stats.isDirectory()).toBe(true); }); test('should append and retrieve memory entries', async () => { const entry = { timestamp: new Date().toISOString(), type: 'analysis' as const, data: { project: 'test-project', result: 'success' }, metadata: { projectId: 'test-proj', tags: ['test'] }, }; const stored = await storage.append(entry); expect(stored.id).toBeDefined(); expect(stored.checksum).toBeDefined(); expect(stored.type).toBe('analysis'); expect(stored.data).toEqual(entry.data); }); test('should handle different entry types', async () => { const entryTypes: Array<MemoryEntry['type']> = [ 'analysis', 'recommendation', 'deployment', 'configuration', 'interaction', ]; for (const type of entryTypes) { const entry = { timestamp: new Date().toISOString(), type, data: { testType: type }, metadata: { projectId: 'test-types' }, }; const stored = await storage.append(entry); expect(stored.type).toBe(type); expect(stored.data.testType).toBe(type); } }); test('should generate unique IDs for different entries', async () => { const entry1 = { timestamp: new Date().toISOString(), type: 'analysis' as const, data: { project: 'test-1' }, metadata: { projectId: 'test-1' }, }; const entry2 = { timestamp: new Date().toISOString(), type: 'analysis' as const, data: { project: 'test-2' }, metadata: { projectId: 'test-2' }, }; const stored1 = await storage.append(entry1); const stored2 = await storage.append(entry2); expect(stored1.id).not.toBe(stored2.id); expect(stored1.checksum).not.toBe(stored2.checksum); }); test('should generate same ID for identical entries', async () => { const entry = { timestamp: new Date().toISOString(), type: 'analysis' as const, data: { project: 'identical-test' }, metadata: { projectId: 'identical' }, }; const stored1 = await storage.append(entry); const stored2 = await storage.append(entry); expect(stored1.id).toBe(stored2.id); expect(stored1.checksum).toBe(stored2.checksum); }); }); describe('File Management', () => { test('should create proper JSONL file structure', async () => { const entry = { timestamp: '2024-01-15T10:30:00.000Z', type: 'analysis' as const, data: { fileTest: true }, metadata: { projectId: 'file-proj' }, }; await storage.append(entry); // Check that file was created with expected name pattern const files = await fs.readdir(tempDir); const jsonlFiles = files.filter((f) => f.endsWith('.jsonl')); expect(jsonlFiles.length).toBeGreaterThan(0); // Should have analysis_2024_01.jsonl const expectedFile = 'analysis_2024_01.jsonl'; expect(jsonlFiles).toContain(expectedFile); // Verify file contains the entry const filePath = path.join(tempDir, expectedFile); const content = await fs.readFile(filePath, 'utf-8'); const lines = content.trim().split('\n'); expect(lines.length).toBeGreaterThan(0); const parsedEntry = JSON.parse(lines[0]); expect(parsedEntry.data.fileTest).toBe(true); }); test('should organize files by type and date', async () => { const entries = [ { timestamp: '2024-01-15T10:30:00.000Z', type: 'analysis' as const, data: { test: 'analysis-jan' }, metadata: { projectId: 'date-test' }, }, { timestamp: '2024-02-15T10:30:00.000Z', type: 'analysis' as const, data: { test: 'analysis-feb' }, metadata: { projectId: 'date-test' }, }, { timestamp: '2024-01-15T10:30:00.000Z', type: 'recommendation' as const, data: { test: 'recommendation-jan' }, metadata: { projectId: 'date-test' }, }, ]; for (const entry of entries) { await storage.append(entry); } const files = await fs.readdir(tempDir); const jsonlFiles = files.filter((f) => f.endsWith('.jsonl')); expect(jsonlFiles).toContain('analysis_2024_01.jsonl'); expect(jsonlFiles).toContain('analysis_2024_02.jsonl'); expect(jsonlFiles).toContain('recommendation_2024_01.jsonl'); }); test('should handle index persistence', async () => { const entry = { timestamp: new Date().toISOString(), type: 'configuration' as const, data: { indexTest: true }, metadata: { projectId: 'index-test' }, }; await storage.append(entry); // Check that index file was created const indexPath = path.join(tempDir, '.index.json'); const indexExists = await fs .access(indexPath) .then(() => true) .catch(() => false); expect(indexExists).toBe(true); // Index should contain entry information const indexContent = await fs.readFile(indexPath, 'utf-8'); const indexData = JSON.parse(indexContent); expect(typeof indexData).toBe('object'); expect(Array.isArray(indexData.entries)).toBe(true); expect(indexData.entries.length).toBeGreaterThan(0); }); }); describe('Data Integrity', () => { test('should generate checksums for data integrity', async () => { const entry = { timestamp: new Date().toISOString(), type: 'deployment' as const, data: { integrity: 'test', checkData: 'important' }, metadata: { projectId: 'integrity-test' }, }; const stored = await storage.append(entry); expect(stored.checksum).toBeDefined(); expect(typeof stored.checksum).toBe('string'); expect(stored.checksum?.length).toBe(32); // MD5 hash length }); test('should handle entry timestamps correctly', async () => { const customTimestamp = '2024-06-15T14:30:00.000Z'; const entry = { timestamp: customTimestamp, type: 'interaction' as const, data: { timestampTest: true }, metadata: { projectId: 'timestamp-test' }, }; const stored = await storage.append(entry); expect(stored.timestamp).toBe(customTimestamp); }); test('should auto-generate timestamp if not provided', async () => { const entry = { timestamp: '', // Will be auto-generated type: 'analysis' as const, data: { autoTimestamp: true }, metadata: { projectId: 'auto-timestamp-test' }, }; const beforeTime = new Date().toISOString(); const stored = await storage.append(entry); const afterTime = new Date().toISOString(); expect(stored.timestamp).toBeDefined(); expect(stored.timestamp >= beforeTime).toBe(true); expect(stored.timestamp <= afterTime).toBe(true); }); }); describe('Metadata Handling', () => { test('should preserve metadata structure', async () => { const metadata = { projectId: 'metadata-test', repository: 'github.com/test/repo', ssg: 'docusaurus', tags: ['frontend', 'typescript'], version: '1.0.0', }; const entry = { timestamp: new Date().toISOString(), type: 'recommendation' as const, data: { recommendation: 'use-docusaurus' }, metadata, }; const stored = await storage.append(entry); expect(stored.metadata).toEqual(metadata); expect(stored.metadata.projectId).toBe('metadata-test'); expect(stored.metadata.tags).toEqual(['frontend', 'typescript']); }); test('should handle optional metadata fields', async () => { const entry = { timestamp: new Date().toISOString(), type: 'analysis' as const, data: { minimal: true }, metadata: { projectId: 'minimal-test' }, }; const stored = await storage.append(entry); expect(stored.metadata.projectId).toBe('minimal-test'); expect(stored.metadata.repository).toBeUndefined(); expect(stored.metadata.tags).toBeUndefined(); }); test('should handle compression metadata', async () => { const metadata = { projectId: 'compression-test', compressed: true, compressionType: 'gzip', compressedAt: new Date().toISOString(), originalSize: 1024, }; const entry = { timestamp: new Date().toISOString(), type: 'configuration' as const, data: { compressed: 'data' }, metadata, }; const stored = await storage.append(entry); expect(stored.metadata.compressed).toBe(true); expect(stored.metadata.compressionType).toBe('gzip'); expect(stored.metadata.originalSize).toBe(1024); }); }); describe('Performance and Concurrency', () => { test('should handle concurrent writes safely', async () => { const concurrentWrites = 10; const promises: Promise<MemoryEntry>[] = []; // Create multiple concurrent append operations for (let i = 0; i < concurrentWrites; i++) { const promise = storage.append({ timestamp: new Date().toISOString(), type: 'analysis', data: { index: i, concurrent: true }, metadata: { projectId: 'concurrent-test' }, }); promises.push(promise); } const results = await Promise.all(promises); expect(results).toHaveLength(concurrentWrites); // All IDs should be unique (since data is different) const ids = results.map((r) => r.id); expect(new Set(ids).size).toBe(concurrentWrites); // All should have correct structure results.forEach((result, index) => { expect(result.data.index).toBe(index); expect(result.metadata.projectId).toBe('concurrent-test'); }); }); test('should handle bulk append operations efficiently', async () => { const startTime = Date.now(); const bulkSize = 50; // Append bulk entries for (let i = 0; i < bulkSize; i++) { await storage.append({ timestamp: new Date().toISOString(), type: i % 2 === 0 ? 'analysis' : 'recommendation', data: { index: i, bulk: true }, metadata: { projectId: 'bulk-test', }, }); } const appendTime = Date.now() - startTime; expect(appendTime).toBeLessThan(5000); // Should complete within 5 seconds // Verify files were created const files = await fs.readdir(tempDir); const jsonlFiles = files.filter((f) => f.endsWith('.jsonl')); expect(jsonlFiles.length).toBeGreaterThan(0); }); test('should maintain performance with large data entries', async () => { const largeData = { description: 'x'.repeat(10000), // 10KB string array: new Array(1000).fill(0).map((_, i) => ({ id: i, data: `large-item-${i}`, metadata: { processed: true }, })), }; const entry = { timestamp: new Date().toISOString(), type: 'analysis' as const, data: largeData, metadata: { projectId: 'large-test' }, }; const startTime = Date.now(); const stored = await storage.append(entry); const appendTime = Date.now() - startTime; expect(appendTime).toBeLessThan(1000); // Should append within 1 second expect(stored.data.description).toHaveLength(10000); expect(stored.data.array).toHaveLength(1000); }); }); describe('Error Handling and Edge Cases', () => { test('should handle special characters in data', async () => { const entry = { timestamp: new Date().toISOString(), type: 'interaction' as const, data: { message: 'Special chars: äöü 🚀 @#$%^&*()[]{}|\\:";\'<>?,./`~', unicode: '测试中文字符', emoji: '🎉🔥💯⚡🚀', json: { nested: { deeply: { value: 'test' } } }, }, metadata: { projectId: 'special-chars-项目-🏗️', tags: ['special', 'unicode', '特殊字符'], }, }; const stored = await storage.append(entry); expect(stored.data.message).toContain('Special chars'); expect(stored.data.unicode).toBe('测试中文字符'); expect(stored.data.emoji).toBe('🎉🔥💯⚡🚀'); expect(stored.metadata.projectId).toBe('special-chars-项目-🏗️'); }); test('should handle empty data gracefully', async () => { const entry = { timestamp: new Date().toISOString(), type: 'configuration' as const, data: {}, metadata: { projectId: 'empty-test' }, }; const stored = await storage.append(entry); expect(stored.data).toEqual({}); expect(stored.id).toBeDefined(); expect(stored.checksum).toBeDefined(); }); test('should handle missing storage directory', async () => { const nonExistentDir = path.join(tempDir, 'non-existent', 'deeply', 'nested'); const newStorage = new JSONLStorage(nonExistentDir); // Should create directory during initialization await newStorage.initialize(); const stats = await fs.stat(nonExistentDir); expect(stats.isDirectory()).toBe(true); // Should be able to append entries const entry = { timestamp: new Date().toISOString(), type: 'analysis' as const, data: { recovery: true }, metadata: { projectId: 'recovery-test' }, }; const stored = await newStorage.append(entry); expect(stored.data.recovery).toBe(true); }); test('should maintain data consistency across operations', async () => { const entries = [ { timestamp: new Date().toISOString(), type: 'analysis' as const, data: { step: 1, consistency: 'test' }, metadata: { projectId: 'consistency-test' }, }, { timestamp: new Date().toISOString(), type: 'recommendation' as const, data: { step: 2, consistency: 'test' }, metadata: { projectId: 'consistency-test' }, }, { timestamp: new Date().toISOString(), type: 'deployment' as const, data: { step: 3, consistency: 'test' }, metadata: { projectId: 'consistency-test' }, }, ]; const storedEntries = []; for (const entry of entries) { const stored = await storage.append(entry); storedEntries.push(stored); } // Verify all entries were stored correctly expect(storedEntries).toHaveLength(3); storedEntries.forEach((stored, index) => { expect(stored.data.step).toBe(index + 1); expect(stored.metadata.projectId).toBe('consistency-test'); expect(stored.id).toBeDefined(); expect(stored.checksum).toBeDefined(); }); // All IDs should be unique const ids = storedEntries.map((s) => s.id); expect(new Set(ids).size).toBe(3); }); }); });