cache-manager.ts•20.6 kB
/**
* Multi-Level Cache Manager for GEPA
* Implements L1 (memory) and L2 (disk) caching with LRU eviction and TTL support
*/
import { EventEmitter } from 'events';
import { promises as fs } from 'fs';
import { join } from 'path';
import { createHash } from 'crypto';
import { gzipSync, gunzipSync } from 'zlib';
import { MemoryLeakIntegration } from '../memory-leak-detector';
/**
* Cache configuration interface
*/
export interface CacheConfig {
// L1 Memory Cache
l1MaxSize: number;
l1MaxEntries: number;
l1DefaultTtl: number;
// L2 Disk Cache
l2Enabled: boolean;
l2Directory: string;
l2MaxSize: number;
l2MaxEntries: number;
l2DefaultTtl: number;
l2CompressionEnabled: boolean;
// Cache Features
enableStatistics: boolean;
enableCacheWarming: boolean;
autoCleanupInterval: number;
// Eviction Policies
l1EvictionPolicy: 'lru' | 'lfu' | 'ttl';
l2EvictionPolicy: 'lru' | 'lfu' | 'ttl';
}
/**
* Cache entry with metadata
*/
export interface CacheEntry<T = unknown> {
key: string;
value: T;
ttl: number;
createdAt: number;
lastAccessed: number;
accessCount: number;
size: number;
compressed: boolean;
}
/**
* Cache statistics
*/
export interface CacheStatistics {
l1: {
hits: number;
misses: number;
hitRate: number;
size: number;
entries: number;
maxSize: number;
maxEntries: number;
};
l2: {
hits: number;
misses: number;
hitRate: number;
size: number;
entries: number;
maxSize: number;
maxEntries: number;
};
overall: {
hits: number;
misses: number;
hitRate: number;
evictions: number;
compressionRatio: number;
};
}
/**
* Cache warming strategy
*/
export interface CacheWarmingStrategy {
enabled: boolean;
keys: string[];
dataLoader: (key: string) => Promise<unknown>;
priority: 'high' | 'medium' | 'low';
}
/**
* Main cache manager implementing multi-level caching
*/
export class CacheManager extends EventEmitter {
private readonly config: CacheConfig;
// L1 Memory Cache
private readonly l1Cache = new Map<string, CacheEntry>();
private readonly l1AccessOrder: string[] = [];
// L2 Disk Cache metadata
private readonly l2Index = new Map<string, { filePath: string; metadata: Omit<CacheEntry, 'value'> }>();
// Statistics
private readonly stats: CacheStatistics = {
l1: { hits: 0, misses: 0, hitRate: 0, size: 0, entries: 0, maxSize: 0, maxEntries: 0 },
l2: { hits: 0, misses: 0, hitRate: 0, size: 0, entries: 0, maxSize: 0, maxEntries: 0 },
overall: { hits: 0, misses: 0, hitRate: 0, evictions: 0, compressionRatio: 0 }
};
// Cleanup and management
private cleanupInterval?: ReturnType<typeof setInterval>;
private isShuttingDown = false;
constructor(config: Partial<CacheConfig> = {}) {
super();
this.config = {
// L1 Defaults (16MB, 10k entries, 1 hour)
l1MaxSize: config.l1MaxSize ?? 16 * 1024 * 1024,
l1MaxEntries: config.l1MaxEntries ?? 10000,
l1DefaultTtl: config.l1DefaultTtl ?? 3600000,
// L2 Defaults (1GB, 100k entries, 24 hours)
l2Enabled: config.l2Enabled ?? true,
l2Directory: config.l2Directory ?? './cache',
l2MaxSize: config.l2MaxSize ?? 1024 * 1024 * 1024,
l2MaxEntries: config.l2MaxEntries ?? 100000,
l2DefaultTtl: config.l2DefaultTtl ?? 86400000,
l2CompressionEnabled: config.l2CompressionEnabled ?? true,
// Feature Defaults
enableStatistics: config.enableStatistics ?? true,
enableCacheWarming: config.enableCacheWarming ?? true,
autoCleanupInterval: config.autoCleanupInterval ?? 300000, // 5 minutes
// Eviction Defaults
l1EvictionPolicy: config.l1EvictionPolicy ?? 'lru',
l2EvictionPolicy: config.l2EvictionPolicy ?? 'lru',
...config
};
this.stats.l1.maxSize = this.config.l1MaxSize;
this.stats.l1.maxEntries = this.config.l1MaxEntries;
this.stats.l2.maxSize = this.config.l2MaxSize;
this.stats.l2.maxEntries = this.config.l2MaxEntries;
this.initializeCache();
this.setupMemoryLeakDetection();
}
/**
* Initialize cache subsystems
*/
private async initializeCache(): Promise<void> {
try {
// Initialize L2 disk cache
if (this.config.l2Enabled) {
await this.initializeL2Cache();
}
// Start cleanup interval
if (this.config.autoCleanupInterval > 0) {
this.cleanupInterval = setInterval(
() => this.performCleanup(),
this.config.autoCleanupInterval
);
}
this.emit('initialized');
} catch (error) {
this.emit('error', error);
throw error;
}
}
/**
* Get value from cache (checks L1 first, then L2)
*/
async get<T = unknown>(key: string): Promise<T | null> {
if (this.isShuttingDown) return null;
try {
// Try L1 cache first
const l1Entry = this.l1Cache.get(key);
if (l1Entry && !this.isExpired(l1Entry)) {
this.updateL1Access(key, l1Entry);
this.updateStats('l1', 'hit');
this.emit('hit', { level: 'l1', key });
return l1Entry.value as T;
} else if (l1Entry) {
// Expired entry in L1
this.l1Cache.delete(key);
this.removeFromAccessOrder(key);
}
// Try L2 cache if enabled
if (this.config.l2Enabled) {
const l2Value = await this.getFromL2<T>(key);
if (l2Value !== null) {
// Promote to L1 cache
await this.set(key, l2Value, { promoteToL1: true });
this.updateStats('l2', 'hit');
this.emit('hit', { level: 'l2', key });
return l2Value;
}
}
// Cache miss
this.updateStats('overall', 'miss');
this.emit('miss', { key });
return null;
} catch (error) {
this.emit('error', error);
return null;
}
}
/**
* Set value in cache with optional configuration
*/
async set<T = unknown>(
key: string,
value: T,
options: {
ttl?: number;
storeInL2?: boolean;
promoteToL1?: boolean;
priority?: 'high' | 'medium' | 'low';
} = {}
): Promise<boolean> {
if (this.isShuttingDown) return false;
try {
const ttl = options.ttl ?? this.config.l1DefaultTtl;
const size = this.calculateSize(value);
const entry: CacheEntry<T> = {
key,
value,
ttl,
createdAt: Date.now(),
lastAccessed: Date.now(),
accessCount: 1,
size,
compressed: false
};
// Store in L1 cache
if (options.promoteToL1 !== false) {
await this.setInL1(entry);
}
// Store in L2 cache if enabled and requested
if (this.config.l2Enabled && (options.storeInL2 ?? true)) {
await this.setInL2(entry);
}
// Track memory allocation for leak detection
MemoryLeakIntegration.trackCacheOperation('set', key, size);
this.emit('set', { key, level: 'both', size });
return true;
} catch (error) {
this.emit('error', error);
return false;
}
}
/**
* Delete value from both cache levels
*/
async delete(key: string): Promise<boolean> {
try {
let deleted = false;
// Delete from L1
if (this.l1Cache.has(key)) {
const entry = this.l1Cache.get(key)!;
this.l1Cache.delete(key);
this.removeFromAccessOrder(key);
this.stats.l1.size -= entry.size;
this.stats.l1.entries--;
deleted = true;
}
// Delete from L2
if (this.config.l2Enabled && this.l2Index.has(key)) {
await this.deleteFromL2(key);
deleted = true;
}
if (deleted) {
// Track memory deallocation for leak detection
MemoryLeakIntegration.trackCacheOperation('delete', key);
this.emit('delete', { key });
}
return deleted;
} catch (error) {
this.emit('error', error);
return false;
}
}
/**
* Clear all cache levels
*/
async clear(): Promise<void> {
try {
// Clear L1
this.l1Cache.clear();
this.l1AccessOrder.length = 0;
this.stats.l1.size = 0;
this.stats.l1.entries = 0;
// Clear L2
if (this.config.l2Enabled) {
await this.clearL2();
}
// Track bulk deallocation for leak detection
MemoryLeakIntegration.trackCacheOperation('clear', 'all');
this.emit('cleared');
} catch (error) {
this.emit('error', error);
throw error;
}
}
/**
* Warm cache with predefined data
*/
async warmCache(strategy: CacheWarmingStrategy): Promise<number> {
if (!this.config.enableCacheWarming || !strategy.enabled) {
return 0;
}
let warmedCount = 0;
try {
// Process keys by priority
const sortedKeys = this.sortKeysByPriority(strategy.keys, strategy.priority);
for (const key of sortedKeys) {
try {
const value = await strategy.dataLoader(key);
const success = await this.set(key, value, {
storeInL2: true,
promoteToL1: strategy.priority === 'high'
});
if (success) {
warmedCount++;
}
} catch (error) {
this.emit('warmingError', { key, error });
}
}
this.emit('warmed', { count: warmedCount, total: strategy.keys.length });
return warmedCount;
} catch (error) {
this.emit('error', error);
return warmedCount;
}
}
/**
* Get comprehensive cache statistics
*/
getStatistics(): CacheStatistics {
if (!this.config.enableStatistics) {
return this.stats;
}
// Calculate hit rates
const l1Total = this.stats.l1.hits + this.stats.l1.misses;
const l2Total = this.stats.l2.hits + this.stats.l2.misses;
const overallTotal = this.stats.overall.hits + this.stats.overall.misses;
this.stats.l1.hitRate = l1Total > 0 ? this.stats.l1.hits / l1Total : 0;
this.stats.l2.hitRate = l2Total > 0 ? this.stats.l2.hits / l2Total : 0;
this.stats.overall.hitRate = overallTotal > 0 ? this.stats.overall.hits / overallTotal : 0;
return { ...this.stats };
}
/**
* Shutdown cache manager and cleanup resources
*/
async shutdown(): Promise<void> {
this.isShuttingDown = true;
try {
// Stop cleanup interval
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
}
// Final cleanup
await this.performCleanup();
this.emit('shutdown');
} catch (error) {
this.emit('error', error);
}
}
// Private Helper Methods
/**
* Initialize L2 disk cache
*/
private async initializeL2Cache(): Promise<void> {
await fs.mkdir(this.config.l2Directory, { recursive: true });
// Load existing L2 index if available
const indexPath = join(this.config.l2Directory, 'cache-index.json');
try {
const indexData = await fs.readFile(indexPath, 'utf-8');
const index = JSON.parse(indexData);
for (const [key, data] of Object.entries(index)) {
this.l2Index.set(key, data as any);
}
this.stats.l2.entries = this.l2Index.size;
} catch {
// Index doesn't exist, start fresh
}
}
/**
* Set value in L1 cache with eviction
*/
private async setInL1<T>(entry: CacheEntry<T>): Promise<void> {
// Check if we need to evict entries
while (this.needsL1Eviction(entry.size)) {
await this.evictFromL1();
}
this.l1Cache.set(entry.key, entry);
this.updateL1AccessOrder(entry.key);
this.stats.l1.size += entry.size;
this.stats.l1.entries++;
}
/**
* Set value in L2 cache with eviction
*/
private async setInL2<T>(entry: CacheEntry<T>): Promise<void> {
const filePath = this.getL2FilePath(entry.key);
let data = JSON.stringify(entry.value);
let compressed = false;
// Apply compression if enabled and beneficial
if (this.config.l2CompressionEnabled && data.length > 1024) {
const compressedData = gzipSync(data);
if (compressedData.length < data.length * 0.8) {
data = compressedData.toString('base64');
compressed = true;
}
}
await fs.writeFile(filePath, data);
const metadata = { ...entry };
delete (metadata as any).value;
metadata.compressed = compressed;
this.l2Index.set(entry.key, { filePath, metadata });
this.stats.l2.size += entry.size;
this.stats.l2.entries++;
// Save updated index
await this.saveL2Index();
}
/**
* Get value from L2 cache
*/
private async getFromL2<T>(key: string): Promise<T | null> {
const indexEntry = this.l2Index.get(key);
if (!indexEntry || this.isExpired(indexEntry.metadata)) {
if (indexEntry) {
await this.deleteFromL2(key);
}
return null;
}
try {
let data = await fs.readFile(indexEntry.filePath, 'utf-8');
if (indexEntry.metadata.compressed) {
const compressedBuffer = Buffer.from(data, 'base64');
data = gunzipSync(compressedBuffer).toString();
}
const value = JSON.parse(data);
// Update access time
indexEntry.metadata.lastAccessed = Date.now();
indexEntry.metadata.accessCount++;
return value as T;
} catch {
// File corrupted or missing, remove from index
await this.deleteFromL2(key);
return null;
}
}
/**
* Delete entry from L2 cache
*/
private async deleteFromL2(key: string): Promise<void> {
const indexEntry = this.l2Index.get(key);
if (indexEntry) {
try {
await fs.unlink(indexEntry.filePath);
} catch {
// File might not exist
}
this.l2Index.delete(key);
this.stats.l2.size -= indexEntry.metadata.size;
this.stats.l2.entries--;
await this.saveL2Index();
}
}
/**
* Clear L2 cache
*/
private async clearL2(): Promise<void> {
// Delete all cache files
for (const [key] of this.l2Index) {
await this.deleteFromL2(key);
}
this.l2Index.clear();
this.stats.l2.size = 0;
this.stats.l2.entries = 0;
}
/**
* Check if entry has expired
*/
private isExpired(entry: Omit<CacheEntry, 'value'>): boolean {
return Date.now() - entry.createdAt > entry.ttl;
}
/**
* Calculate size of value in bytes
*/
private calculateSize(value: unknown): number {
return Buffer.byteLength(JSON.stringify(value), 'utf-8');
}
/**
* Update access patterns for L1
*/
private updateL1Access(key: string, entry: CacheEntry): void {
entry.lastAccessed = Date.now();
entry.accessCount++;
this.updateL1AccessOrder(key);
}
/**
* Update L1 access order for LRU
*/
private updateL1AccessOrder(key: string): void {
this.removeFromAccessOrder(key);
this.l1AccessOrder.push(key);
}
/**
* Remove key from access order
*/
private removeFromAccessOrder(key: string): void {
const index = this.l1AccessOrder.indexOf(key);
if (index !== -1) {
this.l1AccessOrder.splice(index, 1);
}
}
/**
* Check if L1 eviction is needed
*/
private needsL1Eviction(newEntrySize: number): boolean {
return (
this.stats.l1.size + newEntrySize > this.config.l1MaxSize ||
this.stats.l1.entries >= this.config.l1MaxEntries
);
}
/**
* Evict entry from L1 using configured policy
*/
private async evictFromL1(): Promise<void> {
if (this.l1Cache.size === 0) return;
let keyToEvict: string;
switch (this.config.l1EvictionPolicy) {
case 'lru':
keyToEvict = this.l1AccessOrder[0] || '';
break;
case 'lfu':
keyToEvict = this.findLFUKey() || '';
break;
case 'ttl':
keyToEvict = this.findExpiredKey() || this.l1AccessOrder[0] || '';
break;
default:
keyToEvict = this.l1AccessOrder[0] || '';
}
if (keyToEvict) {
const entry = this.l1Cache.get(keyToEvict);
if (entry) {
this.l1Cache.delete(keyToEvict);
this.removeFromAccessOrder(keyToEvict);
this.stats.l1.size -= entry.size;
this.stats.l1.entries--;
this.stats.overall.evictions++;
this.emit('evicted', { level: 'l1', key: keyToEvict, policy: this.config.l1EvictionPolicy });
}
}
}
/**
* Find least frequently used key
*/
private findLFUKey(): string {
let minCount = Infinity;
let lfuKey = '';
for (const [key, entry] of this.l1Cache) {
if (entry.accessCount < minCount) {
minCount = entry.accessCount;
lfuKey = key;
}
}
return lfuKey;
}
/**
* Find expired key
*/
private findExpiredKey(): string | null {
for (const [key, entry] of this.l1Cache) {
if (this.isExpired(entry)) {
return key;
}
}
return null;
}
/**
* Update cache statistics
*/
private updateStats(level: 'l1' | 'l2' | 'overall', type: 'hit' | 'miss'): void {
if (!this.config.enableStatistics) return;
if (type === 'hit') {
this.stats[level].hits++;
if (level !== 'overall') {
this.stats.overall.hits++;
}
} else {
this.stats[level].misses++;
if (level !== 'overall') {
this.stats.overall.misses++;
}
}
}
/**
* Sort keys by priority for cache warming
*/
private sortKeysByPriority(keys: string[], _priority?: string): string[] {
// For now, return as-is. In a more sophisticated implementation,
// we could analyze key patterns or use external priority information
return [...keys];
}
/**
* Get L2 file path for key
*/
private getL2FilePath(key: string): string {
const hash = createHash('sha256').update(key).digest('hex');
const subdir = hash.substring(0, 2);
const filename = `${hash}.cache`;
return join(this.config.l2Directory, subdir, filename);
}
/**
* Save L2 index to disk
*/
private async saveL2Index(): Promise<void> {
const indexPath = join(this.config.l2Directory, 'cache-index.json');
const indexData = Object.fromEntries(this.l2Index);
await fs.writeFile(indexPath, JSON.stringify(indexData, null, 2));
}
/**
* Perform periodic cleanup
*/
private async performCleanup(): Promise<void> {
try {
let cleanedCount = 0;
// Clean expired L1 entries
for (const [key, entry] of this.l1Cache) {
if (this.isExpired(entry)) {
await this.delete(key);
cleanedCount++;
}
}
// Clean expired L2 entries
if (this.config.l2Enabled) {
for (const [key, indexEntry] of this.l2Index) {
if (this.isExpired(indexEntry.metadata)) {
await this.deleteFromL2(key);
cleanedCount++;
}
}
}
this.emit('cleanup', {
l1Entries: this.stats.l1.entries,
l2Entries: this.stats.l2.entries,
cleanedCount
});
} catch (error) {
this.emit('error', error);
}
}
/**
* Setup memory leak detection integration
*/
private setupMemoryLeakDetection(): void {
// Initialize memory leak detection if not already done
MemoryLeakIntegration.initialize();
// Listen for cache eviction requests from memory leak detector
const detector = MemoryLeakIntegration.getDetector();
if (detector) {
detector.on('cacheEvictionRequested', async ({ severity }) => {
try {
if (severity === 'critical') {
// Aggressive cleanup - clear all caches
await this.clear();
} else if (severity === 'high') {
// Force eviction of 50% of L1 cache
const keysToEvict = Array.from(this.l1Cache.keys()).slice(0, Math.floor(this.l1Cache.size / 2));
for (const key of keysToEvict) {
await this.delete(key);
}
} else {
// Regular cleanup
await this.performCleanup();
}
this.emit('memoryLeakMitigation', { severity, action: 'eviction' });
} catch (error) {
this.emit('error', error);
}
});
}
}
}