FilePersistenceAdapter.ts•11.7 kB
import * as fs from 'fs/promises';
import * as path from 'path';
import * as zlib from 'zlib';
import { promisify } from 'util';
import { SyncMetadata } from '../../services/GitHubStateSyncService';
import { Logger } from '../logger/index';
const gzip = promisify(zlib.gzip);
const gunzip = promisify(zlib.gunzip);
export interface PersistenceOptions {
cacheDirectory: string;
enableCompression: boolean;
maxBackups: number;
atomicWrites: boolean;
}
export interface PersistenceStats {
totalMetadataEntries: number;
fileSize: number;
lastModified: Date;
compressionRatio?: number;
}
export class FilePersistenceAdapter {
private readonly logger = Logger.getInstance();
private readonly options: PersistenceOptions;
private readonly metadataFile: string;
private readonly lockFile: string;
private readonly tempDir: string;
private directoryInitialized = false;
constructor(options: Partial<PersistenceOptions> = {}) {
this.options = {
cacheDirectory: options.cacheDirectory || '.mcp-cache',
enableCompression: options.enableCompression ?? true,
maxBackups: options.maxBackups || 5,
atomicWrites: options.atomicWrites ?? true
};
this.metadataFile = path.join(this.options.cacheDirectory, 'metadata.json');
this.lockFile = path.join(this.options.cacheDirectory, 'metadata.lock');
this.tempDir = path.join(this.options.cacheDirectory, 'temp');
}
/**
* Load all metadata from persistence
*/
async loadMetadata(): Promise<SyncMetadata[]> {
try {
await this.ensureDirectoryExists();
await this.acquireLock();
if (!await this.fileExists(this.metadataFile)) {
this.logger.info("No existing metadata file found, starting fresh");
return [];
}
const data = await this.readFile(this.metadataFile);
const metadata = JSON.parse(data) as SyncMetadata[];
this.logger.info(`Loaded ${metadata.length} metadata entries from persistence`);
return metadata;
} catch (error) {
this.logger.error("Failed to load metadata:", error);
// Try to recover from backup
const recovered = await this.recoverFromBackup();
if (recovered) {
return recovered;
}
// If all else fails, start fresh
this.logger.warn("Starting with empty metadata due to load failure");
return [];
} finally {
await this.releaseLock();
}
}
/**
* Save single metadata entry
*/
async saveMetadata(metadata: SyncMetadata): Promise<void> {
try {
await this.ensureDirectoryExists();
await this.acquireLock();
// Load existing metadata
const existingMetadata = await this.loadMetadataInternal();
// Update or add the metadata entry
const existingIndex = existingMetadata.findIndex(
m => m.resourceId === metadata.resourceId && m.resourceType === metadata.resourceType
);
if (existingIndex >= 0) {
existingMetadata[existingIndex] = metadata;
} else {
existingMetadata.push(metadata);
}
// Save updated metadata
await this.saveAllMetadata(existingMetadata);
} catch (error) {
this.logger.error("Failed to save metadata:", error);
throw error;
} finally {
await this.releaseLock();
}
}
/**
* Save all metadata entries
*/
async saveAllMetadata(metadata: SyncMetadata[]): Promise<void> {
try {
await this.acquireLock();
// Create backup before saving
await this.createBackup();
// Prepare data for writing
const data = JSON.stringify(metadata, null, 2);
if (this.options.atomicWrites) {
await this.writeFileAtomic(this.metadataFile, data);
} else {
await this.writeFile(this.metadataFile, data);
}
this.logger.debug(`Saved ${metadata.length} metadata entries to persistence`);
} catch (error) {
this.logger.error("Failed to save all metadata:", error);
throw error;
} finally {
await this.releaseLock();
}
}
/**
* Get persistence statistics
*/
async getStats(): Promise<PersistenceStats> {
try {
const stats = await fs.stat(this.metadataFile);
const metadata = await this.loadMetadata();
const result: PersistenceStats = {
totalMetadataEntries: metadata.length,
fileSize: stats.size,
lastModified: new Date(stats.mtime)
};
// Calculate compression ratio if compression is enabled
if (this.options.enableCompression) {
const uncompressedSize = JSON.stringify(metadata).length;
result.compressionRatio = stats.size / uncompressedSize;
}
return result;
} catch (error) {
this.logger.error("Failed to get persistence stats:", error);
throw error;
}
}
/**
* Clean up old backups and temporary files
*/
async cleanup(): Promise<void> {
try {
// Clean up old backups
await this.cleanupBackups();
// Clean up temporary files
await this.cleanupTempFiles();
this.logger.info("Persistence cleanup completed");
} catch (error) {
this.logger.error("Failed to cleanup persistence files:", error);
}
}
/**
* Ensure cache directory exists
*/
private async ensureDirectoryExists(): Promise<void> {
if (this.directoryInitialized) {
return;
}
try {
// Create cache directory first
await fs.mkdir(this.options.cacheDirectory, { recursive: true });
// Then create temp directory
await fs.mkdir(this.tempDir, { recursive: true });
this.directoryInitialized = true;
} catch (error) {
this.logger.error("Failed to create cache directory:", error);
throw error;
}
}
/**
* Acquire file lock
*/
private async acquireLock(timeout = 5000): Promise<void> {
// Skip locking in test environment
if (process.env.NODE_ENV === 'test') {
return;
}
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
try {
await fs.writeFile(this.lockFile, process.pid.toString(), { flag: 'wx' });
return;
} catch (error) {
// Lock file exists, wait and retry
await new Promise(resolve => setTimeout(resolve, 100));
}
}
throw new Error("Failed to acquire file lock within timeout");
}
/**
* Release file lock
*/
private async releaseLock(): Promise<void> {
// Skip locking in test environment
if (process.env.NODE_ENV === 'test') {
return;
}
try {
await fs.unlink(this.lockFile);
} catch (error) {
// Lock file might not exist, which is fine
this.logger.debug("Lock file already removed or doesn't exist");
}
}
/**
* Check if file exists
*/
private async fileExists(filePath: string): Promise<boolean> {
try {
await fs.access(filePath);
return true;
} catch {
return false;
}
}
/**
* Read file with optional compression
*/
private async readFile(filePath: string): Promise<string> {
const buffer = await fs.readFile(filePath);
if (this.options.enableCompression) {
try {
const decompressed = await gunzip(buffer);
return decompressed.toString('utf8');
} catch {
// File might not be compressed, try reading as plain text
return buffer.toString('utf8');
}
}
return buffer.toString('utf8');
}
/**
* Write file with optional compression
*/
private async writeFile(filePath: string, data: string): Promise<void> {
let buffer: Buffer;
if (this.options.enableCompression) {
buffer = await gzip(Buffer.from(data, 'utf8'));
} else {
buffer = Buffer.from(data, 'utf8');
}
await fs.writeFile(filePath, buffer);
}
/**
* Write file atomically using temporary file
*/
private async writeFileAtomic(filePath: string, data: string): Promise<void> {
const tempFile = path.join(this.tempDir, `metadata-${Date.now()}-${Math.random().toString(36).substr(2, 9)}.tmp`);
try {
await this.writeFile(tempFile, data);
await fs.rename(tempFile, filePath);
} catch (error) {
// Clean up temp file on error
try {
await fs.unlink(tempFile);
} catch {
// Ignore cleanup errors
}
throw error;
}
}
/**
* Load metadata without locking (internal use)
*/
private async loadMetadataInternal(): Promise<SyncMetadata[]> {
if (!await this.fileExists(this.metadataFile)) {
return [];
}
const data = await this.readFile(this.metadataFile);
return JSON.parse(data) as SyncMetadata[];
}
/**
* Create backup of current metadata file
*/
private async createBackup(): Promise<void> {
if (!await this.fileExists(this.metadataFile)) {
return;
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const backupFile = path.join(this.options.cacheDirectory, `metadata-backup-${timestamp}.json`);
try {
await fs.copyFile(this.metadataFile, backupFile);
this.logger.debug(`Created backup: ${backupFile}`);
} catch (error) {
this.logger.warn("Failed to create backup:", error);
}
}
/**
* Recover from backup file
*/
private async recoverFromBackup(): Promise<SyncMetadata[] | null> {
try {
const files = await fs.readdir(this.options.cacheDirectory);
const backupFiles = files
.filter(f => f.startsWith('metadata-backup-') && f.endsWith('.json'))
.sort()
.reverse(); // Most recent first
for (const backupFile of backupFiles) {
try {
const backupPath = path.join(this.options.cacheDirectory, backupFile);
const data = await this.readFile(backupPath);
const metadata = JSON.parse(data) as SyncMetadata[];
this.logger.info(`Recovered ${metadata.length} metadata entries from backup: ${backupFile}`);
return metadata;
} catch (error) {
this.logger.warn(`Failed to recover from backup ${backupFile}:`, error);
}
}
} catch (error) {
this.logger.error("Failed to list backup files:", error);
}
return null;
}
/**
* Clean up old backup files
*/
private async cleanupBackups(): Promise<void> {
try {
const files = await fs.readdir(this.options.cacheDirectory);
const backupFiles = files
.filter(f => f.startsWith('metadata-backup-') && f.endsWith('.json'))
.sort();
if (backupFiles.length > this.options.maxBackups) {
const filesToDelete = backupFiles.slice(0, backupFiles.length - this.options.maxBackups);
for (const file of filesToDelete) {
const filePath = path.join(this.options.cacheDirectory, file);
await fs.unlink(filePath);
this.logger.debug(`Deleted old backup: ${file}`);
}
}
} catch (error) {
this.logger.warn("Failed to cleanup backup files:", error);
}
}
/**
* Clean up temporary files
*/
private async cleanupTempFiles(): Promise<void> {
try {
const files = await fs.readdir(this.tempDir);
const now = Date.now();
const maxAge = 24 * 60 * 60 * 1000; // 24 hours
for (const file of files) {
const filePath = path.join(this.tempDir, file);
const stats = await fs.stat(filePath);
if (now - stats.mtime.getTime() > maxAge) {
await fs.unlink(filePath);
this.logger.debug(`Deleted old temp file: ${file}`);
}
}
} catch (error) {
this.logger.warn("Failed to cleanup temp files:", error);
}
}
}