/**
* Performance tests for cache system using real large repositories.
* Tests cold vs cached startup and validates 80% performance improvement target.
*/
import { describe, it, expect, beforeAll, afterAll } from '@jest/globals';
import { SymbolIndexer } from '../../src/symbol-search/symbol-indexer.js';
import { CacheManager } from '../../src/cache/cache-manager.js';
import { isCTagsAvailable } from '../../src/symbol-search/ctags-integration.js';
import { promises as fs } from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import os from 'os';
import { execSync } from 'child_process';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const TEST_DIR = path.join(__dirname, 'cache-performance-test');
const CACHE_DIR = path.join(os.tmpdir(), `.code-search-mcp-cache-performance-test-${Date.now()}`);
// Test repositories (we'll use popular open-source projects)
const TEST_REPOS = [
{
name: 'express',
url: 'https://github.com/expressjs/express.git',
description: 'Fast, unopinionated, minimalist web framework for Node.js',
},
{
name: 'lodash',
url: 'https://github.com/lodash/lodash.git',
description: 'A modern JavaScript utility library',
},
];
interface PerformanceResult {
repoName: string;
coldStartTime: number;
cachedStartTime: number;
improvement: number;
improvementPercent: number;
totalSymbols: number;
fileCount: number;
cacheSize: number;
}
describe('Cache Performance Tests', () => {
let ctagsAvailable = false;
let results: PerformanceResult[] = [];
beforeAll(async () => {
ctagsAvailable = await isCTagsAvailable();
if (!ctagsAvailable) {
console.warn('⚠️ ctags not available - skipping cache performance tests');
return;
}
// Create test directory
await fs.mkdir(TEST_DIR, { recursive: true });
console.log('\n' + '='.repeat(80));
console.log('CACHE PERFORMANCE TEST SUITE');
console.log('='.repeat(80));
});
afterAll(async () => {
if (results.length > 0) {
console.log('\n' + '='.repeat(80));
console.log('PERFORMANCE TEST RESULTS SUMMARY');
console.log('='.repeat(80));
for (const result of results) {
console.log(`\nRepository: ${result.repoName}`);
console.log(` Total Symbols: ${result.totalSymbols.toLocaleString()}`);
console.log(` Files Indexed: ${result.fileCount.toLocaleString()}`);
console.log(` Cache Size: ${(result.cacheSize / 1024 / 1024).toFixed(2)} MB`);
console.log(` Cold Start: ${result.coldStartTime.toLocaleString()}ms`);
console.log(` Cached Start: ${result.cachedStartTime.toLocaleString()}ms`);
console.log(` Improvement: ${result.improvement.toFixed(2)}x faster`);
console.log(` Time Saved: ${result.improvementPercent.toFixed(1)}%`);
}
// Calculate average improvement
const avgImprovement = results.reduce((sum, r) => sum + r.improvement, 0) / results.length;
const avgImprovementPercent = results.reduce((sum, r) => sum + r.improvementPercent, 0) / results.length;
console.log('\n' + '-'.repeat(80));
console.log(`AVERAGE IMPROVEMENT: ${avgImprovement.toFixed(2)}x faster (${avgImprovementPercent.toFixed(1)}% time saved)`);
// Informational performance rating (not pass/fail)
if (avgImprovementPercent >= 80) {
console.log('✅ EXCELLENT: Achieved 80%+ performance improvement target!');
} else if (avgImprovementPercent >= 50) {
console.log(`✓ GOOD: ${avgImprovementPercent.toFixed(1)}% time saved (target: 80%)`);
} else if (avgImprovementPercent >= 20) {
console.log(`ℹ MODERATE: ${avgImprovementPercent.toFixed(1)}% time saved (target: 80%)`);
} else {
console.log(`ℹ MEASURED: ${avgImprovementPercent.toFixed(1)}% time saved (varies by environment)`);
}
console.log('Note: Performance metrics are informational and vary by system resources');
console.log('='.repeat(80) + '\n');
}
// Cleanup
await fs.rm(TEST_DIR, { recursive: true, force: true }).catch(() => {});
await fs.rm(CACHE_DIR, { recursive: true, force: true }).catch(() => {});
});
describe('Real Repository Performance', () => {
for (const repo of TEST_REPOS) {
it(`should achieve 80%+ performance improvement on ${repo.name}`, async () => {
if (!ctagsAvailable) {
console.log('Skipping - ctags not available');
return;
}
const repoDir = path.join(TEST_DIR, repo.name);
const workspaceId = `perf-test-${repo.name}`;
console.log(`\n${'='.repeat(80)}`);
console.log(`Testing Repository: ${repo.name}`);
console.log(`Description: ${repo.description}`);
console.log(`${'='.repeat(80)}`);
// Clone repository
console.log(`\n[1/5] Cloning repository...`);
try {
execSync(`git clone --depth 1 ${repo.url} "${repoDir}"`, {
stdio: 'pipe',
timeout: 60000, // 60 second timeout
});
console.log(`✓ Repository cloned successfully`);
} catch (error) {
console.error(`✗ Failed to clone repository: ${error}`);
throw new Error(`Failed to clone ${repo.name}: ${error}`);
}
// Count files
const countFiles = async (dir: string): Promise<number> => {
let count = 0;
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (entry.name === '.git' || entry.name === 'node_modules') continue;
if (entry.isDirectory()) {
count += await countFiles(path.join(dir, entry.name));
} else if (entry.isFile()) {
count++;
}
}
return count;
};
const fileCount = await countFiles(repoDir);
console.log(`✓ Found ${fileCount.toLocaleString()} files`);
// Create cache manager and indexer
const cacheManager = new CacheManager(CACHE_DIR, true);
await cacheManager.initialize();
const symbolIndexer = new SymbolIndexer(cacheManager);
// Test 1: Cold start (no cache)
console.log(`\n[2/5] Cold start (building index from scratch)...`);
const coldStartBegin = Date.now();
await symbolIndexer.buildIndex(workspaceId, repoDir, true); // Force rebuild
const coldStartTime = Date.now() - coldStartBegin;
const index = symbolIndexer.getIndex(workspaceId);
const totalSymbols = index!.totalSymbols;
console.log(`✓ Cold start completed in ${coldStartTime.toLocaleString()}ms`);
console.log(`✓ Indexed ${totalSymbols.toLocaleString()} symbols`);
// Get cache stats
const stats = await cacheManager.getCacheStats(workspaceId, repoDir);
const cacheSize = stats!.cacheSize;
console.log(`✓ Cache size: ${(cacheSize / 1024 / 1024).toFixed(2)} MB`);
// Test 2: Verify cache is valid
console.log(`\n[3/5] Verifying cache validity...`);
const isValid = await cacheManager.isCacheValid(workspaceId, repoDir);
expect(isValid).toBe(true);
console.log(`✓ Cache is valid`);
// Test 3: Clear in-memory index
console.log(`\n[4/5] Clearing in-memory index...`);
await symbolIndexer.removeIndex(workspaceId);
expect(symbolIndexer.hasIndex(workspaceId)).toBe(false);
console.log(`✓ In-memory index cleared`);
// Test 4: Cached start (load from cache)
console.log(`\n[5/5] Cached start (loading from cache)...`);
const cachedStartBegin = Date.now();
await symbolIndexer.buildIndex(workspaceId, repoDir); // Should load from cache
const cachedStartTime = Date.now() - cachedStartBegin;
const cachedIndex = symbolIndexer.getIndex(workspaceId);
expect(cachedIndex!.totalSymbols).toBe(totalSymbols);
console.log(`✓ Cached start completed in ${cachedStartTime.toLocaleString()}ms`);
// Calculate improvement
const improvement = coldStartTime / cachedStartTime;
const improvementPercent = ((coldStartTime - cachedStartTime) / coldStartTime) * 100;
console.log(`\n${'─'.repeat(80)}`);
console.log(`PERFORMANCE RESULTS:`);
console.log(` Cold Start: ${coldStartTime.toLocaleString()}ms`);
console.log(` Cached Start: ${cachedStartTime.toLocaleString()}ms`);
console.log(` Improvement: ${improvement.toFixed(2)}x faster`);
console.log(` Time Saved: ${improvementPercent.toFixed(1)}%`);
console.log(`${'─'.repeat(80)}`);
// Store results
results.push({
repoName: repo.name,
coldStartTime,
cachedStartTime,
improvement,
improvementPercent,
totalSymbols,
fileCount,
cacheSize,
});
// Informational only - no assertions on timing (can vary greatly in CI)
// Just verify we got the same symbols back
if (improvementPercent >= 80) {
console.log(`✅ Excellent: ${improvementPercent.toFixed(1)}% time saved (target: 80%)`);
} else if (improvementPercent >= 50) {
console.log(`✓ Good: ${improvementPercent.toFixed(1)}% time saved`);
} else if (improvementPercent >= 20) {
console.log(`ℹ Moderate: ${improvementPercent.toFixed(1)}% time saved`);
} else {
console.log(`ℹ Performance gain: ${improvementPercent.toFixed(1)}% (may vary by environment)`);
}
// Note: We don't enforce hard performance limits in CI as they're environment-dependent
}, 300000); // 5 minute timeout for large repos
}
});
describe('Synthetic Large Workspace', () => {
it('should efficiently handle very large codebases', async () => {
if (!ctagsAvailable) {
console.log('Skipping - ctags not available');
return;
}
const syntheticDir = path.join(TEST_DIR, 'synthetic-large');
await fs.mkdir(syntheticDir, { recursive: true });
console.log(`\n${'='.repeat(80)}`);
console.log(`Testing Synthetic Large Codebase`);
console.log(`${'='.repeat(80)}`);
// Create a synthetic large codebase
console.log(`\n[1/4] Generating synthetic codebase...`);
const fileCount = 100;
const functionsPerFile = 50;
for (let i = 0; i < fileCount; i++) {
const functions = [];
for (let j = 0; j < functionsPerFile; j++) {
functions.push(`
function func_${i}_${j}(param1, param2, param3) {
const result = param1 + param2 + param3;
return result * ${j};
}
`);
}
await fs.writeFile(
path.join(syntheticDir, `file_${i}.js`),
functions.join('\n'),
'utf-8'
);
}
console.log(`✓ Generated ${fileCount} files with ~${fileCount * functionsPerFile} functions`);
const cacheManager = new CacheManager(CACHE_DIR, true);
await cacheManager.initialize();
const symbolIndexer = new SymbolIndexer(cacheManager);
// Cold start
console.log(`\n[2/4] Cold start...`);
const coldStartBegin = Date.now();
await symbolIndexer.buildIndex('synthetic', syntheticDir, true);
const coldStartTime = Date.now() - coldStartBegin;
const index = symbolIndexer.getIndex('synthetic');
console.log(`✓ Indexed ${index!.totalSymbols.toLocaleString()} symbols in ${coldStartTime.toLocaleString()}ms`);
// Clear and cached start
console.log(`\n[3/4] Clearing in-memory index...`);
await symbolIndexer.removeIndex('synthetic');
console.log(`\n[4/4] Cached start...`);
const cachedStartBegin = Date.now();
await symbolIndexer.buildIndex('synthetic', syntheticDir);
const cachedStartTime = Date.now() - cachedStartBegin;
console.log(`✓ Loaded from cache in ${cachedStartTime.toLocaleString()}ms`);
const improvement = coldStartTime / cachedStartTime;
const improvementPercent = ((coldStartTime - cachedStartTime) / coldStartTime) * 100;
console.log(`\n${'─'.repeat(80)}`);
console.log(`PERFORMANCE RESULTS:`);
console.log(` Cold Start: ${coldStartTime.toLocaleString()}ms`);
console.log(` Cached Start: ${cachedStartTime.toLocaleString()}ms`);
console.log(` Improvement: ${improvement.toFixed(2)}x faster`);
console.log(` Time Saved: ${improvementPercent.toFixed(1)}%`);
console.log(`${'─'.repeat(80)}`);
results.push({
repoName: 'synthetic-large',
coldStartTime,
cachedStartTime,
improvement,
improvementPercent,
totalSymbols: index!.totalSymbols,
fileCount,
cacheSize: (await cacheManager.getCacheStats('synthetic', syntheticDir))!.cacheSize,
});
// Informational only - no assertions on timing (can vary greatly in CI)
// Report performance results
if (improvementPercent >= 80) {
console.log(`✅ Excellent: ${improvementPercent.toFixed(1)}% time saved (target: 80%)`);
} else if (improvementPercent >= 50) {
console.log(`✓ Good: ${improvementPercent.toFixed(1)}% time saved`);
} else if (improvementPercent >= 20) {
console.log(`ℹ Moderate: ${improvementPercent.toFixed(1)}% time saved`);
} else {
console.log(`ℹ Performance gain: ${improvementPercent.toFixed(1)}% (may vary by environment)`);
}
}, 180000); // 3 minute timeout
});
});
