#!/usr/bin/env node
/**
* Performance Benchmark Suite for MCP Calendar Server
* Tests various performance aspects of the server
*/
import { CalendarService } from '../src/services/CalendarService';
import { ReminderService } from '../src/services/ReminderService';
import { DatabaseManager } from '../src/database';
import { createCacheManager } from '../src/cache';
import { CalendarTools } from '../src/tools/CalendarTools';
import { metricsCollector } from '../src/monitoring/metrics';
import { Logger } from '../src/utils/logger';
import chalk from 'chalk';
import { performance } from 'perf_hooks';
interface BenchmarkResult {
name: string;
totalOperations: number;
totalDuration: number;
averageLatency: number;
minLatency: number;
maxLatency: number;
p50Latency: number;
p95Latency: number;
p99Latency: number;
operationsPerSecond: number;
successRate: number;
errors: string[];
}
interface BenchmarkSuite {
name: string;
results: BenchmarkResult[];
totalDuration: number;
summary: {
totalOperations: number;
overallSuccessRate: number;
averageOps: number;
};
}
class PerformanceBenchmark {
private database: DatabaseManager;
private cache: ReturnType<typeof createCacheManager>;
private calendarService: CalendarService;
private reminderService: ReminderService;
private calendarTools: CalendarTools;
private testUserId: string = '';
constructor() {
this.database = new DatabaseManager();
this.cache = createCacheManager();
this.calendarService = new CalendarService(this.cache, this.database);
this.reminderService = new ReminderService(this.database);
this.calendarTools = new CalendarTools(this.calendarService, this.reminderService);
}
async initialize(): Promise<void> {
console.log(chalk.blue('🚀 Initializing Performance Benchmark Suite\n'));
// Initialize database
await this.database.initialize();
// Initialize cache
if ('connect' in this.cache) {
await (this.cache as any).connect();
}
// Create test user
const testUser = await this.database.createUser({
email: 'benchmark-test@example.com',
displayName: 'Benchmark User',
timezone: 'UTC',
preferences: {
defaultCalendarId: 'primary',
workingHours: {
monday: { startTime: '09:00', endTime: '17:00' },
tuesday: { startTime: '09:00', endTime: '17:00' },
wednesday: { startTime: '09:00', endTime: '17:00' },
thursday: { startTime: '09:00', endTime: '17:00' },
friday: { startTime: '09:00', endTime: '17:00' }
},
defaultEventDuration: 60,
bufferTime: 15,
autoDeclineConflicts: false,
reminderDefaults: {
defaultReminders: [15],
emailReminders: true,
pushReminders: false
}
}
});
this.testUserId = testUser.id;
console.log(chalk.green(`✅ Initialized with test user: ${testUser.email}`));
}
async cleanup(): Promise<void> {
console.log(chalk.yellow('\n🧹 Cleaning up benchmark resources...'));
try {
await this.database.close();
if ('disconnect' in this.cache) {
await (this.cache as any).disconnect();
}
console.log(chalk.green('✅ Cleanup completed'));
} catch (error) {
console.error(chalk.red('❌ Cleanup error:'), error);
}
}
// Run a single operation and measure performance
async measureOperation<T>(
name: string,
operation: () => Promise<T>
): Promise<{ result: T; duration: number; success: boolean; error?: string }> {
const startTime = performance.now();
try {
const result = await operation();
const duration = performance.now() - startTime;
return { result, duration, success: true };
} catch (error) {
const duration = performance.now() - startTime;
return {
result: null as T,
duration,
success: false,
error: error instanceof Error ? error.message : 'Unknown error'
};
}
}
// Run multiple iterations of an operation
async benchmarkOperation(
name: string,
operation: () => Promise<any>,
iterations: number = 100
): Promise<BenchmarkResult> {
console.log(chalk.blue(`📊 Benchmarking: ${name} (${iterations} iterations)`));
const latencies: number[] = [];
const errors: string[] = [];
let successCount = 0;
const startTime = performance.now();
for (let i = 0; i < iterations; i++) {
const result = await this.measureOperation(name, operation);
latencies.push(result.duration);
if (result.success) {
successCount++;
} else if (result.error) {
errors.push(result.error);
}
// Show progress for long-running benchmarks
if (iterations > 50 && (i + 1) % 25 === 0) {
process.stdout.write(chalk.gray(` Progress: ${i + 1}/${iterations}\r`));
}
}
const totalDuration = performance.now() - startTime;
// Calculate statistics
latencies.sort((a, b) => a - b);
const averageLatency = latencies.reduce((a, b) => a + b, 0) / latencies.length;
const minLatency = latencies[0];
const maxLatency = latencies[latencies.length - 1];
const p50Latency = latencies[Math.floor(latencies.length * 0.50)];
const p95Latency = latencies[Math.floor(latencies.length * 0.95)];
const p99Latency = latencies[Math.floor(latencies.length * 0.99)];
const operationsPerSecond = (iterations / totalDuration) * 1000;
const successRate = (successCount / iterations) * 100;
console.log(chalk.green(` ✅ Completed in ${totalDuration.toFixed(2)}ms`));
return {
name,
totalOperations: iterations,
totalDuration,
averageLatency,
minLatency,
maxLatency,
p50Latency,
p95Latency,
p99Latency,
operationsPerSecond,
successRate,
errors: [...new Set(errors)] // Remove duplicates
};
}
// Benchmark reminder operations
async benchmarkReminders(): Promise<BenchmarkResult[]> {
console.log(chalk.yellow('\n📝 Reminder Operations Benchmark'));
const results: BenchmarkResult[] = [];
// Benchmark reminder creation
results.push(await this.benchmarkOperation(
'Create Reminder',
async () => {
return await this.calendarTools.handleToolCall(
'create_reminder',
{
title: `Benchmark Reminder ${Date.now()}`,
description: 'Created during performance benchmark',
priority: 'medium'
},
this.testUserId
);
},
100
));
// Benchmark reminder retrieval
results.push(await this.benchmarkOperation(
'Get Reminders',
async () => {
return await this.calendarTools.handleToolCall(
'get_reminders',
{},
this.testUserId
);
},
200
));
// Create some reminders for update/complete benchmarks
const createdReminders = [];
for (let i = 0; i < 50; i++) {
const result = await this.calendarTools.handleToolCall(
'create_reminder',
{
title: `Update Test Reminder ${i}`,
description: 'For update benchmark',
priority: 'low'
},
this.testUserId
);
if (result.success && result.data) {
createdReminders.push(result.data.id);
}
}
// Benchmark reminder updates
let updateIndex = 0;
results.push(await this.benchmarkOperation(
'Update Reminder',
async () => {
const reminderId = createdReminders[updateIndex % createdReminders.length];
updateIndex++;
return await this.calendarTools.handleToolCall(
'update_reminder',
{
reminder_id: reminderId,
title: `Updated Reminder ${Date.now()}`,
priority: 'high'
},
this.testUserId
);
},
50
));
// Benchmark reminder completion
let completeIndex = 0;
results.push(await this.benchmarkOperation(
'Complete Reminder',
async () => {
const reminderId = createdReminders[completeIndex % createdReminders.length];
completeIndex++;
return await this.calendarTools.handleToolCall(
'complete_reminder',
{ reminder_id: reminderId },
this.testUserId
);
},
50
));
return results;
}
// Benchmark calendar operations
async benchmarkCalendar(): Promise<BenchmarkResult[]> {
console.log(chalk.yellow('\n📅 Calendar Operations Benchmark'));
const results: BenchmarkResult[] = [];
// Benchmark working hours retrieval
results.push(await this.benchmarkOperation(
'Get Working Hours',
async () => {
return await this.calendarTools.handleToolCall(
'get_working_hours',
{ calendar_id: 'primary' },
this.testUserId
);
},
200
));
// Benchmark contact suggestions (with caching)
results.push(await this.benchmarkOperation(
'Get Contact Suggestions',
async () => {
return await this.calendarTools.handleToolCall(
'get_contact_suggestions',
{ partial_name: 'test', limit: 5 },
this.testUserId
);
},
100
));
return results;
}
// Benchmark database operations
async benchmarkDatabase(): Promise<BenchmarkResult[]> {
console.log(chalk.yellow('\n💾 Database Operations Benchmark'));
const results: BenchmarkResult[] = [];
// Benchmark user retrieval
results.push(await this.benchmarkOperation(
'Database User Lookup',
async () => {
return await this.database.getUser(this.testUserId);
},
500
));
// Benchmark user creation
results.push(await this.benchmarkOperation(
'Database User Creation',
async () => {
const randomEmail = `bench-${Date.now()}-${Math.random()}@example.com`;
return await this.database.createUser({
email: randomEmail,
displayName: 'Benchmark User',
timezone: 'UTC'
});
},
50
));
// Benchmark reminder queries
results.push(await this.benchmarkOperation(
'Database Reminder Query',
async () => {
return await this.database.getReminders(this.testUserId, {});
},
200
));
return results;
}
// Benchmark caching operations
async benchmarkCache(): Promise<BenchmarkResult[]> {
console.log(chalk.yellow('\n🗄️ Cache Operations Benchmark'));
const results: BenchmarkResult[] = [];
// Benchmark cache write operations
results.push(await this.benchmarkOperation(
'Cache Write',
async () => {
const key = `benchmark-${Date.now()}-${Math.random()}`;
const data = { test: true, timestamp: Date.now() };
await this.cache.setUserPreferences(this.testUserId, data as any);
return true;
},
200
));
// Benchmark cache read operations
results.push(await this.benchmarkOperation(
'Cache Read',
async () => {
return await this.cache.getUserPreferences(this.testUserId);
},
500
));
return results;
}
// Benchmark concurrent operations
async benchmarkConcurrency(): Promise<BenchmarkResult[]> {
console.log(chalk.yellow('\n🔄 Concurrency Benchmark'));
const results: BenchmarkResult[] = [];
// Concurrent reminder creation
results.push(await this.benchmarkOperation(
'Concurrent Reminder Creation',
async () => {
const promises = Array.from({ length: 10 }, (_, i) =>
this.calendarTools.handleToolCall(
'create_reminder',
{
title: `Concurrent Reminder ${i}-${Date.now()}`,
priority: 'low'
},
this.testUserId
)
);
return await Promise.all(promises);
},
20
));
// Concurrent database queries
results.push(await this.benchmarkOperation(
'Concurrent Database Queries',
async () => {
const promises = Array.from({ length: 5 }, () =>
this.database.getUser(this.testUserId)
);
return await Promise.all(promises);
},
50
));
return results;
}
// Run all benchmarks
async runAllBenchmarks(): Promise<BenchmarkSuite[]> {
const suites: BenchmarkSuite[] = [];
// Run reminder benchmarks
const reminderStart = performance.now();
const reminderResults = await this.benchmarkReminders();
const reminderDuration = performance.now() - reminderStart;
suites.push(this.createSuite('Reminder Operations', reminderResults, reminderDuration));
// Run calendar benchmarks
const calendarStart = performance.now();
const calendarResults = await this.benchmarkCalendar();
const calendarDuration = performance.now() - calendarStart;
suites.push(this.createSuite('Calendar Operations', calendarResults, calendarDuration));
// Run database benchmarks
const databaseStart = performance.now();
const databaseResults = await this.benchmarkDatabase();
const databaseDuration = performance.now() - databaseStart;
suites.push(this.createSuite('Database Operations', databaseResults, databaseDuration));
// Run cache benchmarks
const cacheStart = performance.now();
const cacheResults = await this.benchmarkCache();
const cacheDuration = performance.now() - cacheStart;
suites.push(this.createSuite('Cache Operations', cacheResults, cacheDuration));
// Run concurrency benchmarks
const concurrencyStart = performance.now();
const concurrencyResults = await this.benchmarkConcurrency();
const concurrencyDuration = performance.now() - concurrencyStart;
suites.push(this.createSuite('Concurrency Tests', concurrencyResults, concurrencyDuration));
return suites;
}
private createSuite(name: string, results: BenchmarkResult[], duration: number): BenchmarkSuite {
const totalOperations = results.reduce((sum, r) => sum + r.totalOperations, 0);
const totalSuccessful = results.reduce((sum, r) => sum + (r.totalOperations * r.successRate / 100), 0);
const overallSuccessRate = (totalSuccessful / totalOperations) * 100;
const averageOps = results.reduce((sum, r) => sum + r.operationsPerSecond, 0) / results.length;
return {
name,
results,
totalDuration: duration,
summary: {
totalOperations,
overallSuccessRate,
averageOps
}
};
}
// Generate detailed report
generateReport(suites: BenchmarkSuite[]): void {
console.log(chalk.blue('\n' + '='.repeat(80)));
console.log(chalk.blue.bold('📊 PERFORMANCE BENCHMARK REPORT'));
console.log(chalk.blue('='.repeat(80)));
// Overall summary
const totalOperations = suites.reduce((sum, s) => sum + s.summary.totalOperations, 0);
const totalDuration = suites.reduce((sum, s) => sum + s.totalDuration, 0);
const overallOps = (totalOperations / totalDuration) * 1000;
console.log(chalk.green('\n📈 Overall Summary:'));
console.log(` Total Operations: ${chalk.bold(totalOperations.toLocaleString())}`);
console.log(` Total Duration: ${chalk.bold((totalDuration / 1000).toFixed(2))}s`);
console.log(` Overall Throughput: ${chalk.bold(overallOps.toFixed(2))} ops/sec`);
// Suite-by-suite results
suites.forEach(suite => {
console.log(chalk.yellow(`\n🔍 ${suite.name}`));
console.log(chalk.gray('─'.repeat(50)));
console.log(` Suite Duration: ${(suite.totalDuration / 1000).toFixed(2)}s`);
console.log(` Total Operations: ${suite.summary.totalOperations.toLocaleString()}`);
console.log(` Success Rate: ${suite.summary.overallSuccessRate.toFixed(1)}%`);
console.log(` Average Throughput: ${suite.summary.averageOps.toFixed(2)} ops/sec\n`);
suite.results.forEach(result => {
const statusColor = result.successRate >= 95 ? chalk.green :
result.successRate >= 90 ? chalk.yellow : chalk.red;
console.log(` ${chalk.bold(result.name)}:`);
console.log(` Operations: ${result.totalOperations.toLocaleString()}`);
console.log(` Success Rate: ${statusColor(result.successRate.toFixed(1) + '%')}`);
console.log(` Throughput: ${chalk.bold(result.operationsPerSecond.toFixed(2))} ops/sec`);
console.log(` Latency (avg): ${result.averageLatency.toFixed(2)}ms`);
console.log(` Latency (p95): ${result.p95Latency.toFixed(2)}ms`);
console.log(` Latency (p99): ${result.p99Latency.toFixed(2)}ms`);
if (result.errors.length > 0) {
console.log(` ${chalk.red('Errors:')} ${result.errors.slice(0, 3).join(', ')}`);
if (result.errors.length > 3) {
console.log(` ... and ${result.errors.length - 3} more`);
}
}
console.log('');
});
});
// Performance recommendations
console.log(chalk.blue('\n💡 Performance Recommendations:'));
this.generateRecommendations(suites);
console.log(chalk.blue('\n' + '='.repeat(80)));
console.log(chalk.green.bold('✅ Benchmark Complete!'));
console.log(chalk.blue('='.repeat(80)));
}
private generateRecommendations(suites: BenchmarkSuite[]): void {
const allResults = suites.flatMap(s => s.results);
// Find slow operations
const slowOperations = allResults.filter(r => r.averageLatency > 100);
if (slowOperations.length > 0) {
console.log(chalk.yellow(' ⚠️ Slow Operations (>100ms average):'));
slowOperations.forEach(op => {
console.log(` - ${op.name}: ${op.averageLatency.toFixed(2)}ms`);
});
}
// Find operations with low success rates
const unreliableOperations = allResults.filter(r => r.successRate < 95);
if (unreliableOperations.length > 0) {
console.log(chalk.red(' 🚨 Operations with Low Success Rate (<95%):'));
unreliableOperations.forEach(op => {
console.log(` - ${op.name}: ${op.successRate.toFixed(1)}%`);
});
}
// Find operations with high p99 latency
const inconsistentOperations = allResults.filter(r => r.p99Latency > r.averageLatency * 3);
if (inconsistentOperations.length > 0) {
console.log(chalk.yellow(' 📊 Operations with High Latency Variance:'));
inconsistentOperations.forEach(op => {
console.log(` - ${op.name}: p99=${op.p99Latency.toFixed(2)}ms (avg=${op.averageLatency.toFixed(2)}ms)`);
});
}
// General recommendations
console.log(chalk.green(' ✅ General Recommendations:'));
console.log(' - Monitor operations with >50ms average latency');
console.log(' - Consider connection pooling for database operations');
console.log(' - Implement caching for frequently accessed data');
console.log(' - Add circuit breakers for external service calls');
console.log(' - Monitor memory usage during high concurrency');
}
}
// CLI interface
async function main() {
const args = process.argv.slice(2);
const benchmark = new PerformanceBenchmark();
try {
await benchmark.initialize();
let suites: BenchmarkSuite[];
if (args.includes('--quick')) {
console.log(chalk.blue('🏃 Running Quick Benchmark (reduced iterations)\n'));
// Run with reduced iterations for quick testing
suites = await benchmark.runAllBenchmarks();
} else if (args.includes('--reminder-only')) {
console.log(chalk.blue('📝 Running Reminder-Only Benchmark\n'));
const results = await benchmark.benchmarkReminders();
suites = [benchmark['createSuite']('Reminder Operations', results, 0)];
} else {
console.log(chalk.blue('🔥 Running Full Performance Benchmark Suite\n'));
suites = await benchmark.runAllBenchmarks();
}
benchmark.generateReport(suites);
// Export results to JSON if requested
if (args.includes('--json')) {
const fs = require('fs');
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `benchmark-results-${timestamp}.json`;
fs.writeFileSync(filename, JSON.stringify(suites, null, 2));
console.log(chalk.green(`\n💾 Results exported to: ${filename}`));
}
} catch (error) {
console.error(chalk.red('❌ Benchmark failed:'), error);
process.exit(1);
} finally {
await benchmark.cleanup();
}
}
if (require.main === module) {
main().catch(console.error);
}
export { PerformanceBenchmark };
