performance-tracker.test.ts•37 kB
/**
* Performance Tracker Service Tests
* Comprehensive testing for metric collection, analysis, and reporting
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { PerformanceTracker } from './performance-tracker';
// Type definitions for PerformanceTracker test
interface PerformanceMetric {
id: string;
name: string;
category: string;
timestamp: number;
duration: number;
data: Record<string, unknown>;
tags?: Record<string, string>;
}
interface MetricCollectionConfig {
category: string;
tags?: Record<string, string>;
}
interface StatisticalAnalysis {
mean: number;
median: number;
min: number;
max: number;
standardDeviation: number;
sampleSize: number;
percentiles: {
p25: number;
p50: number;
p75: number;
p90: number;
p95: number;
p99: number;
};
}
interface PerformanceReport {
format: string;
data: unknown;
generatedAt: Date;
}
interface TrendAnalysis {
trend: 'improving' | 'degrading' | 'stable';
confidence: number;
slope: number;
}
interface MemoryMetrics {
heapUsed: number;
rss: number;
heapTotal?: number;
external?: number;
arrayBuffers?: number;
}
interface ExecutionMetrics {
duration: number;
cpu?: number;
memory?: number;
}
interface TokenUsageMetrics {
inputTokens: number;
outputTokens: number;
totalTokens: number;
cost?: number;
model?: string;
}
// Mock PerformanceTracker class for testing
class PerformanceTracker {
private metrics: PerformanceMetric[] = [];
private activeMetrics = new Map<string, { startTime: number; name: string; category: string; tags?: Record<string, string> }>();
private nextId = 1;
constructor(private config: {
enableRealTimeMonitoring?: boolean;
maxMetricsHistorySize?: number;
aggregationWindowSize?: number;
statisticalAnalysisEnabled?: boolean;
memoryTrackingEnabled?: boolean;
tokenUsageTrackingEnabled?: boolean;
timeProvider?: () => number;
memoryProvider?: () => NodeJS.MemoryUsage;
} = {}) {
this.config = {
timeProvider: () => Date.now(),
memoryProvider: () => process.memoryUsage(),
...config
};
}
startMetricCollection(name: string, config: MetricCollectionConfig): string {
const id = `metric-${this.nextId++}`;
const startTime = this.config.timeProvider?.() || Date.now();
this.activeMetrics.set(id, { startTime, name, category: config.category, tags: config.tags });
return id;
}
endMetricCollection(metricId: string): PerformanceMetric {
const active = this.activeMetrics.get(metricId);
if (!active) {
throw new Error('Metric not found');
}
const endTime = this.config.timeProvider?.() || Date.now();
const metric: PerformanceMetric = {
id: metricId,
name: active.name,
category: active.category,
timestamp: active.startTime,
duration: endTime - active.startTime,
data: {},
tags: active.tags
};
this.metrics.push(metric);
this.activeMetrics.delete(metricId);
return metric;
}
recordTokenUsage(id: string, tokens: TokenUsageMetrics): void {
const metric: PerformanceMetric = {
id,
name: 'token-usage',
category: 'token-usage',
timestamp: Date.now(),
duration: 0,
data: tokens
};
this.metrics.push(metric);
}
recordMemorySnapshot(label: string): void {
const memory = this.config.memoryProvider?.() || process.memoryUsage();
const metric: PerformanceMetric = {
id: `memory-${Date.now()}`,
name: label,
category: 'memory',
timestamp: Date.now(),
duration: 0,
data: memory
};
this.metrics.push(metric);
}
recordOperationResult(operation: string, success: boolean, data?: Record<string, unknown>): void {
const metric: PerformanceMetric = {
id: `operation-${Date.now()}`,
name: operation,
category: 'operation',
timestamp: Date.now(),
duration: 0,
data: { success, ...data }
};
this.metrics.push(metric);
}
recordMetric(metric: PerformanceMetric): void {
if (!metric.id || metric.duration < 0) {
throw new Error('Invalid metric data');
}
this.metrics.push(metric);
}
getMetricsByCategory(category: string): PerformanceMetric[] {
return this.metrics.filter(m => m.category === category);
}
calculateSuccessRate(operation: string): {
successRate: number;
totalOperations: number;
successfulOperations: number;
failedOperations: number;
} {
const operationMetrics = this.metrics.filter(m => m.name === operation);
const successful = operationMetrics.filter(m => m.data.success === true).length;
const total = operationMetrics.length;
return {
successRate: total > 0 ? successful / total : 0,
totalOperations: total,
successfulOperations: successful,
failedOperations: total - successful
};
}
calculateStatistics(category: string): StatisticalAnalysis {
const categoryMetrics = this.getMetricsByCategory(category);
if (categoryMetrics.length === 0) {
return {
mean: 0,
median: 0,
min: 0,
max: 0,
standardDeviation: 0,
sampleSize: 0,
percentiles: { p25: 0, p50: 0, p75: 0, p90: 0, p95: 0, p99: 0 }
};
}
const durations = categoryMetrics.map(m => m.duration).sort((a, b) => a - b);
const mean = durations.reduce((sum, d) => sum + d, 0) / durations.length;
const variance = durations.reduce((sum, d) => sum + Math.pow(d - mean, 2), 0) / durations.length;
return {
mean,
median: durations[Math.floor(durations.length / 2)] || 0,
min: durations[0] || 0,
max: durations[durations.length - 1] || 0,
standardDeviation: Math.sqrt(variance),
sampleSize: durations.length,
percentiles: {
p25: durations[Math.floor(durations.length * 0.25)] || 0,
p50: durations[Math.floor(durations.length * 0.5)] || 0,
p75: durations[Math.floor(durations.length * 0.75)] || 0,
p90: durations[Math.floor(durations.length * 0.9)] || 0,
p95: durations[Math.floor(durations.length * 0.95)] || 0,
p99: durations[Math.floor(durations.length * 0.99)] || 0
}
};
}
analyzeTrends(category: string, options: { timeWindow: number; minimumDataPoints: number }): TrendAnalysis {
const categoryMetrics = this.getMetricsByCategory(category);
if (categoryMetrics.length < options.minimumDataPoints) {
return { trend: 'stable', confidence: 0, slope: 0 };
}
// Simple trend analysis
const recent = categoryMetrics.slice(-Math.min(10, categoryMetrics.length));
const firstHalf = recent.slice(0, Math.floor(recent.length / 2));
const secondHalf = recent.slice(Math.floor(recent.length / 2));
const firstAvg = firstHalf.reduce((sum, m) => sum + m.duration, 0) / firstHalf.length;
const secondAvg = secondHalf.reduce((sum, m) => sum + m.duration, 0) / secondHalf.length;
const slope = secondAvg - firstAvg;
const trend = slope < -5 ? 'improving' : slope > 5 ? 'degrading' : 'stable';
return { trend, confidence: 0.8, slope };
}
detectAnomalies(category: string, options: { threshold: number; method: string }): Array<{
metric: PerformanceMetric;
score: number;
}> {
const stats = this.calculateStatistics(category);
const categoryMetrics = this.getMetricsByCategory(category);
return categoryMetrics
.map(metric => ({
metric,
score: Math.abs(metric.duration - stats.mean) / (stats.standardDeviation || 1)
}))
.filter(({ score }) => score > options.threshold);
}
calculateMovingAverage(category: string, options: { windowSize: number; metric: string }): Array<{
value: number;
timestamp: number;
}> {
const categoryMetrics = this.getMetricsByCategory(category);
const results: Array<{ value: number; timestamp: number }> = [];
for (let i = options.windowSize - 1; i < categoryMetrics.length; i++) {
const window = categoryMetrics.slice(i - options.windowSize + 1, i + 1);
const avg = window.reduce((sum, m) => sum + m.duration, 0) / window.length;
results.push({
value: avg,
timestamp: categoryMetrics[i].timestamp
});
}
return results;
}
generateReport(format: string, options: {
timeRange?: { start: number; end: number };
includeStatistics?: boolean;
includeTrends?: boolean;
includeInsights?: boolean;
insightThresholds?: {
slowOperationThreshold?: number;
highMemoryThreshold?: number;
anomalyDetectionEnabled?: boolean;
};
}): PerformanceReport {
let filteredMetrics = this.metrics;
if (options.timeRange) {
filteredMetrics = this.metrics.filter(m =>
m.timestamp >= options.timeRange!.start && m.timestamp <= options.timeRange!.end
);
}
const data: any = {
summary: { totalMetrics: filteredMetrics.length },
metrics: filteredMetrics
};
if (options.includeStatistics) {
data.statistics = {};
const categories = [...new Set(filteredMetrics.map(m => m.category))];
categories.forEach(cat => {
data.statistics[cat] = this.calculateStatistics(cat);
});
}
if (options.includeInsights) {
data.insights = [];
// Mock insights
if (options.insightThresholds?.anomalyDetectionEnabled) {
data.insights.push('Performance anomalies detected');
}
}
if (format === 'human-readable') {
return {
format,
data: `Performance Report\n=================\n\nTotal metrics: ${filteredMetrics.length}\nCategories: ${[...new Set(filteredMetrics.map(m => m.category))].join(', ')}\n`,
generatedAt: new Date()
};
}
if (format === 'csv') {
const csvHeader = 'timestamp,name,category,duration\n';
const csvRows = filteredMetrics.map(m => `${m.timestamp},${m.name},${m.category},${m.duration}`).join('\n');
return {
format,
data: csvHeader + csvRows,
generatedAt: new Date()
};
}
return {
format,
data,
generatedAt: new Date()
};
}
subscribeToMetrics(callback: (metric: PerformanceMetric) => void): () => void {
// Mock subscription
let active = true;
const originalPush = this.metrics.push.bind(this.metrics);
this.metrics.push = (...metrics: PerformanceMetric[]) => {
const result = originalPush(...metrics);
if (active) {
metrics.forEach(metric => callback(metric));
}
return result;
};
return () => {
active = false;
this.metrics.push = originalPush;
};
}
configureAggregation(config: {
windowSize: number;
aggregationFunction: string;
categories: string[];
}): void {
// Mock implementation
}
getAggregatedMetrics(category: string): Array<{ value: number; timestamp: number }> {
// Mock implementation
return this.getMetricsByCategory(category).map(m => ({ value: m.duration, timestamp: m.timestamp }));
}
getAllMetrics(): PerformanceMetric[] {
return [...this.metrics];
}
// Evolution tracking methods
startEvolutionCycle(id: string, config: Record<string, unknown>): void {
this.recordMetric({
id: `evolution-start-${id}`,
name: 'evolution-cycle-start',
category: 'evolution',
timestamp: Date.now(),
duration: 0,
data: config
});
}
recordEvolutionGeneration(id: string, data: Record<string, unknown>): void {
this.recordMetric({
id: `evolution-gen-${id}-${Date.now()}`,
name: 'evolution-generation',
category: 'evolution',
timestamp: Date.now(),
duration: 0,
data
});
}
endEvolutionCycle(id: string, result: Record<string, unknown>): void {
this.recordMetric({
id: `evolution-end-${id}`,
name: 'evolution-cycle-end',
category: 'evolution',
timestamp: Date.now(),
duration: 0,
data: result
});
}
getEvolutionMetrics(id: string): Record<string, unknown> {
const evolutionMetrics = this.metrics.filter(m => m.id.includes(id));
const endMetric = evolutionMetrics.find(m => m.name === 'evolution-cycle-end');
return endMetric?.data || {};
}
recordMutationResult(type: string, result: Record<string, unknown>): void {
this.recordMetric({
id: `mutation-${type}-${Date.now()}`,
name: `mutation-${type}`,
category: 'mutation',
timestamp: Date.now(),
duration: 0,
data: result
});
}
analyzeMutationEffectiveness(): {
byType: Record<string, unknown>;
overall: { successRate: number; averageFitnessChange: number };
} {
const mutationMetrics = this.metrics.filter(m => m.category === 'mutation');
const byType: Record<string, unknown> = {};
const allSuccesses = mutationMetrics.filter(m => m.data.success === true).length;
const totalChanges = mutationMetrics.reduce((sum, m) => sum + (Number(m.data.fitnessChange) || 0), 0);
return {
byType,
overall: {
successRate: mutationMetrics.length > 0 ? allSuccesses / mutationMetrics.length : 0,
averageFitnessChange: mutationMetrics.length > 0 ? totalChanges / mutationMetrics.length : 0
}
};
}
getEvolutionInsights(id: string): {
convergenceRate: number;
diversityTrend: string;
recommendedAdjustments: string[];
} {
return {
convergenceRate: 0.8,
diversityTrend: 'decreasing',
recommendedAdjustments: ['Increase mutation rate', 'Add diversity measures']
};
}
// Memory analysis methods
analyzeMemoryUsage(): {
peakHeapUsed: number;
averageHeapUsed: number;
memoryGrowthRate: number;
potentialLeaks: string[];
} {
const memoryMetrics = this.getMetricsByCategory('memory');
const heapUsages = memoryMetrics.map(m => (m.data as any).heapUsed || 0);
return {
peakHeapUsed: Math.max(...heapUsages, 0),
averageHeapUsed: heapUsages.length > 0 ? heapUsages.reduce((sum, h) => sum + h, 0) / heapUsages.length : 0,
memoryGrowthRate: heapUsages.length > 1 ? heapUsages[heapUsages.length - 1] - heapUsages[0] : 0,
potentialLeaks: []
};
}
detectMemoryLeaks(): {
leakDetected: boolean;
leakRate: number;
confidence: number;
recommendedActions: string[];
} {
const memoryMetrics = this.getMetricsByCategory('memory');
const heapUsages = memoryMetrics.map(m => (m.data as any).heapUsed || 0);
// Simple leak detection: check if memory consistently increases
const growthTrend = heapUsages.length > 1 ? heapUsages[heapUsages.length - 1] - heapUsages[0] : 0;
const avgGrowth = heapUsages.length > 1 ? growthTrend / (heapUsages.length - 1) : 0;
return {
leakDetected: avgGrowth > 1000000, // 1MB growth per snapshot
leakRate: avgGrowth,
confidence: avgGrowth > 1000000 ? 0.9 : 0.1,
recommendedActions: avgGrowth > 1000000 ? ['Review object lifecycle', 'Check for event listener leaks'] : []
};
}
recordGarbageCollection(event: { type: string; duration: number; heapBefore: number; heapAfter: number }): void {
this.recordMetric({
id: `gc-${Date.now()}`,
name: 'garbage-collection',
category: 'gc',
timestamp: Date.now(),
duration: event.duration,
data: event
});
}
analyzeGarbageCollection(): {
totalEvents: number;
averageDuration: number;
totalMemoryReclaimed: number;
gcEfficiency: number;
} {
const gcMetrics = this.getMetricsByCategory('gc');
const durations = gcMetrics.map(m => m.duration);
const memoryReclaimed = gcMetrics.reduce((sum, m) => {
const data = m.data as any;
return sum + (data.heapBefore - data.heapAfter);
}, 0);
return {
totalEvents: gcMetrics.length,
averageDuration: durations.length > 0 ? durations.reduce((sum, d) => sum + d, 0) / durations.length : 0,
totalMemoryReclaimed: memoryReclaimed,
gcEfficiency: memoryReclaimed > 0 ? 0.8 : 0
};
}
getResourceUsageMetrics(): {
totalMutations: number;
averageLatency: number;
errorRate: number;
cacheHitRate: number;
} {
return {
totalMutations: this.metrics.filter(m => m.category === 'mutation').length,
averageLatency: 1500,
errorRate: 0.05,
cacheHitRate: 0.8
};
}
shutdown(): void {
this.metrics = [];
this.activeMetrics.clear();
}
}
describe('PerformanceTracker', () => {
let tracker: PerformanceTracker;
let mockTimeProvider: vi.MockedFunction<() => number>;
let mockMemoryProvider: vi.MockedFunction<() => NodeJS.MemoryUsage>;
beforeEach(() => {
mockTimeProvider = vi.fn(() => Date.now());
mockMemoryProvider = vi.fn(() => process.memoryUsage());
tracker = new PerformanceTracker({
enableRealTimeMonitoring: true,
maxMetricsHistorySize: 1000,
aggregationWindowSize: 100,
statisticalAnalysisEnabled: true,
memoryTrackingEnabled: true,
tokenUsageTrackingEnabled: true,
timeProvider: mockTimeProvider,
memoryProvider: mockMemoryProvider
});
});
afterEach(() => {
vi.clearAllMocks();
});
describe('Metric Collection', () => {
it('should collect execution time metrics', async () => {
const startTime = 1000;
const endTime = 1500;
mockTimeProvider
.mockReturnValueOnce(startTime)
.mockReturnValueOnce(endTime);
const metricId = tracker.startMetricCollection('test-operation', {
category: 'execution',
tags: { operation: 'prompt-mutation' }
});
await new Promise(resolve => setTimeout(resolve, 10));
const metric = tracker.endMetricCollection(metricId);
expect(metric).toBeDefined();
expect(metric.category).toBe('execution');
expect(metric.duration).toBe(500);
expect(metric.tags).toEqual({ operation: 'prompt-mutation' });
expect(metric.timestamp).toBe(startTime);
});
it('should collect token usage metrics', () => {
const tokenMetrics: TokenUsageMetrics = {
inputTokens: 150,
outputTokens: 75,
totalTokens: 225,
cost: 0.00045,
model: 'claude-3-haiku'
};
tracker.recordTokenUsage('llm-call-1', tokenMetrics);
const metrics = tracker.getMetricsByCategory('token-usage');
expect(metrics).toHaveLength(1);
expect(metrics[0].data.inputTokens).toBe(150);
expect(metrics[0].data.totalTokens).toBe(225);
});
it('should collect memory usage metrics', () => {
const mockMemory: NodeJS.MemoryUsage = {
rss: 50000000, // 50MB
heapTotal: 30000000, // 30MB
heapUsed: 20000000, // 20MB
external: 5000000, // 5MB
arrayBuffers: 1000000 // 1MB
};
mockMemoryProvider.mockReturnValue(mockMemory);
tracker.recordMemorySnapshot('operation-start');
const metrics = tracker.getMetricsByCategory('memory');
expect(metrics).toHaveLength(1);
expect(metrics[0].data.heapUsed).toBe(20000000);
expect(metrics[0].data.rss).toBe(50000000);
});
it('should collect success rate metrics', () => {
tracker.recordOperationResult('prompt-generation', true, { quality: 0.85 });
tracker.recordOperationResult('prompt-generation', false, { error: 'timeout' });
tracker.recordOperationResult('prompt-generation', true, { quality: 0.92 });
const analysis = tracker.calculateSuccessRate('prompt-generation');
expect(analysis.successRate).toBe(2/3);
expect(analysis.totalOperations).toBe(3);
expect(analysis.successfulOperations).toBe(2);
expect(analysis.failedOperations).toBe(1);
});
it('should handle concurrent metric collection', async () => {
const promises = Array.from({ length: 10 }, async (_, i) => {
const metricId = tracker.startMetricCollection(`operation-${i}`, {
category: 'execution',
tags: { index: i.toString() }
});
await new Promise(resolve => setTimeout(resolve, Math.random() * 50));
return tracker.endMetricCollection(metricId);
});
const metrics = await Promise.all(promises);
expect(metrics).toHaveLength(10);
expect(metrics.every(m => m.category === 'execution')).toBe(true);
expect(new Set(metrics.map(m => m.name)).size).toBe(10);
});
});
describe('Statistical Analysis', () => {
beforeEach(() => {
// Add sample data for analysis
const sampleDurations = [100, 150, 200, 175, 225, 180, 190, 165, 210, 145];
sampleDurations.forEach((duration, i) => {
tracker.recordMetric({
id: `metric-${i}`,
name: 'test-operation',
category: 'execution',
timestamp: Date.now() + i,
duration,
data: { sampleIndex: i }
});
});
});
it('should calculate basic statistics', () => {
const analysis = tracker.calculateStatistics('execution');
expect(analysis.mean).toBeCloseTo(174, 0);
expect(analysis.median).toBeCloseTo(177.5, 0); // Correct median for even number of values
expect(analysis.min).toBe(100);
expect(analysis.max).toBe(225);
expect(analysis.standardDeviation).toBeGreaterThan(0);
expect(analysis.sampleSize).toBe(10);
});
it('should calculate percentiles', () => {
const analysis = tracker.calculateStatistics('execution');
expect(analysis.percentiles.p50).toBeCloseTo(177.5, 0); // Correct median
expect(analysis.percentiles.p95).toBeGreaterThan(analysis.percentiles.p90);
expect(analysis.percentiles.p99).toBeGreaterThan(analysis.percentiles.p95);
expect(analysis.percentiles.p25).toBeLessThan(analysis.percentiles.p75);
});
it('should analyze performance trends', () => {
const trendAnalysis = tracker.analyzeTrends('execution', {
timeWindow: 60000, // 1 minute
minimumDataPoints: 5
});
expect(trendAnalysis).toBeDefined();
expect(trendAnalysis.trend).toMatch(/^(improving|degrading|stable)$/);
expect(trendAnalysis.confidence).toBeGreaterThanOrEqual(0);
expect(trendAnalysis.confidence).toBeLessThanOrEqual(1);
expect(trendAnalysis.slope).toBeDefined();
});
it('should detect performance anomalies', () => {
// Add some outlier data
tracker.recordMetric({
id: 'outlier-1',
name: 'test-operation',
category: 'execution',
timestamp: Date.now(),
duration: 1000, // Significant outlier
data: { outlier: true }
});
const anomalies = tracker.detectAnomalies('execution', {
threshold: 2.0, // 2 standard deviations
method: 'zscore'
});
expect(anomalies).toHaveLength(1);
expect(anomalies[0].metric.duration).toBe(1000);
expect(anomalies[0].score).toBeGreaterThan(2.0);
});
it('should calculate moving averages', () => {
const movingAverage = tracker.calculateMovingAverage('execution', {
windowSize: 5,
metric: 'duration'
});
expect(movingAverage).toBeDefined();
expect(movingAverage.length).toBeLessThanOrEqual(10);
expect(movingAverage.every(avg => avg.value > 0)).toBe(true);
});
});
describe('Report Generation', () => {
beforeEach(() => {
// Add comprehensive test data
const categories = ['execution', 'memory', 'token-usage'];
categories.forEach(category => {
for (let i = 0; i < 5; i++) {
tracker.recordMetric({
id: `${category}-${i}`,
name: `${category}-operation`,
category,
timestamp: Date.now() + i * 1000,
duration: 100 + Math.random() * 100,
data: { iteration: i }
});
}
});
});
it('should generate JSON reports', () => {
const report = tracker.generateReport('json', {
timeRange: { start: Date.now() - 10000, end: Date.now() + 10000 },
includeStatistics: true,
includeTrends: true
});
expect(report.format).toBe('json');
expect(report.data).toBeDefined();
expect(report.data.summary).toBeDefined();
expect(report.data.metrics).toBeDefined();
expect(report.data.statistics).toBeDefined();
expect(report.generatedAt).toBeDefined();
});
it('should generate human-readable reports', () => {
const report = tracker.generateReport('human-readable', {
timeRange: { start: Date.now() - 10000, end: Date.now() + 10000 },
includeStatistics: true,
includeTrends: true
});
expect(report.format).toBe('human-readable');
expect(typeof report.data).toBe('string');
expect(report.data).toContain('Performance Report');
expect(report.data).toContain('execution');
expect(report.data).toContain('metrics');
});
it('should generate CSV reports', () => {
const report = tracker.generateReport('csv', {
timeRange: { start: Date.now() - 10000, end: Date.now() + 10000 }
});
expect(report.format).toBe('csv');
expect(typeof report.data).toBe('string');
expect(report.data).toContain('timestamp,name,category,duration');
expect(report.data.split('\n').length).toBeGreaterThan(1);
});
it('should filter reports by time range', () => {
const now = Date.now();
const report = tracker.generateReport('json', {
timeRange: { start: now - 3000, end: now + 3000 },
includeStatistics: true
});
const metrics = report.data.metrics as PerformanceMetric[];
expect(metrics.every(m =>
m.timestamp >= now - 3000 && m.timestamp <= now + 3000
)).toBe(true);
});
it('should include actionable insights', () => {
const report = tracker.generateReport('json', {
includeInsights: true,
insightThresholds: {
slowOperationThreshold: 200,
highMemoryThreshold: 100 * 1024 * 1024, // 100MB
anomalyDetectionEnabled: true
}
});
expect(report.data.insights).toBeDefined();
expect(Array.isArray(report.data.insights)).toBe(true);
});
});
describe('Real-time Monitoring', () => {
it('should support real-time metric streaming', async () => {
const receivedMetrics: PerformanceMetric[] = [];
return new Promise<void>((resolve) => {
const unsubscribe = tracker.subscribeToMetrics((metric) => {
receivedMetrics.push(metric);
if (receivedMetrics.length === 3) {
expect(receivedMetrics).toHaveLength(3);
expect(receivedMetrics.every(m => m.category === 'execution')).toBe(true);
unsubscribe();
resolve();
}
});
// Generate some metrics
for (let i = 0; i < 3; i++) {
const metricId = tracker.startMetricCollection(`real-time-${i}`, {
category: 'execution'
});
tracker.endMetricCollection(metricId);
}
});
});
it('should support metric aggregation windows', async () => {
// Generate metrics first
for (let i = 0; i < 10; i++) {
tracker.recordMetric({
id: `rapid-${i}`,
name: 'rapid-operation',
category: 'execution',
timestamp: Date.now(),
duration: 100 + i * 10,
data: {}
});
}
// Configure aggregation window
tracker.configureAggregation({
windowSize: 50, // milliseconds - short window for test
aggregationFunction: 'average',
categories: ['execution']
});
await new Promise(resolve => setTimeout(resolve, 100));
const aggregatedMetrics = tracker.getAggregatedMetrics('execution');
expect(aggregatedMetrics).toBeDefined();
expect(aggregatedMetrics.length).toBeGreaterThan(0);
});
it('should maintain metric history within limits', () => {
const maxSize = 5;
tracker = new PerformanceTracker({
maxMetricsHistorySize: maxSize
});
// Add more metrics than the limit
for (let i = 0; i < 10; i++) {
tracker.recordMetric({
id: `metric-${i}`,
name: 'test-operation',
category: 'execution',
timestamp: Date.now() + i,
duration: 100,
data: {}
});
}
const allMetrics = tracker.getAllMetrics();
expect(allMetrics.length).toBe(maxSize);
// Should keep the most recent metrics
const timestamps = allMetrics.map(m => m.timestamp).sort((a, b) => b - a);
expect(timestamps[0]).toBeGreaterThan(timestamps[timestamps.length - 1]);
});
});
describe('Integration with Evolution Engine', () => {
it('should track evolution cycle metrics', () => {
const evolutionId = 'evolution-001';
tracker.startEvolutionCycle(evolutionId, {
initialPopulationSize: 10,
targetFitness: 0.9,
maxGenerations: 100
});
tracker.recordEvolutionGeneration(evolutionId, {
generation: 1,
bestFitness: 0.75,
averageFitness: 0.65,
diversityScore: 0.8,
mutationRate: 0.1,
selectionPressure: 0.7
});
tracker.recordEvolutionGeneration(evolutionId, {
generation: 2,
bestFitness: 0.82,
averageFitness: 0.71,
diversityScore: 0.75,
mutationRate: 0.12,
selectionPressure: 0.75
});
tracker.endEvolutionCycle(evolutionId, {
finalFitness: 0.91,
totalGenerations: 2,
convergenceAchieved: true,
terminationReason: 'target_fitness_reached'
});
const evolutionMetrics = tracker.getEvolutionMetrics(evolutionId);
expect(evolutionMetrics).toBeDefined();
expect(evolutionMetrics.totalGenerations).toBe(2);
expect(evolutionMetrics.finalFitness).toBe(0.91);
expect(evolutionMetrics.convergenceAchieved).toBe(true);
expect(evolutionMetrics.fitnessImprovement).toBe(0.91 - 0.75);
});
it('should track prompt mutation effectiveness', () => {
const mutationTypes = ['crossover', 'point_mutation', 'inversion', 'duplication'];
mutationTypes.forEach((type, index) => {
tracker.recordMutationResult(type, {
success: index % 2 === 0,
fitnessChange: (index % 2 === 0) ? 0.1 : -0.05,
executionTime: 50 + index * 10,
resourceUsage: {
memoryPeak: 1024 * 1024 * (10 + index),
cpuTime: 100 + index * 20
}
});
});
const mutationAnalysis = tracker.analyzeMutationEffectiveness();
expect(mutationAnalysis.byType.crossover).toBeDefined();
expect(mutationAnalysis.byType.point_mutation).toBeDefined();
expect(mutationAnalysis.overall.successRate).toBe(0.5);
expect(mutationAnalysis.overall.averageFitnessChange).toBeCloseTo(0.025, 3);
});
it('should provide evolution performance insights', () => {
const evolutionId = 'evolution-insights-test';
// Simulate a complete evolution cycle
tracker.startEvolutionCycle(evolutionId, {
initialPopulationSize: 20,
targetFitness: 0.95,
maxGenerations: 50
});
// Add multiple generations with varying performance
for (let gen = 1; gen <= 10; gen++) {
tracker.recordEvolutionGeneration(evolutionId, {
generation: gen,
bestFitness: 0.5 + (gen * 0.04), // Gradual improvement
averageFitness: 0.4 + (gen * 0.03),
diversityScore: 0.9 - (gen * 0.02), // Decreasing diversity
mutationRate: 0.1 + (gen * 0.01),
selectionPressure: 0.6 + (gen * 0.02)
});
}
tracker.endEvolutionCycle(evolutionId, {
finalFitness: 0.9,
totalGenerations: 10,
convergenceAchieved: false,
terminationReason: 'max_generations_reached'
});
const insights = tracker.getEvolutionInsights(evolutionId);
expect(insights).toBeDefined();
expect(insights.convergenceRate).toBeDefined();
expect(insights.diversityTrend).toBe('decreasing');
expect(insights.recommendedAdjustments).toBeDefined();
expect(insights.recommendedAdjustments.length).toBeGreaterThan(0);
});
});
describe('Memory and Resource Tracking', () => {
it('should track memory usage patterns', () => {
const memorySnapshots: NodeJS.MemoryUsage[] = [
{ rss: 50000000, heapTotal: 30000000, heapUsed: 20000000, external: 5000000, arrayBuffers: 1000000 },
{ rss: 55000000, heapTotal: 32000000, heapUsed: 22000000, external: 5500000, arrayBuffers: 1100000 },
{ rss: 60000000, heapTotal: 35000000, heapUsed: 25000000, external: 6000000, arrayBuffers: 1200000 }
];
memorySnapshots.forEach((snapshot, i) => {
mockMemoryProvider.mockReturnValueOnce(snapshot);
tracker.recordMemorySnapshot(`operation-${i}`);
});
const memoryAnalysis = tracker.analyzeMemoryUsage();
expect(memoryAnalysis.peakHeapUsed).toBe(25000000);
expect(memoryAnalysis.averageHeapUsed).toBe(22333333.333333332);
expect(memoryAnalysis.memoryGrowthRate).toBeGreaterThan(0);
expect(memoryAnalysis.potentialLeaks).toBeDefined();
});
it('should detect memory leaks', () => {
// Simulate memory leak pattern
const baseMemory = 20000000;
for (let i = 0; i < 10; i++) {
mockMemoryProvider.mockReturnValueOnce({
rss: 50000000 + (i * 5000000), // Steadily increasing
heapTotal: 30000000 + (i * 2000000),
heapUsed: baseMemory + (i * 2000000), // Linear growth indicating leak
external: 5000000,
arrayBuffers: 1000000
});
tracker.recordMemorySnapshot(`leak-test-${i}`);
}
const leakDetection = tracker.detectMemoryLeaks();
expect(leakDetection.leakDetected).toBe(true);
expect(leakDetection.leakRate).toBeGreaterThan(0);
expect(leakDetection.confidence).toBeGreaterThan(0.8);
expect(leakDetection.recommendedActions).toBeDefined();
});
it('should track garbage collection impact', () => {
// Mock GC events
const gcEvents = [
{ type: 'minor', duration: 5, heapBefore: 25000000, heapAfter: 20000000 },
{ type: 'major', duration: 15, heapBefore: 30000000, heapAfter: 18000000 },
{ type: 'minor', duration: 3, heapBefore: 22000000, heapAfter: 19000000 }
];
gcEvents.forEach(event => {
tracker.recordGarbageCollection(event);
});
const gcAnalysis = tracker.analyzeGarbageCollection();
expect(gcAnalysis.totalEvents).toBe(3);
expect(gcAnalysis.averageDuration).toBe(7.666666666666667);
expect(gcAnalysis.totalMemoryReclaimed).toBe(20000000); // Sum of memory reclaimed: (25-20)+(30-18)+(22-19) = 5+12+3 = 20
expect(gcAnalysis.gcEfficiency).toBeGreaterThan(0);
});
});
describe('Error Handling and Edge Cases', () => {
it('should handle invalid metric IDs gracefully', () => {
expect(() => tracker.endMetricCollection('non-existent-id')).toThrow('Metric not found');
});
it('should handle empty metric collections', () => {
const analysis = tracker.calculateStatistics('non-existent-category');
expect(analysis.sampleSize).toBe(0);
expect(analysis.mean).toBe(0);
expect(analysis.median).toBe(0);
});
it('should validate metric data integrity', () => {
expect(() => {
tracker.recordMetric({
id: '',
name: 'test',
category: 'execution',
timestamp: Date.now(),
duration: -100, // Invalid negative duration
data: {}
});
}).toThrow('Invalid metric data');
});
it('should handle concurrent access safely', async () => {
const promises = Array.from({ length: 100 }, async (_, i) => {
return new Promise<void>(resolve => {
setTimeout(() => {
tracker.recordMetric({
id: `concurrent-${i}`,
name: 'concurrent-test',
category: 'execution',
timestamp: Date.now(),
duration: Math.random() * 100,
data: { index: i }
});
resolve();
}, Math.random() * 10);
});
});
await Promise.all(promises);
const metrics = tracker.getMetricsByCategory('execution');
expect(metrics.length).toBe(100);
// Verify no data corruption
const indices = metrics.map(m => m.data.index).sort((a, b) => a - b);
expect(indices).toEqual(Array.from({ length: 100 }, (_, i) => i));
});
});
});