reflection-optimizations.ts•11.6 kB
/**
* OPTIMIZATION: Performance enhancements for Reflection Engine
* These provide pattern matching, batch processing, and memory management optimizations
*/
import type {
ExecutionTrajectory,
FailurePattern
} from '../types/gepa';
import type { LLMAdapter } from '../services/llm-adapter';
// Suffix tree implementation removed to eliminate unused code
// Would be implemented here for advanced pattern matching if needed
/**
* OPTIMIZATION: Advanced pattern matcher with suffix trees and caching
*/
export class PatternMatcher {
// Suffix tree would be used for advanced pattern matching - currently unused
private readonly patternMinFrequency: number;
constructor(patternMinFrequency: number = 2) {
this.patternMinFrequency = patternMinFrequency;
}
/**
* Filter trajectories to only include those likely to contain relevant patterns
*/
filterRelevantTrajectories(trajectories: ExecutionTrajectory[]): ExecutionTrajectory[] {
// OPTIMIZATION: Quick filtering based on failure indicators
return trajectories.filter(trajectory => {
// Include failed trajectories
if (!trajectory.finalResult.success) return true;
// Include trajectories with errors in steps
if (trajectory.steps.some(step => step.error)) return true;
// Include trajectories with low scores
if (trajectory.finalResult.score < 0.5) return true;
// Include trajectories with long execution times (potential inefficiency)
if (trajectory.executionTime > 30000) return true; // 30 seconds
return false;
});
}
/**
* Extract patterns using optimized string matching
*/
extractPatterns(trajectories: ExecutionTrajectory[]): FailurePattern[] {
if (trajectories.length === 0) return [];
// OPTIMIZATION: Could build suffix tree for efficient pattern detection
// this.buildSuffixTree(trajectories);
const patterns: FailurePattern[] = [];
const patternCounts = new Map<string, number>();
const patternExamples = new Map<string, string[]>();
// Extract error patterns
trajectories.forEach(trajectory => {
const errorPatterns = this.extractErrorPatterns(trajectory);
errorPatterns.forEach(pattern => {
const key = `${pattern.type}-${pattern.description}`;
patternCounts.set(key, (patternCounts.get(key) || 0) + 1);
if (!patternExamples.has(key)) {
patternExamples.set(key, []);
}
const examples = patternExamples.get(key);
if (examples) {
examples.push(...pattern.examples);
}
});
});
// Convert to FailurePattern objects
patternCounts.forEach((frequency, key) => {
if (frequency >= this.patternMinFrequency) {
const [type, description] = key.split('-', 2);
const patternType = type ?? 'unknown';
const patternDescription = description ?? 'No description available';
const examples = patternExamples.get(key)?.slice(0, 5) ?? [];
patterns.push({
type: patternType,
frequency,
description: patternDescription,
examples
});
}
});
return patterns.sort((a, b) => b.frequency - a.frequency);
}
// Method removed to eliminate unused code warnings
// Would build suffix tree for advanced pattern matching if needed
private extractErrorPatterns(trajectory: ExecutionTrajectory): FailurePattern[] {
const patterns: FailurePattern[] = [];
// Timeout patterns
if (this.isTimeoutError(trajectory)) {
patterns.push({
type: 'timeout',
frequency: 1,
description: 'Task execution timeout',
examples: [trajectory.finalResult.error || 'Execution timeout']
});
}
// API error patterns
if (this.isAPIError(trajectory)) {
patterns.push({
type: 'api_error',
frequency: 1,
description: 'External API failure',
examples: [trajectory.finalResult.error || 'API call failed']
});
}
// Logic error patterns
if (this.isLogicError(trajectory)) {
patterns.push({
type: 'logic_error',
frequency: 1,
description: 'Logical reasoning failure',
examples: [String(trajectory.finalResult.output) || 'Logic error detected']
});
}
return patterns;
}
private isTimeoutError(trajectory: ExecutionTrajectory): boolean {
const error = trajectory.finalResult.error?.toLowerCase() || '';
return error.includes('timeout') || error.includes('exceeded');
}
private isAPIError(trajectory: ExecutionTrajectory): boolean {
const error = trajectory.finalResult.error?.toLowerCase() || '';
return error.includes('api') || error.includes('network') ||
error.includes('connection') || error.includes('http');
}
private isLogicError(trajectory: ExecutionTrajectory): boolean {
// Check for logical inconsistencies in output
const output = String(trajectory.finalResult.output || '').toLowerCase();
const hasContradiction = output.includes('but also') ||
output.includes('however') ||
output.includes('contradiction');
return hasContradiction || trajectory.finalResult.score < 0.3;
}
}
/**
* OPTIMIZATION: Batch analysis processor with parallel processing
*/
export class BatchAnalysisProcessor {
private llmAdapter: LLMAdapter;
constructor(llmAdapter: LLMAdapter) {
this.llmAdapter = llmAdapter;
}
/**
* Process trajectories in optimized batches
*/
async processTrajectories(trajectories: ExecutionTrajectory[], batchSize: number): Promise<any[]> {
const results: any[] = [];
// OPTIMIZATION: Process multiple batches in parallel
const batchPromises: Promise<any>[] = [];
for (let i = 0; i < trajectories.length; i += batchSize) {
const batch = trajectories.slice(i, i + batchSize);
batchPromises.push(this.processBatch(batch));
// Limit concurrent batch processing to avoid overwhelming the system
if (batchPromises.length >= 3) {
const batchResults = await Promise.allSettled(batchPromises);
batchResults.forEach(result => {
if (result.status === 'fulfilled') {
results.push(result.value);
} else {
// eslint-disable-next-line no-console
console.warn(`Batch processing failed: ${result.reason}`);
}
});
batchPromises.length = 0; // Clear array
}
}
// Process remaining batches
if (batchPromises.length > 0) {
const batchResults = await Promise.allSettled(batchPromises);
batchResults.forEach(result => {
if (result.status === 'fulfilled') {
results.push(result.value);
}
});
}
return results;
}
private async processBatch(trajectories: ExecutionTrajectory[]): Promise<any> {
const analysisPrompt = this.buildOptimizedBatchPrompt(trajectories);
try {
const response = await this.llmAdapter.callLLM(analysisPrompt);
if (!response || !response.content) {
throw new Error('No content in LLM response');
}
return JSON.parse(response.content);
} catch (analysisError: any) {
// OPTIMIZATION: Fallback to simplified analysis
console.warn('LLM analysis failed:', analysisError.message || analysisError);
return this.createFallbackAnalysis(trajectories);
}
}
private buildOptimizedBatchPrompt(trajectories: ExecutionTrajectory[]): string {
// OPTIMIZATION: Reduce prompt size by extracting only essential information
const essentialData = trajectories.map(traj => ({
id: traj.id,
success: traj.finalResult.success,
score: traj.finalResult.score,
errorType: this.categorizeError(traj.finalResult.error),
stepCount: traj.steps.length,
hasStepErrors: traj.steps.some(step => step.error)
}));
return `
Analyze these trajectory summaries to identify patterns efficiently:
${JSON.stringify(essentialData, null, 2)}
Respond with JSON containing:
{
"commonPatterns": [{"type": "string", "frequency": number, "description": "string", "examples": ["string"]}],
"recommendations": [{"priority": "high|medium|low", "type": "string", "targetSection": "string", "proposedChange": "string", "rationale": "string", "expectedImpact": number, "affectedTrajectories": ["string"]}],
"overallConfidence": number
}`;
}
private categorizeError(error?: string): string {
if (!error) return 'none';
const errorLower = error.toLowerCase();
if (errorLower.includes('timeout')) return 'timeout';
if (errorLower.includes('api') || errorLower.includes('network')) return 'api';
if (errorLower.includes('logic') || errorLower.includes('reasoning')) return 'logic';
if (errorLower.includes('format') || errorLower.includes('parse')) return 'format';
return 'unknown';
}
private createFallbackAnalysis(trajectories: ExecutionTrajectory[]): any {
// Simple pattern detection as fallback
const errorTypes = new Map<string, number>();
trajectories.forEach(traj => {
const errorType = this.categorizeError(traj.finalResult.error);
errorTypes.set(errorType, (errorTypes.get(errorType) || 0) + 1);
});
const patterns = Array.from(errorTypes.entries()).map(([type, frequency]) => ({
type,
frequency,
description: `${type} errors detected`,
examples: [type]
}));
return {
commonPatterns: patterns,
recommendations: [],
overallConfidence: 0.5
};
}
}
/**
* OPTIMIZATION: Memory manager for analysis caching and optimization
*/
export class AnalysisMemoryManager {
private patternCache = new Map<string, { patterns: FailurePattern[]; timestamp: number }>();
private analysisCache = new Map<string, { analysis: any; timestamp: number }>();
private readonly ttl: number;
constructor(ttl: number) {
this.ttl = ttl;
}
getCachedPatterns(key: string): FailurePattern[] | null {
const cached = this.patternCache.get(key);
if (!cached) return null;
if (Date.now() - cached.timestamp > this.ttl) {
this.patternCache.delete(key);
return null;
}
return cached.patterns;
}
cachePatterns(key: string, patterns: FailurePattern[]): void {
this.patternCache.set(key, {
patterns,
timestamp: Date.now()
});
// Cleanup old entries
this.cleanupCache();
}
getCachedAnalysis(key: string): any | null {
const cached = this.analysisCache.get(key);
if (!cached) return null;
if (Date.now() - cached.timestamp > this.ttl) {
this.analysisCache.delete(key);
return null;
}
return cached.analysis;
}
cacheAnalysis(key: string, analysis: unknown): void {
this.analysisCache.set(key, {
analysis,
timestamp: Date.now()
});
this.cleanupCache();
}
private cleanupCache(): void {
const now = Date.now();
// Clean pattern cache
Array.from(this.patternCache.entries()).forEach(([key, entry]) => {
if (now - entry.timestamp > this.ttl) {
this.patternCache.delete(key);
}
});
// Clean analysis cache
Array.from(this.analysisCache.entries()).forEach(([key, entry]) => {
if (now - entry.timestamp > this.ttl) {
this.analysisCache.delete(key);
}
});
}
clear(): void {
this.patternCache.clear();
this.analysisCache.clear();
}
}