sample-data.ts•13.2 kB
/**
* Sample test data for GEPA MCP Server tests
* Provides realistic test fixtures and data generators
*/
import type {
PromptCandidate,
ExecutionTrajectory,
ReflectionAnalysis,
TaskContext,
LLMResponse
} from '../../types/gepa';
/**
* Sample prompt candidates for testing
*/
export const samplePromptCandidates: PromptCandidate[] = [
{
id: 'prompt-basic-001',
content: 'You are a helpful assistant. Answer the question clearly and concisely.',
generation: 0,
taskPerformance: new Map([
['task-qa-001', 0.75],
['task-qa-002', 0.68],
['task-qa-003', 0.82]
]),
averageScore: 0.75,
rolloutCount: 3,
createdAt: new Date('2024-01-01T00:00:00Z'),
lastEvaluated: new Date('2024-01-01T01:00:00Z'),
mutationType: 'initial'
},
{
id: 'prompt-improved-001',
content: `You are a knowledgeable assistant specializing in clear, step-by-step explanations.
When answering questions:
1. Break down complex problems into manageable steps
2. Provide specific examples when helpful
3. Verify your reasoning before concluding
4. If information is missing, state what additional context would be helpful
Answer the question thoroughly but concisely.`,
parentId: 'prompt-basic-001',
generation: 1,
taskPerformance: new Map([
['task-qa-001', 0.89],
['task-qa-002', 0.85],
['task-qa-003', 0.92],
['task-qa-004', 0.78]
]),
averageScore: 0.86,
rolloutCount: 4,
createdAt: new Date('2024-01-01T02:00:00Z'),
lastEvaluated: new Date('2024-01-01T03:00:00Z'),
mutationType: 'reflection'
},
{
id: 'prompt-specialized-001',
content: `You are an expert research assistant with strong analytical capabilities.
Your approach to answering questions:
- Systematically analyze the query to identify key components
- Use available tools to gather comprehensive information
- Synthesize findings into a coherent, well-structured response
- Acknowledge limitations and uncertainties when present
For complex questions requiring multiple steps:
1. Decompose the problem into sub-questions
2. Address each sub-question methodically
3. Integrate partial answers into a complete solution
4. Provide confidence levels for different aspects of your answer
Be thorough yet efficient in your analysis.`,
parentId: 'prompt-improved-001',
generation: 2,
taskPerformance: new Map([
['task-research-001', 0.94],
['task-research-002', 0.91],
['task-analysis-001', 0.87]
]),
averageScore: 0.91,
rolloutCount: 3,
createdAt: new Date('2024-01-01T04:00:00Z'),
lastEvaluated: new Date('2024-01-01T05:00:00Z'),
mutationType: 'crossover'
}
];
/**
* Sample execution trajectories
*/
export const sampleTrajectories: ExecutionTrajectory[] = [
{
id: 'trajectory-success-001',
promptId: 'prompt-improved-001',
taskId: 'task-qa-001',
timestamp: new Date('2024-01-01T10:00:00Z'),
steps: [
{
stepNumber: 1,
action: 'analyze_question',
reasoning: 'Breaking down the multi-part question about renewable energy',
timestamp: new Date('2024-01-01T10:00:01Z')
},
{
stepNumber: 2,
action: 'search_information',
reasoning: 'Looking for current statistics on renewable energy adoption',
toolName: 'web_search',
toolInput: { query: 'renewable energy statistics 2024' },
toolOutput: {
results: [
'Global renewable capacity increased by 15% in 2024',
'Solar power now accounts for 12% of global electricity'
]
},
timestamp: new Date('2024-01-01T10:00:03Z')
},
{
stepNumber: 3,
action: 'synthesize_answer',
reasoning: 'Combining search results with analytical framework',
timestamp: new Date('2024-01-01T10:00:08Z')
}
],
finalResult: {
success: true,
score: 0.89,
output: 'Comprehensive analysis of renewable energy trends with supporting data and projections.'
},
llmCalls: [
{
model: 'claude-3-sonnet',
prompt: 'Analyze this question about renewable energy...',
response: 'This is a multi-faceted question requiring...',
tokens: { prompt: 200, completion: 150 },
latency: 1800,
timestamp: new Date()
}
],
toolCalls: [
{
toolName: 'web_search',
input: { query: 'renewable energy statistics 2024' },
output: { results: ['stat1', 'stat2'] },
success: true,
latency: 2200
}
],
totalTokens: 350,
executionTime: 8000
},
{
id: 'trajectory-failure-001',
promptId: 'prompt-basic-001',
taskId: 'task-complex-001',
timestamp: new Date('2024-01-01T11:00:00Z'),
steps: [
{
stepNumber: 1,
action: 'attempt_direct_answer',
reasoning: 'Trying to answer complex question without breaking it down',
timestamp: new Date('2024-01-01T11:00:01Z')
},
{
stepNumber: 2,
action: 'realize_insufficient_info',
reasoning: 'Recognizing that the question requires more detailed analysis',
error: 'Insufficient context for complete answer',
timestamp: new Date('2024-01-01T11:00:05Z')
}
],
finalResult: {
success: false,
score: 0.35,
output: 'Partial answer provided but lacks depth and supporting evidence.',
error: 'Failed to properly decompose complex question'
},
llmCalls: [
{
model: 'claude-3-sonnet',
prompt: 'Answer this complex question...',
response: 'I can provide a basic answer but...',
tokens: { prompt: 180, completion: 80 },
latency: 1200,
timestamp: new Date()
}
],
toolCalls: [],
totalTokens: 260,
executionTime: 5000
}
];
/**
* Sample reflection analyses
*/
export const sampleReflectionAnalyses: ReflectionAnalysis[] = [
{
trajectoryId: 'trajectory-failure-001',
promptId: 'prompt-basic-001',
diagnosis: {
failurePoint: 'Step 1: attempt_direct_answer',
rootCause: 'Prompt lacks guidance for handling complex, multi-part questions',
moduleResponsibility: new Map([
['question_analysis', 0.6],
['decomposition_strategy', 0.3],
['information_gathering', 0.1]
]),
patterns: [
{
type: 'insufficient_decomposition',
frequency: 4,
description: 'Failing to break down complex questions into manageable components',
examples: [
'Attempting to answer multi-part question as single unit',
'Missing intermediate reasoning steps',
'Overlooking dependencies between sub-questions'
]
},
{
type: 'premature_conclusion',
frequency: 2,
description: 'Drawing conclusions without sufficient analysis',
examples: [
'Providing surface-level answers to deep questions',
'Not seeking additional information when needed'
]
}
]
},
suggestions: [
{
type: 'add_instruction',
targetSection: 'question_handling',
proposedChange: 'Add explicit instruction to analyze question complexity and break down multi-part questions',
rationale: 'Systematic decomposition improves handling of complex queries',
expectedImpact: 0.25
},
{
type: 'add_example',
targetSection: 'examples',
proposedChange: 'Include example of step-by-step analysis for a complex question',
rationale: 'Concrete examples demonstrate the expected approach',
expectedImpact: 0.18
},
{
type: 'restructure',
targetSection: 'main_instructions',
proposedChange: 'Reorganize prompt to emphasize analytical approach before response generation',
rationale: 'Process-focused structure encourages systematic thinking',
expectedImpact: 0.15
}
],
confidence: 0.82,
rationale: 'Analysis based on 6 similar failure patterns across recent trajectories, with consistent issues in question decomposition and systematic approach.'
}
];
/**
* Sample task contexts
*/
export const sampleTaskContexts: TaskContext[] = [
{
taskId: 'task-qa-001',
description: 'Answer a factual question about renewable energy trends',
category: 'factual_qa',
difficulty: 'medium',
requiredCapabilities: ['web_search', 'data_analysis', 'synthesis'],
expectedDuration: 15000
},
{
taskId: 'task-complex-001',
description: 'Analyze the relationship between climate policies and economic outcomes across different regions',
category: 'analytical_reasoning',
difficulty: 'hard',
requiredCapabilities: ['web_search', 'data_analysis', 'multi_hop_reasoning', 'synthesis', 'comparison'],
expectedDuration: 45000
},
{
taskId: 'task-simple-001',
description: 'Define a technical term clearly',
category: 'definition',
difficulty: 'easy',
requiredCapabilities: ['knowledge_retrieval'],
expectedDuration: 5000
}
];
/**
* Sample LLM responses
*/
export const sampleLLMResponses: LLMResponse[] = [
{
content: 'Based on the available data, renewable energy adoption has accelerated significantly...',
model: 'claude-3-sonnet',
tokens: { prompt: 250, completion: 180, total: 430 },
finishReason: 'stop',
latency: 2100,
timestamp: new Date('2024-01-01T10:00:00Z')
},
{
content: 'I need more context to provide a complete answer...',
model: 'claude-3-sonnet',
tokens: { prompt: 180, completion: 45, total: 225 },
finishReason: 'stop',
latency: 900,
timestamp: new Date('2024-01-01T11:00:00Z')
}
];
/**
* Data generators for creating test data at scale
*/
export class TestDataGenerator {
/**
* Generate a prompt candidate with random performance data
*/
static generatePromptCandidate(overrides: Partial<PromptCandidate> = {}): PromptCandidate {
const id = `generated-prompt-${Math.random().toString(36).substr(2, 9)}`;
const taskCount = Math.floor(Math.random() * 5) + 1;
const taskPerformance = new Map<string, number>();
let totalScore = 0;
for (let i = 0; i < taskCount; i++) {
const score = Math.random() * 0.4 + 0.5; // Scores between 0.5 and 0.9
taskPerformance.set(`task-${i + 1}`, score);
totalScore += score;
}
return {
id,
content: `Generated prompt content for testing - ${id}`,
generation: Math.floor(Math.random() * 5),
taskPerformance,
averageScore: totalScore / taskCount,
rolloutCount: taskCount,
createdAt: new Date(Date.now() - Math.random() * 86400000), // Random time in last 24h
lastEvaluated: new Date(),
mutationType: ['initial', 'reflection', 'crossover', 'random'][Math.floor(Math.random() * 4)] as 'initial' | 'reflection' | 'crossover' | 'random',
...overrides
};
}
/**
* Generate multiple prompt candidates
*/
static generatePromptCandidates(count: number): PromptCandidate[] {
return Array.from({ length: count }, () => this.generatePromptCandidate());
}
/**
* Generate an execution trajectory with random success/failure
*/
static generateExecutionTrajectory(overrides: Partial<ExecutionTrajectory> = {}): ExecutionTrajectory {
const id = `generated-trajectory-${Math.random().toString(36).substr(2, 9)}`;
const success = Math.random() > 0.3; // 70% success rate
const stepCount = Math.floor(Math.random() * 5) + 2;
const steps = Array.from({ length: stepCount }, (_, i) => ({
stepNumber: i + 1,
action: `action_${i + 1}`,
reasoning: `Generated reasoning for step ${i + 1}`,
timestamp: new Date(Date.now() + i * 1000)
}));
return {
id,
promptId: `prompt-${Math.random().toString(36).substr(2, 6)}`,
taskId: `task-${Math.random().toString(36).substr(2, 6)}`,
timestamp: new Date(),
steps,
finalResult: {
success,
score: success ? Math.random() * 0.3 + 0.7 : Math.random() * 0.5 + 0.1,
output: success ? 'Generated successful output' : 'Generated failure output',
...(success ? {} : { error: 'Generated error message' })
},
llmCalls: [
{
model: 'claude-3-sonnet',
prompt: 'Generated test prompt',
response: 'Generated test response',
tokens: { prompt: Math.floor(Math.random() * 200) + 50, completion: Math.floor(Math.random() * 150) + 25 },
latency: Math.floor(Math.random() * 2000) + 500,
timestamp: new Date()
}
],
toolCalls: [],
totalTokens: Math.floor(Math.random() * 350) + 75,
executionTime: Math.floor(Math.random() * 10000) + 2000,
...overrides
};
}
/**
* Generate multiple execution trajectories
*/
static generateExecutionTrajectories(count: number, overrides: Partial<ExecutionTrajectory> = {}): ExecutionTrajectory[] {
return Array.from({ length: count }, () => this.generateExecutionTrajectory(overrides));
}
}