Nutrient Document Engine MCP Server

/** * Result reporting and formatting for evaluation results */ import { FocusedEvaluationResults } from './types.js'; import fs from 'fs'; import path, { dirname } from 'path'; import { fileURLToPath } from 'url'; /** * Reporter class for formatting and displaying evaluation results */ export class EvaluationReporter { /** * Print comparison results */ printComparison(results: FocusedEvaluationResults[]): void { console.log('\n' + '='.repeat(80)); console.log('🏆 MODEL COMPARISON - TOOL USAGE EFFECTIVENESS'); console.log('='.repeat(80)); // Sort by overall score const sorted = [...results].sort((a, b) => b.overallScore - a.overallScore); console.log('\n📊 OVERALL RANKINGS:'); sorted.forEach((result, index) => { const rank = index + 1; const emoji = rank === 1 ? '🥇' : rank === 2 ? '🥈' : rank === 3 ? '🥉' : ' '; console.log( `${emoji} ${rank}. ${result.model.padEnd(20)} ${(result.overallScore * 100).toFixed(1)}%` ); }); console.log('\n📋 DETAILED BREAKDOWN:'); console.log( 'Model'.padEnd(20) + 'Overall'.padEnd(10) + 'Tools'.padEnd(10) + 'Order'.padEnd(10) + 'Efficiency' ); console.log('-'.repeat(60)); sorted.forEach(result => { const overall = (result.overallScore * 100).toFixed(1) + '%'; const tools = (result.correctToolUsage * 100).toFixed(1) + '%'; const order = (result.correctOrderUsage * 100).toFixed(1) + '%'; const efficiency = (result.efficiencyScore * 100).toFixed(1) + '%'; console.log( result.model.padEnd(20) + overall.padEnd(10) + tools.padEnd(10) + order.padEnd(10) + efficiency ); }); // Find best and worst scenarios console.log('\n🎯 KEY INSIGHTS:'); const allResults = results.flatMap(r => r.results); const bestScenarios = allResults.filter(r => r.score === 1.0); const worstScenarios = allResults.filter(r => r.score < 0.5); if (bestScenarios.length > 0) { const commonBest = this.findMostCommon(bestScenarios.map(r => r.scenarioId)); console.log(`✅ Best performing scenario: ${commonBest} (consistent across models)`); } if (worstScenarios.length > 0) { const commonWorst = this.findMostCommon(worstScenarios.map(r => r.scenarioId)); console.log(`❌ Most challenging scenario: ${commonWorst} (needs improvement)`); } console.log('='.repeat(80)); } /** * Print progress update for a single scenario */ printScenarioProgress( index: number, total: number, scenario: { description: string }, result: { score: number } ): void { const status = result.score > 0.8 ? '✅' : result.score > 0.5 ? '⚠️' : '❌'; console.log( ` ${index + 1}/${total} ${status} ${scenario.description} (${(result.score * 100).toFixed(0)}%)` ); } /** * Print error for failed scenario */ printScenarioError( index: number, total: number, scenario: { description: string }, error: Error | string ): void { console.log(` ${index + 1}/${total} ❌ ${scenario.description} - ERROR: ${error}`); } /** * Print model evaluation summary */ printModelSummary(modelName: string, result: FocusedEvaluationResults): void { console.log(` Overall Score: ${(result.overallScore * 100).toFixed(1)}%`); console.log(` Correct Tools: ${(result.correctToolUsage * 100).toFixed(1)}%`); console.log(` Correct Order: ${(result.correctOrderUsage * 100).toFixed(1)}%`); console.log(` Efficiency: ${(result.efficiencyScore * 100).toFixed(1)}%`); } /** * Save results to file */ async saveResults( results: FocusedEvaluationResults[], filename: string = path.join( dirname(fileURLToPath(import.meta.url)), 'results', `${new Date(Date.now()).toISOString()}.json` ) ): Promise<void> { try { await fs.promises.mkdir(path.dirname(filename)); await fs.promises.writeFile(filename, JSON.stringify(results, null, 2)); console.log(`\n📁 Results saved to: ${filename}`); } catch (error) { console.warn(`⚠️ Could not save results: ${error}`); } } /** * Find most common item in array */ private findMostCommon(items: string[]): string { const counts = new Map<string, number>(); items.forEach(item => counts.set(item, (counts.get(item) || 0) + 1)); let maxCount = 0; let mostCommon = ''; counts.forEach((count, item) => { if (count > maxCount) { maxCount = count; mostCommon = item; } }); return mostCommon; } }