import { z } from 'zod';
import { createLogger } from '../utils/logger.js';
/**
* Metadata Enricher Service
*
* Enhances responses with confidence scores, data quality metrics,
* assumption tracking, and contextual metadata for AI agents.
*/
// Metadata schemas
export const DataQualityMetricsSchema = z.object({
completeness: z.number().min(0).max(1),
accuracy: z.number().min(0).max(1),
consistency: z.number().min(0).max(1),
timeliness: z.number().min(0).max(1),
overall: z.enum(['low', 'medium', 'high'])
});
export const ConfidenceMetricsSchema = z.object({
overall: z.number().min(0).max(1),
breakdown: z.object({
data_quality: z.number().min(0).max(1),
model_accuracy: z.number().min(0).max(1),
assumption_validity: z.number().min(0).max(1),
benchmark_alignment: z.number().min(0).max(1)
}),
factors: z.array(z.object({
factor: z.string(),
impact: z.enum(['positive', 'negative']),
weight: z.number()
}))
});
export const AssumptionTrackingSchema = z.object({
assumptions: z.array(z.object({
id: z.string(),
category: z.string(),
description: z.string(),
confidence: z.number().min(0).max(1),
impact: z.enum(['low', 'medium', 'high']),
sensitivity: z.number().optional(),
validation_method: z.string().optional()
})),
overall_impact: z.enum(['low', 'medium', 'high']),
key_dependencies: z.array(z.string())
});
export const ContextualMetadataSchema = z.object({
industry_context: z.object({
industry: z.string(),
market_maturity: z.enum(['emerging', 'growing', 'mature', 'declining']),
competitive_intensity: z.enum(['low', 'medium', 'high']),
regulatory_complexity: z.enum(['low', 'medium', 'high'])
}).optional(),
organization_context: z.object({
size: z.enum(['small', 'medium', 'large', 'enterprise']),
ai_maturity: z.enum(['beginner', 'intermediate', 'advanced', 'leader']),
change_readiness: z.enum(['low', 'medium', 'high']),
resource_availability: z.enum(['constrained', 'adequate', 'abundant'])
}).optional(),
temporal_context: z.object({
analysis_date: z.string().datetime(),
data_freshness: z.enum(['real-time', 'recent', 'historical']),
projection_horizon: z.string(),
seasonality_considered: z.boolean()
}),
calculation_context: z.object({
methodology: z.string(),
key_parameters: z.record(z.any()),
sensitivity_tested: z.boolean(),
scenario_count: z.number()
})
});
export type DataQualityMetrics = z.infer<typeof DataQualityMetricsSchema>;
export type ConfidenceMetrics = z.infer<typeof ConfidenceMetricsSchema>;
export type AssumptionTracking = z.infer<typeof AssumptionTrackingSchema>;
export type ContextualMetadata = z.infer<typeof ContextualMetadataSchema>;
export class MetadataEnricher {
private logger = createLogger({ component: 'MetadataEnricher' });
/**
* Enrich response with comprehensive metadata
*/
async enrichResponse(
response: any,
tool: string,
executionContext: any
): Promise<{
confidence: ConfidenceMetrics;
dataQuality: DataQualityMetrics;
assumptions: AssumptionTracking;
context: ContextualMetadata;
provenance: any;
}> {
this.logger.debug('Enriching response metadata', { tool });
const confidence = await this.calculateConfidence(response, tool, executionContext);
const dataQuality = this.assessDataQuality(response, executionContext);
const assumptions = this.trackAssumptions(response, tool);
const context = this.gatherContextualMetadata(response, tool, executionContext);
const provenance = this.generateProvenance(tool, executionContext);
return {
confidence,
dataQuality,
assumptions,
context,
provenance
};
}
/**
* Calculate comprehensive confidence metrics
*/
private async calculateConfidence(
response: any,
tool: string,
context: any
): Promise<ConfidenceMetrics> {
const breakdown = {
data_quality: 0.85, // Default, will be calculated
model_accuracy: 0.9, // Based on tool type
assumption_validity: 0.8, // Based on assumptions
benchmark_alignment: 0.75 // Based on benchmark usage
};
// Tool-specific confidence adjustments
switch (tool) {
case 'predict_roi':
breakdown.model_accuracy = this.assessROIModelAccuracy(response);
breakdown.benchmark_alignment = context.enable_benchmarks ? 0.9 : 0.7;
break;
case 'compare_projects':
breakdown.model_accuracy = response.projects?.length > 2 ? 0.95 : 0.85;
breakdown.benchmark_alignment = this.assessComparativeBenchmarks(response);
break;
}
// Calculate data quality impact
const dataCompleteness = this.assessDataCompleteness(response);
breakdown.data_quality = dataCompleteness;
// Assess assumption validity
breakdown.assumption_validity = this.assessAssumptionValidity(response, context);
// Calculate overall confidence
const weights = {
data_quality: 0.3,
model_accuracy: 0.3,
assumption_validity: 0.25,
benchmark_alignment: 0.15
};
const overall = Object.entries(breakdown).reduce((sum, [key, value]) => {
return sum + value * weights[key as keyof typeof weights];
}, 0);
// Identify confidence factors
const factors = this.identifyConfidenceFactors(breakdown, response, context);
return {
overall,
breakdown,
factors
};
}
/**
* Assess data quality metrics
*/
private assessDataQuality(response: any, context: any): DataQualityMetrics {
const metrics = {
completeness: 1.0,
accuracy: 0.9,
consistency: 0.95,
timeliness: 0.85
};
// Completeness check
if (response.use_cases) {
const missingFields = response.use_cases.filter((uc: any) =>
!uc.monthly_benefit || !uc.category || !uc.name
).length;
metrics.completeness = 1 - (missingFields / response.use_cases.length);
}
// Accuracy assessment (based on calculation methods)
if (response.metadata?.calculated_with_benchmarks) {
metrics.accuracy = 0.95;
} else if (context.confidence_level && context.confidence_level > 0.9) {
metrics.accuracy = 0.9;
}
// Consistency check
if (response.financial_metrics) {
const hasConsistentMetrics = this.checkMetricConsistency(response.financial_metrics);
metrics.consistency = hasConsistentMetrics ? 0.95 : 0.7;
}
// Timeliness (based on data freshness)
if (context.benchmark_config?.enable_real_time) {
metrics.timeliness = 0.95;
} else if (response.metadata?.calculation_timestamp) {
const ageHours = (Date.now() - new Date(response.metadata.calculation_timestamp).getTime()) / 3600000;
metrics.timeliness = Math.max(0.5, 1 - (ageHours / 168)); // Decay over a week
}
// Calculate overall quality
const avgScore = Object.values(metrics).reduce((a, b) => a + b, 0) / 4;
const overall = avgScore >= 0.85 ? 'high' : avgScore >= 0.7 ? 'medium' : 'low';
return {
...metrics,
overall
};
}
/**
* Track and analyze assumptions
*/
private trackAssumptions(response: any, tool: string): AssumptionTracking {
const assumptions = [];
// Common assumptions across all tools
assumptions.push({
id: 'A001',
category: 'Financial',
description: 'Discount rate remains stable at 10% annually',
confidence: 0.9,
impact: 'medium' as const,
sensitivity: 0.15,
validation_method: 'Industry standard rates'
});
// Tool-specific assumptions
switch (tool) {
case 'predict_roi':
assumptions.push(
{
id: 'A002',
category: 'Implementation',
description: 'Linear ramp-up over 6-month period',
confidence: 0.8,
impact: 'high' as const,
sensitivity: 0.25,
validation_method: 'Historical project data'
},
{
id: 'A003',
category: 'Adoption',
description: 'User adoption follows typical S-curve',
confidence: 0.85,
impact: 'medium' as const,
sensitivity: 0.2,
validation_method: 'Industry benchmarks'
}
);
break;
case 'compare_projects':
assumptions.push({
id: 'A004',
category: 'Comparison',
description: 'Projects evaluated under similar market conditions',
confidence: 0.75,
impact: 'medium' as const,
validation_method: 'Temporal alignment check'
});
break;
}
// Add response-specific assumptions
if (response.metadata?.assumptions) {
response.metadata.assumptions.forEach((assumption: any, index: number) => {
assumptions.push({
id: `A${(100 + index).toString().padStart(3, '0')}`,
category: assumption.category || 'General',
description: assumption.description,
confidence: assumption.confidence || 0.75,
impact: assumption.impact || 'medium',
validation_method: 'User provided'
});
});
}
// Calculate overall impact
const highImpactCount = assumptions.filter(a => a.impact === 'high').length;
const overall_impact = highImpactCount > assumptions.length / 3 ? 'high' :
highImpactCount > 0 ? 'medium' : 'low';
// Identify key dependencies
const key_dependencies = assumptions
.filter(a => a.impact === 'high' && a.confidence < 0.8)
.map(a => a.description);
return {
assumptions,
overall_impact,
key_dependencies
};
}
/**
* Gather contextual metadata
*/
private gatherContextualMetadata(
response: any,
tool: string,
context: any
): ContextualMetadata {
// Industry context
const industry_context = context.industry ? {
industry: context.industry,
market_maturity: this.assessMarketMaturity(context.industry),
competitive_intensity: 'medium' as const, // Could be enhanced with real data
regulatory_complexity: this.assessRegulatoryComplexity(context.industry)
} : undefined;
// Organization context
const organization_context = context.company_size ? {
size: context.company_size,
ai_maturity: this.inferAIMaturity(response, context),
change_readiness: 'medium' as const, // Default, could be assessed
resource_availability: this.inferResourceAvailability(context)
} : undefined;
// Temporal context
const temporal_context = {
analysis_date: new Date().toISOString(),
data_freshness: (context.benchmark_config?.enable_real_time ? 'real-time' : 'recent') as 'real-time' | 'recent' | 'historical',
projection_horizon: `${context.timeline_months || 60} months`,
seasonality_considered: false // Could be enhanced
};
// Calculation context
const calculation_context = {
methodology: this.getMethodologyName(tool),
key_parameters: {
discount_rate: 0.1,
confidence_level: context.confidence_level || 0.95,
simulation_iterations: context.simulation_iterations || 10000
},
sensitivity_tested: tool === 'predict_roi',
scenario_count: this.countScenarios(response)
};
return {
industry_context,
organization_context,
temporal_context,
calculation_context
};
}
/**
* Generate provenance information
*/
private generateProvenance(tool: string, context: any): any {
return {
tool_name: tool,
tool_version: '2.0',
execution_id: this.generateExecutionId(),
timestamp: new Date().toISOString(),
data_sources: this.identifyDataSources(context),
calculation_methods: this.identifyCalculationMethods(tool),
external_apis_used: this.identifyExternalAPIs(context),
caching_used: false, // Could track actual cache usage
processing_location: 'server'
};
}
// Helper methods
private assessROIModelAccuracy(response: any): number {
// Factors: calculation completeness, use case detail, projection quality
let accuracy = 0.9;
if (response.use_cases?.length > 5) {
accuracy -= 0.05; // More complexity, slightly less accurate
}
if (response.financial_metrics?.conservative && response.financial_metrics?.optimistic) {
accuracy += 0.05; // Multiple scenarios increase confidence
}
return Math.min(0.95, Math.max(0.7, accuracy));
}
private assessComparativeBenchmarks(response: any): number {
const projectsWithBenchmarks = response.projects?.filter((p: any) =>
p.benchmark_comparison?.available
).length || 0;
const totalProjects = response.projects?.length || 1;
const benchmarkCoverage = projectsWithBenchmarks / totalProjects;
return 0.6 + (benchmarkCoverage * 0.35); // Base 0.6, up to 0.95
}
private assessDataCompleteness(response: any): number {
const requiredFields = [
'summary',
'financial_metrics',
'use_cases',
'metadata'
];
const presentFields = requiredFields.filter(field =>
response[field] && Object.keys(response[field]).length > 0
).length;
return presentFields / requiredFields.length;
}
private assessAssumptionValidity(response: any, context: any): number {
let validity = 0.8; // Base validity
// Adjust based on data sources
if (context.enable_benchmarks || context.use_industry_benchmarks) {
validity += 0.1;
}
// Adjust based on confidence level
if (context.confidence_level && context.confidence_level > 0.9) {
validity += 0.05;
}
// Adjust based on complexity
if (response.use_cases?.length > 10) {
validity -= 0.1; // More assumptions with more use cases
}
return Math.min(0.95, Math.max(0.6, validity));
}
private identifyConfidenceFactors(
breakdown: any,
response: any,
context: any
): Array<{factor: string; impact: 'positive' | 'negative'; weight: number}> {
const factors = [];
// Positive factors
if (breakdown.benchmark_alignment > 0.8) {
factors.push({
factor: 'Strong benchmark alignment',
impact: 'positive' as const,
weight: 0.2
});
}
if (response.use_cases?.length >= 3 && response.use_cases?.length <= 7) {
factors.push({
factor: 'Optimal use case portfolio size',
impact: 'positive' as const,
weight: 0.15
});
}
// Negative factors
if (breakdown.data_quality < 0.8) {
factors.push({
factor: 'Data quality concerns',
impact: 'negative' as const,
weight: 0.25
});
}
if (!context.enable_benchmarks && !context.use_industry_benchmarks) {
factors.push({
factor: 'No external benchmarks used',
impact: 'negative' as const,
weight: 0.15
});
}
return factors;
}
private checkMetricConsistency(financialMetrics: any): boolean {
if (!financialMetrics.conservative || !financialMetrics.expected || !financialMetrics.optimistic) {
return false;
}
// Check logical ordering
const conservative = financialMetrics.conservative.total_monthly_benefit;
const expected = financialMetrics.expected.total_monthly_benefit;
const optimistic = financialMetrics.optimistic.total_monthly_benefit;
return conservative <= expected && expected <= optimistic;
}
private assessMarketMaturity(industry: string): 'emerging' | 'growing' | 'mature' | 'declining' {
const maturityMap: Record<string, 'emerging' | 'growing' | 'mature' | 'declining'> = {
'technology': 'growing',
'financial_services': 'mature',
'healthcare': 'growing',
'retail': 'mature',
'manufacturing': 'mature',
'education': 'emerging',
'government': 'emerging'
};
return maturityMap[industry] || 'growing';
}
private assessRegulatoryComplexity(industry: string): 'low' | 'medium' | 'high' {
const complexityMap: Record<string, 'low' | 'medium' | 'high'> = {
'financial_services': 'high',
'healthcare': 'high',
'government': 'high',
'education': 'medium',
'technology': 'low',
'retail': 'low',
'manufacturing': 'medium'
};
return complexityMap[industry] || 'medium';
}
private inferAIMaturity(response: any, context: any): 'beginner' | 'intermediate' | 'advanced' | 'leader' {
const useCaseCount = response.use_cases?.length || 0;
const investment = response.summary?.total_investment || 0;
if (useCaseCount > 10 && investment > 1000000) {
return 'leader';
} else if (useCaseCount > 5 || investment > 500000) {
return 'advanced';
} else if (useCaseCount > 2) {
return 'intermediate';
}
return 'beginner';
}
private inferResourceAvailability(context: any): 'constrained' | 'adequate' | 'abundant' {
const size = context.company_size;
if (size === 'enterprise' || size === 'large') {
return 'abundant';
} else if (size === 'medium') {
return 'adequate';
}
return 'constrained';
}
private getMethodologyName(tool: string): string {
const methodologies: Record<string, string> = {
'predict_roi': 'DCF with Monte Carlo simulation',
'compare_projects': 'Multi-criteria portfolio analysis'
};
return methodologies[tool] || 'Standard financial analysis';
}
private countScenarios(response: any): number {
if (response.scenarios) {
return Object.keys(response.scenarios).length;
} else if (response.financial_metrics) {
return Object.keys(response.financial_metrics).length;
}
return 1;
}
private generateExecutionId(): string {
return `exec_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
private identifyDataSources(context: any): string[] {
const sources = ['Internal calculations'];
if (context.enable_benchmarks || context.use_industry_benchmarks) {
sources.push('Industry benchmarks');
}
if (context.benchmark_config?.enable_real_time) {
sources.push('Real-time market data');
}
return sources;
}
private identifyCalculationMethods(tool: string): string[] {
const methods: Record<string, string[]> = {
'predict_roi': [
'Discounted Cash Flow (DCF)',
'Net Present Value (NPV)',
'Internal Rate of Return (IRR)',
'Monte Carlo simulation'
],
'compare_projects': [
'Comparative analysis',
'Portfolio optimization',
'Risk-return assessment',
'ML pattern detection'
],
};
return methods[tool] || ['Standard analysis'];
}
private identifyExternalAPIs(context: any): string[] {
const apis = [];
if (context.benchmark_config?.sonar_api_key) {
apis.push('Perplexity Sonar API');
}
if (context.benchmark_config?.fmp_api_key) {
apis.push('Financial Modeling Prep API');
}
return apis;
}
}
// Export singleton instance
export const metadataEnricher = new MetadataEnricher();
