geo-metrics.ts•9.51 kB
// GEO (Generative Engine Optimization) Metrics and Monitoring
export interface GEOMetrics {
// Citation success rate - percentage of successful tool calls
citationSuccessRate: number;
// AI traffic conversion rate - percentage of AI agents that successfully use the tool
aiTrafficConversionRate: number;
// Average citation position - where the tool is typically referenced in AI responses
averageCitationPosition: number;
// Link carry rate - percentage of responses that include source links
linkCarryRate: number;
// Query coverage rate - percentage of queries that return relevant results
queryCoverageRate: number;
// Tool call success rate - percentage of successful MCP tool calls
toolCallSuccessRate: number;
// Average response time in milliseconds
averageResponseTime: number;
// Error rate - percentage of failed requests
errorRate: number;
// Categorization accuracy - percentage of correctly categorized news
categorizationAccuracy: number;
// AI agent satisfaction score (1-10)
aiAgentSatisfactionScore: number;
// Usage frequency - number of tool calls per day
usageFrequency: number;
// Query diversity - number of unique query patterns
queryDiversity: number;
}
export interface GEOMonitoringConfig {
trackingEnabled: boolean;
metricsEndpoint: string;
reportingInterval: number; // minutes
alertThresholds: {
errorRate: number;
responseTime: number;
successRate: number;
satisfactionScore: number;
};
dataRetentionDays: number;
}
export interface QueryMetrics {
query: string;
timestamp: Date;
success: boolean;
responseTime: number;
resultCount: number;
categories: string[];
errorMessage?: string;
}
export interface AgentMetrics {
agentId: string;
toolCalls: number;
successfulCalls: number;
averageResponseTime: number;
lastActive: Date;
preferredCategories: string[];
}
export class GEOMetricsCollector {
private metrics: GEOMetrics;
private queryHistory: QueryMetrics[];
private agentHistory: Map<string, AgentMetrics>;
private config: GEOMonitoringConfig;
constructor(config: GEOMonitoringConfig) {
this.config = config;
this.metrics = this.initializeMetrics();
this.queryHistory = [];
this.agentHistory = new Map();
}
private initializeMetrics(): GEOMetrics {
return {
citationSuccessRate: 0,
aiTrafficConversionRate: 0,
averageCitationPosition: 0,
linkCarryRate: 0,
queryCoverageRate: 0,
toolCallSuccessRate: 0,
averageResponseTime: 0,
errorRate: 0,
categorizationAccuracy: 0,
aiAgentSatisfactionScore: 0,
usageFrequency: 0,
queryDiversity: 0
};
}
public trackToolCall(
query: string,
success: boolean,
responseTime: number,
resultCount: number,
categories: string[],
agentId?: string,
errorMessage?: string
): void {
if (!this.config.trackingEnabled) return;
const queryMetric: QueryMetrics = {
query,
timestamp: new Date(),
success,
responseTime,
resultCount,
categories,
errorMessage
};
this.queryHistory.push(queryMetric);
// Update agent metrics
if (agentId) {
this.updateAgentMetrics(agentId, success, responseTime, categories);
}
// Update overall metrics
this.updateOverallMetrics();
}
private updateAgentMetrics(
agentId: string,
success: boolean,
responseTime: number,
categories: string[]
): void {
const existing = this.agentHistory.get(agentId) || {
agentId,
toolCalls: 0,
successfulCalls: 0,
averageResponseTime: 0,
lastActive: new Date(),
preferredCategories: []
};
existing.toolCalls++;
if (success) {
existing.successfulCalls++;
}
existing.averageResponseTime =
(existing.averageResponseTime * (existing.toolCalls - 1) + responseTime) / existing.toolCalls;
existing.lastActive = new Date();
// Update preferred categories
categories.forEach(cat => {
if (!existing.preferredCategories.includes(cat)) {
existing.preferredCategories.push(cat);
}
});
this.agentHistory.set(agentId, existing);
}
private updateOverallMetrics(): void {
const recentQueries = this.getRecentQueries(24); // Last 24 hours
this.metrics.toolCallSuccessRate = this.calculateSuccessRate(recentQueries);
this.metrics.averageResponseTime = this.calculateAverageResponseTime(recentQueries);
this.metrics.errorRate = 1 - this.metrics.toolCallSuccessRate;
this.metrics.queryCoverageRate = this.calculateQueryCoverageRate(recentQueries);
this.metrics.queryDiversity = this.calculateQueryDiversity(recentQueries);
this.metrics.usageFrequency = recentQueries.length;
this.metrics.categorizationAccuracy = this.calculateCategorizationAccuracy(recentQueries);
this.metrics.linkCarryRate = this.calculateLinkCarryRate(recentQueries);
this.metrics.aiTrafficConversionRate = this.calculateAITrafficConversionRate();
}
private getRecentQueries(hours: number): QueryMetrics[] {
const cutoff = new Date(Date.now() - hours * 60 * 60 * 1000);
return this.queryHistory.filter(q => q.timestamp >= cutoff);
}
private calculateSuccessRate(queries: QueryMetrics[]): number {
if (queries.length === 0) return 0;
const successful = queries.filter(q => q.success).length;
return successful / queries.length;
}
private calculateAverageResponseTime(queries: QueryMetrics[]): number {
if (queries.length === 0) return 0;
const totalTime = queries.reduce((sum, q) => sum + q.responseTime, 0);
return totalTime / queries.length;
}
private calculateQueryCoverageRate(queries: QueryMetrics[]): number {
if (queries.length === 0) return 0;
const queriesWithResults = queries.filter(q => q.resultCount > 0).length;
return queriesWithResults / queries.length;
}
private calculateQueryDiversity(queries: QueryMetrics[]): number {
const uniqueQueries = new Set(queries.map(q => q.query.toLowerCase().trim()));
return uniqueQueries.size;
}
private calculateCategorizationAccuracy(queries: QueryMetrics[]): number {
// This would need to be implemented based on feedback or validation
// For now, return a placeholder value
return 0.85; // 85% accuracy placeholder
}
private calculateLinkCarryRate(queries: QueryMetrics[]): number {
// This would need to be tracked based on how AI agents use the results
// For now, return a placeholder value
return 0.75; // 75% link carry rate placeholder
}
private calculateAITrafficConversionRate(): number {
const totalAgents = this.agentHistory.size;
const activeAgents = Array.from(this.agentHistory.values())
.filter(agent => {
const hoursSinceLastActive = (Date.now() - agent.lastActive.getTime()) / (1000 * 60 * 60);
return hoursSinceLastActive < 24;
}).length;
return totalAgents > 0 ? activeAgents / totalAgents : 0;
}
public getMetrics(): GEOMetrics {
return { ...this.metrics };
}
public getAgentMetrics(agentId: string): AgentMetrics | undefined {
return this.agentHistory.get(agentId);
}
public getAllAgentMetrics(): AgentMetrics[] {
return Array.from(this.agentHistory.values());
}
public generateReport(): string {
const metrics = this.getMetrics();
const agents = this.getAllAgentMetrics();
return `
# GEO Metrics Report - ${new Date().toISOString()}
## Overall Performance
- Tool Call Success Rate: ${(metrics.toolCallSuccessRate * 100).toFixed(2)}%
- Average Response Time: ${metrics.averageResponseTime.toFixed(2)}ms
- Error Rate: ${(metrics.errorRate * 100).toFixed(2)}%
- Query Coverage Rate: ${(metrics.queryCoverageRate * 100).toFixed(2)}%
## Usage Statistics
- Daily Usage Frequency: ${metrics.usageFrequency} calls
- Query Diversity: ${metrics.queryDiversity} unique queries
- Active AI Agents: ${agents.length}
- AI Traffic Conversion Rate: ${(metrics.aiTrafficConversionRate * 100).toFixed(2)}%
## Quality Metrics
- Categorization Accuracy: ${(metrics.categorizationAccuracy * 100).toFixed(2)}%
- Link Carry Rate: ${(metrics.linkCarryRate * 100).toFixed(2)}%
- AI Agent Satisfaction Score: ${metrics.aiAgentSatisfactionScore.toFixed(2)}/10
## Top Agent Activity
${agents
.sort((a, b) => b.toolCalls - a.toolCalls)
.slice(0, 5)
.map(agent => `- ${agent.agentId}: ${agent.toolCalls} calls, ${(agent.averageResponseTime).toFixed(2)}ms avg`)
.join('\n')}
`.trim();
}
public shouldAlert(): boolean {
const metrics = this.getMetrics();
const thresholds = this.config.alertThresholds;
return (
metrics.errorRate > thresholds.errorRate ||
metrics.averageResponseTime > thresholds.responseTime ||
metrics.toolCallSuccessRate < thresholds.successRate ||
metrics.aiAgentSatisfactionScore < thresholds.satisfactionScore
);
}
public cleanup(): void {
const cutoff = new Date(Date.now() - this.config.dataRetentionDays * 24 * 60 * 60 * 1000);
this.queryHistory = this.queryHistory.filter(q => q.timestamp >= cutoff);
// Clean up inactive agents
for (const [agentId, agent] of this.agentHistory.entries()) {
const hoursSinceLastActive = (Date.now() - agent.lastActive.getTime()) / (1000 * 60 * 60);
if (hoursSinceLastActive > 24 * 7) { // 7 days
this.agentHistory.delete(agentId);
}
}
}
}