import { EventEmitter } from 'events';
import { Logger } from '../utils/logger';
import { metricsCollector, MetricsSummary } from './metrics';
export interface AlertRule {
id: string;
name: string;
description: string;
condition: (metrics: MetricsSummary) => boolean;
severity: 'low' | 'medium' | 'high' | 'critical';
cooldownMinutes: number;
enabled: boolean;
}
export interface Alert {
id: string;
ruleId: string;
name: string;
description: string;
severity: 'low' | 'medium' | 'high' | 'critical';
triggeredAt: Date;
resolvedAt?: Date;
status: 'active' | 'resolved';
metrics: MetricsSummary;
}
export interface AlertChannel {
name: string;
type: 'console' | 'webhook' | 'email';
config: Record<string, any>;
enabled: boolean;
}
export class AlertManager extends EventEmitter {
private rules: Map<string, AlertRule> = new Map();
private activeAlerts: Map<string, Alert> = new Map();
private alertHistory: Alert[] = [];
private channels: AlertChannel[] = [];
private lastAlertTimes: Map<string, Date> = new Map();
private checkInterval: NodeJS.Timeout;
constructor() {
super();
this.setupDefaultRules();
this.setupDefaultChannels();
this.startMonitoring();
}
// Add or update an alert rule
addRule(rule: AlertRule): void {
this.rules.set(rule.id, rule);
Logger.info(`Alert rule added: ${rule.name}`);
}
// Remove an alert rule
removeRule(ruleId: string): void {
this.rules.delete(ruleId);
Logger.info(`Alert rule removed: ${ruleId}`);
}
// Get all rules
getRules(): AlertRule[] {
return Array.from(this.rules.values());
}
// Get active alerts
getActiveAlerts(): Alert[] {
return Array.from(this.activeAlerts.values());
}
// Get alert history
getAlertHistory(limit: number = 100): Alert[] {
return this.alertHistory.slice(-limit);
}
// Add alert channel
addChannel(channel: AlertChannel): void {
this.channels.push(channel);
Logger.info(`Alert channel added: ${channel.name} (${channel.type})`);
}
// Remove alert channel
removeChannel(channelName: string): void {
this.channels = this.channels.filter(c => c.name !== channelName);
Logger.info(`Alert channel removed: ${channelName}`);
}
// Manually trigger alert check
async checkAlerts(): Promise<void> {
const metrics = metricsCollector.getMetricsSummary();
for (const rule of this.rules.values()) {
if (!rule.enabled) continue;
try {
const shouldTrigger = rule.condition(metrics);
const existingAlert = this.activeAlerts.get(rule.id);
const lastAlertTime = this.lastAlertTimes.get(rule.id);
const cooldownExpired = !lastAlertTime ||
(Date.now() - lastAlertTime.getTime()) > (rule.cooldownMinutes * 60 * 1000);
if (shouldTrigger && !existingAlert && cooldownExpired) {
// Trigger new alert
await this.triggerAlert(rule, metrics);
} else if (!shouldTrigger && existingAlert) {
// Resolve existing alert
await this.resolveAlert(rule.id);
}
} catch (error) {
Logger.error(`Error checking alert rule ${rule.name}`, error);
}
}
}
// Trigger an alert
private async triggerAlert(rule: AlertRule, metrics: MetricsSummary): Promise<void> {
const alert: Alert = {
id: `${rule.id}_${Date.now()}`,
ruleId: rule.id,
name: rule.name,
description: rule.description,
severity: rule.severity,
triggeredAt: new Date(),
status: 'active',
metrics
};
this.activeAlerts.set(rule.id, alert);
this.alertHistory.push(alert);
this.lastAlertTimes.set(rule.id, new Date());
Logger.warn(`Alert triggered: ${rule.name}`, {
alertId: alert.id,
severity: rule.severity,
metrics
});
// Send notifications
await this.sendNotifications(alert);
// Emit event
this.emit('alert', alert);
}
// Resolve an alert
private async resolveAlert(ruleId: string): Promise<void> {
const alert = this.activeAlerts.get(ruleId);
if (!alert) return;
alert.resolvedAt = new Date();
alert.status = 'resolved';
this.activeAlerts.delete(ruleId);
Logger.info(`Alert resolved: ${alert.name}`, {
alertId: alert.id,
duration: alert.resolvedAt.getTime() - alert.triggeredAt.getTime()
});
// Send resolution notifications
await this.sendResolutionNotifications(alert);
// Emit event
this.emit('alertResolved', alert);
}
// Send alert notifications through configured channels
private async sendNotifications(alert: Alert): Promise<void> {
const enabledChannels = this.channels.filter(c => c.enabled);
for (const channel of enabledChannels) {
try {
await this.sendNotification(channel, alert);
} catch (error) {
Logger.error(`Failed to send alert through ${channel.name}`, error);
}
}
}
// Send resolution notifications
private async sendResolutionNotifications(alert: Alert): Promise<void> {
const enabledChannels = this.channels.filter(c => c.enabled);
for (const channel of enabledChannels) {
try {
await this.sendResolutionNotification(channel, alert);
} catch (error) {
Logger.error(`Failed to send resolution notification through ${channel.name}`, error);
}
}
}
// Send notification through specific channel
private async sendNotification(channel: AlertChannel, alert: Alert): Promise<void> {
switch (channel.type) {
case 'console':
console.log(`🚨 ALERT: ${alert.name} (${alert.severity.toUpperCase()})`);
console.log(` ${alert.description}`);
console.log(` Triggered: ${alert.triggeredAt.toISOString()}`);
break;
case 'webhook':
if (channel.config.url) {
const payload = {
alert: {
name: alert.name,
description: alert.description,
severity: alert.severity,
triggeredAt: alert.triggeredAt,
metrics: alert.metrics
},
type: 'alert'
};
const response = await fetch(channel.config.url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
});
if (!response.ok) {
throw new Error(`Webhook returned ${response.status}: ${response.statusText}`);
}
}
break;
case 'email':
// Email implementation would go here
Logger.info(`Email alert notification not implemented: ${alert.name}`);
break;
default:
Logger.warn(`Unknown alert channel type: ${channel.type}`);
}
}
// Send resolution notification
private async sendResolutionNotification(channel: AlertChannel, alert: Alert): Promise<void> {
switch (channel.type) {
case 'console':
console.log(`✅ RESOLVED: ${alert.name}`);
console.log(` Duration: ${alert.resolvedAt!.getTime() - alert.triggeredAt.getTime()}ms`);
break;
case 'webhook':
if (channel.config.url) {
const payload = {
alert: {
name: alert.name,
description: alert.description,
severity: alert.severity,
triggeredAt: alert.triggeredAt,
resolvedAt: alert.resolvedAt,
duration: alert.resolvedAt!.getTime() - alert.triggeredAt.getTime()
},
type: 'resolution'
};
await fetch(channel.config.url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
});
}
break;
}
}
// Setup default alert rules
private setupDefaultRules(): void {
// High error rate
this.addRule({
id: 'high_error_rate',
name: 'High Error Rate',
description: 'Error rate exceeds 5%',
condition: (metrics) => metrics.errorRate > 5,
severity: 'high',
cooldownMinutes: 5,
enabled: true
});
// High response time
this.addRule({
id: 'high_response_time',
name: 'High Response Time',
description: 'Average response time exceeds 2 seconds',
condition: (metrics) => metrics.averageResponseTime > 2000,
severity: 'medium',
cooldownMinutes: 5,
enabled: true
});
// High memory usage
this.addRule({
id: 'high_memory_usage',
name: 'High Memory Usage',
description: 'Memory usage exceeds 500MB',
condition: (metrics) => metrics.memoryUsage > 500,
severity: 'medium',
cooldownMinutes: 10,
enabled: true
});
// Cache hit rate too low
this.addRule({
id: 'low_cache_hit_rate',
name: 'Low Cache Hit Rate',
description: 'Cache hit rate below 70%',
condition: (metrics) => metrics.cacheHitRate < 70 && metrics.totalRequests > 100,
severity: 'low',
cooldownMinutes: 15,
enabled: true
});
// Too many tool calls per minute
this.addRule({
id: 'high_tool_call_rate',
name: 'High Tool Call Rate',
description: 'Tool calls per minute exceeds 100',
condition: (metrics) => metrics.toolCallsPerMinute > 100,
severity: 'medium',
cooldownMinutes: 5,
enabled: true
});
}
// Setup default alert channels
private setupDefaultChannels(): void {
// Console channel (always enabled for development)
this.addChannel({
name: 'console',
type: 'console',
config: {},
enabled: true
});
// Webhook channel (disabled by default)
if (process.env.ALERT_WEBHOOK_URL) {
this.addChannel({
name: 'webhook',
type: 'webhook',
config: {
url: process.env.ALERT_WEBHOOK_URL
},
enabled: true
});
}
}
// Start monitoring
private startMonitoring(): void {
// Check alerts every 30 seconds
this.checkInterval = setInterval(() => {
this.checkAlerts().catch(error => {
Logger.error('Error during alert check', error);
});
}, 30000);
Logger.info('Alert monitoring started');
}
// Stop monitoring
stopMonitoring(): void {
if (this.checkInterval) {
clearInterval(this.checkInterval);
Logger.info('Alert monitoring stopped');
}
}
// Get alert statistics
getStatistics(): {
totalRules: number;
enabledRules: number;
activeAlerts: number;
totalAlerts: number;
alertsByseverity: Record<string, number>;
} {
const alertsByServerity: Record<string, number> = {
low: 0,
medium: 0,
high: 0,
critical: 0
};
this.alertHistory.forEach(alert => {
alertsByServerity[alert.severity]++;
});
return {
totalRules: this.rules.size,
enabledRules: Array.from(this.rules.values()).filter(r => r.enabled).length,
activeAlerts: this.activeAlerts.size,
totalAlerts: this.alertHistory.length,
alertsByServerity
};
}
}
// Singleton instance
export const alertManager = new AlertManager();
// Graceful shutdown
process.on('SIGTERM', () => {
alertManager.stopMonitoring();
});
process.on('SIGINT', () => {
alertManager.stopMonitoring();
});
