ThoughtMCP

/** * Tests for BiasLearningSystem * * Tests the learning system that improves bias detection accuracy through * user feedback integration, adaptive pattern recognition, and personalized * sensitivity adjustment. */ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { BiasLearningSystem } from "../../../bias/bias-learning-system"; import type { BiasFeedback, DetectedBias } from "../../../bias/types"; import { BiasType } from "../../../bias/types"; import { seededRandom, TIME } from "../../utils/test-helpers"; // Test fixtures and helpers function createMockDetectedBias(overrides?: Partial<DetectedBias>): DetectedBias { return { type: BiasType.CONFIRMATION, severity: 0.7, confidence: 0.75, evidence: ["Only supporting evidence considered"], location: { stepIndex: 0, reasoning: "Test reasoning", }, explanation: "Test explanation", detectedAt: new Date(), ...overrides, }; } function createMockFeedback(overrides?: Partial<BiasFeedback>): BiasFeedback { return { detectedBias: createMockDetectedBias(), correct: true, userId: "test-user", timestamp: new Date(), ...overrides, }; } // Removed unused createMockReasoningChain function describe("BiasLearningSystem", () => { let learningSystem: BiasLearningSystem; beforeEach(() => { learningSystem = new BiasLearningSystem(); seededRandom.reset(); }); afterEach(() => { // Cleanup }); describe("User Feedback Integration", () => { it("should record user feedback on bias detections", () => { const feedback = createMockFeedback({ correct: true }); learningSystem.integrateFeedback(feedback); const metrics = learningSystem.getAccuracyMetrics(); expect(metrics.truePositives).toBe(1); }); it("should store and retrieve feedback by user ID", () => { const feedback1 = createMockFeedback({ userId: "user1", correct: true }); const feedback2 = createMockFeedback({ userId: "user2", correct: false }); learningSystem.integrateFeedback(feedback1); learningSystem.integrateFeedback(feedback2); const sensitivity1 = learningSystem.getUserSensitivity("user1", BiasType.CONFIRMATION); const sensitivity2 = learningSystem.getUserSensitivity("user2", BiasType.CONFIRMATION); expect(sensitivity1).toBeGreaterThanOrEqual(0); expect(sensitivity2).toBeGreaterThanOrEqual(0); }); it("should aggregate feedback by bias type", () => { const feedback1 = createMockFeedback({ detectedBias: createMockDetectedBias({ type: BiasType.CONFIRMATION }), correct: true, }); const feedback2 = createMockFeedback({ detectedBias: createMockDetectedBias({ type: BiasType.ANCHORING }), correct: true, }); learningSystem.integrateFeedback(feedback1); learningSystem.integrateFeedback(feedback2); const confirmationMetrics = learningSystem.getAccuracyMetrics(BiasType.CONFIRMATION); const anchoringMetrics = learningSystem.getAccuracyMetrics(BiasType.ANCHORING); expect(confirmationMetrics.truePositives).toBe(1); expect(anchoringMetrics.truePositives).toBe(1); }); it("should influence detection thresholds based on feedback", () => { const correctFeedback = createMockFeedback({ correct: true }); learningSystem.integrateFeedback(correctFeedback); learningSystem.integrateFeedback(correctFeedback); learningSystem.integrateFeedback(correctFeedback); // After multiple correct feedbacks, system should have learned const metrics = learningSystem.getAccuracyMetrics(); expect(metrics.truePositives).toBe(3); expect(metrics.precision).toBe(1.0); }); it("should handle invalid feedback data gracefully", () => { const invalidFeedback = { detectedBias: null as any, correct: true, userId: "test", timestamp: new Date(), }; expect(() => learningSystem.integrateFeedback(invalidFeedback)).toThrow(); }); it("should handle concurrent feedback from multiple users", () => { const users = ["user1", "user2", "user3"]; for (const userId of users) { const feedback = createMockFeedback({ userId, correct: true }); learningSystem.integrateFeedback(feedback); } const metrics = learningSystem.getLearningMetrics(); expect(metrics.userCount).toBe(0); // Users are created on sensitivity adjustment expect(metrics.totalFeedback).toBe(3); }); }); describe("Adaptive Pattern Recognition", () => { it("should learn patterns from feedback data", () => { const feedbackItems = [ createMockFeedback({ correct: false }), createMockFeedback({ correct: false }), createMockFeedback({ correct: false }), ]; const pattern = learningSystem.learnNewPattern(feedbackItems); expect(pattern).not.toBeNull(); if (pattern) { expect(pattern.frequency).toBeGreaterThanOrEqual(2); } }); it("should return null when learning pattern with insufficient data (<3 items)", () => { const feedbackItems = [ createMockFeedback({ correct: false }), createMockFeedback({ correct: false }), ]; const pattern = learningSystem.learnNewPattern(feedbackItems); expect(pattern).toBeNull(); }); it("should adjust pattern weights based on accuracy", () => { learningSystem.updatePatternWeights(BiasType.CONFIRMATION, 0.9); // Weight should be adjusted based on high accuracy // We can't directly access weights, but we can verify no errors expect(() => learningSystem.updatePatternWeights(BiasType.CONFIRMATION, 0.9)).not.toThrow(); }); it("should increase pattern weight for high accuracy (>0.8)", () => { // High accuracy should increase weight learningSystem.updatePatternWeights(BiasType.CONFIRMATION, 0.85); expect(() => learningSystem.updatePatternWeights(BiasType.CONFIRMATION, 0.85)).not.toThrow(); }); it("should decrease pattern weight for low accuracy (<0.6)", () => { // Low accuracy should decrease weight learningSystem.updatePatternWeights(BiasType.ANCHORING, 0.5); expect(() => learningSystem.updatePatternWeights(BiasType.ANCHORING, 0.5)).not.toThrow(); }); it("should discover new patterns from false negatives", () => { const falseNegatives = [ createMockFeedback({ correct: false }), createMockFeedback({ correct: false }), createMockFeedback({ correct: false }), ]; const pattern = learningSystem.learnNewPattern(falseNegatives); expect(pattern).not.toBeNull(); }); it("should prune patterns that cause false positives", () => { // Add false positive feedback for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback(createMockFeedback({ correct: false })); } learningSystem.pruneIneffectivePatterns(); // System should have adjusted weights const metrics = learningSystem.getAccuracyMetrics(); expect(metrics.falsePositives).toBe(5); }); it("should track pattern effectiveness over time", () => { // Add feedback over time for (let i = 0; i < 10; i++) { learningSystem.integrateFeedback(createMockFeedback({ correct: i % 2 === 0 })); } const metrics = learningSystem.getAccuracyMetrics(); expect(metrics.truePositives).toBe(5); expect(metrics.falsePositives).toBe(5); }); }); describe("Detection Accuracy Improvement", () => { it("should measure baseline accuracy", () => { const metrics = learningSystem.getAccuracyMetrics(); expect(metrics.truePositives).toBe(0); expect(metrics.falsePositives).toBe(0); expect(metrics.precision).toBe(0); }); it("should track accuracy improvement over time with feedback", () => { // Add early feedback (lower accuracy) - 8 days ago // Note: Using Date.now() here because getImprovementRate compares against current time const now = Date.now(); learningSystem.integrateFeedback( createMockFeedback({ correct: true, timestamp: new Date(now - 8 * TIME.DAY), }) ); learningSystem.integrateFeedback( createMockFeedback({ correct: false, timestamp: new Date(now - 8 * TIME.DAY), }) ); // Add recent feedback (higher accuracy) learningSystem.integrateFeedback( createMockFeedback({ correct: true, timestamp: new Date(now) }) ); learningSystem.integrateFeedback( createMockFeedback({ correct: true, timestamp: new Date(now) }) ); const improvement = learningSystem.getImprovementRate("week"); expect(improvement).toBeGreaterThanOrEqual(0); }); it("should track accuracy per bias type", () => { learningSystem.integrateFeedback( createMockFeedback({ detectedBias: createMockDetectedBias({ type: BiasType.CONFIRMATION }), correct: true, }) ); learningSystem.integrateFeedback( createMockFeedback({ detectedBias: createMockDetectedBias({ type: BiasType.ANCHORING }), correct: false, }) ); const confirmationMetrics = learningSystem.getAccuracyMetrics(BiasType.CONFIRMATION); const anchoringMetrics = learningSystem.getAccuracyMetrics(BiasType.ANCHORING); expect(confirmationMetrics.precision).toBe(1.0); expect(anchoringMetrics.precision).toBe(0.0); }); it("should achieve >70% overall detection rate", () => { // Add feedback with >70% accuracy for (let i = 0; i < 10; i++) { learningSystem.integrateFeedback(createMockFeedback({ correct: i < 8 })); } const metrics = learningSystem.getAccuracyMetrics(); expect(metrics.precision).toBeGreaterThanOrEqual(0.7); }); it("should calculate precision, recall, and F1 score", () => { learningSystem.integrateFeedback(createMockFeedback({ correct: true })); learningSystem.integrateFeedback(createMockFeedback({ correct: true })); learningSystem.integrateFeedback(createMockFeedback({ correct: false })); const metrics = learningSystem.getAccuracyMetrics(); expect(metrics.precision).toBeCloseTo(0.667, 2); expect(metrics.recall).toBe(1.0); // No false negatives tracked expect(metrics.f1Score).toBeGreaterThan(0); }); }); describe("Personalized Sensitivity Adjustment", () => { it("should store user-specific sensitivity preferences", () => { learningSystem.adjustSensitivity("user1", BiasType.CONFIRMATION, 0.2); const sensitivity = learningSystem.getUserSensitivity("user1", BiasType.CONFIRMATION); expect(sensitivity).toBe(0.7); // 0.5 default + 0.2 }); it("should automatically tune sensitivity based on feedback", () => { // Add multiple correct feedbacks for a user for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback(createMockFeedback({ userId: "user1", correct: true })); } // Sensitivity should be adjusted automatically const sensitivity = learningSystem.getUserSensitivity("user1", BiasType.CONFIRMATION); expect(sensitivity).toBeGreaterThanOrEqual(0); }); it("should adjust sensitivity per bias type", () => { learningSystem.adjustSensitivity("user1", BiasType.CONFIRMATION, 0.1); learningSystem.adjustSensitivity("user1", BiasType.ANCHORING, -0.1); const confirmationSensitivity = learningSystem.getUserSensitivity( "user1", BiasType.CONFIRMATION ); const anchoringSensitivity = learningSystem.getUserSensitivity("user1", BiasType.ANCHORING); expect(confirmationSensitivity).toBeGreaterThan(anchoringSensitivity); }); it("should persist sensitivity across sessions", () => { learningSystem.adjustSensitivity("user1", BiasType.CONFIRMATION, 0.3); const sensitivity1 = learningSystem.getUserSensitivity("user1", BiasType.CONFIRMATION); const sensitivity2 = learningSystem.getUserSensitivity("user1", BiasType.CONFIRMATION); expect(sensitivity1).toBe(sensitivity2); }); it("should enforce sensitivity bounds [0, 1]", () => { learningSystem.adjustSensitivity("user1", BiasType.CONFIRMATION, 2.0); const high = learningSystem.getUserSensitivity("user1", BiasType.CONFIRMATION); expect(high).toBeLessThanOrEqual(1.0); learningSystem.adjustSensitivity("user1", BiasType.CONFIRMATION, -5.0); const low = learningSystem.getUserSensitivity("user1", BiasType.CONFIRMATION); expect(low).toBeGreaterThanOrEqual(0.0); }); }); describe("Learning Effectiveness", () => { it("should measure learning rate", () => { for (let i = 0; i < 20; i++) { learningSystem.integrateFeedback(createMockFeedback({ correct: i > 5 })); } const improvement = learningSystem.getImprovementRate("all"); expect(improvement).toBeGreaterThanOrEqual(0); }); it("should return 0 improvement rate when insufficient data (<10 feedback items)", () => { // Add only 5 feedback items for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback(createMockFeedback({ correct: true })); } const improvement = learningSystem.getImprovementRate("all"); expect(improvement).toBe(0); }); it("should calculate improvement rate for day period", () => { // Note: Using Date.now() here because getImprovementRate compares against current time const now = Date.now(); // Add feedback from yesterday (25 hours ago) for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: false, timestamp: new Date(now - 25 * TIME.HOUR), }) ); } // Add recent feedback (today) for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: true, timestamp: new Date(now) }) ); } const improvement = learningSystem.getImprovementRate("day"); expect(improvement).toBeGreaterThan(0); }); it("should calculate improvement rate for week period", () => { // Note: Using Date.now() here because getImprovementRate compares against current time const now = Date.now(); // Add feedback from 8 days ago for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: false, timestamp: new Date(now - 8 * TIME.DAY), }) ); } // Add recent feedback (this week) for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: true, timestamp: new Date(now) }) ); } const improvement = learningSystem.getImprovementRate("week"); expect(improvement).toBeGreaterThan(0); }); it("should calculate improvement rate for month period", () => { // Note: Using Date.now() here because getImprovementRate compares against current time const now = Date.now(); // Add feedback from 31 days ago for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: false, timestamp: new Date(now - 31 * TIME.DAY), }) ); } // Add recent feedback (this month) for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: true, timestamp: new Date(now) }) ); } const improvement = learningSystem.getImprovementRate("month"); expect(improvement).toBeGreaterThan(0); }); it("should converge to optimal thresholds", () => { // Simulate learning over time with deterministic random for (let i = 0; i < 50; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: seededRandom.next() > 0.3 }) ); } const learningMetrics = learningSystem.getLearningMetrics(); expect(learningMetrics.totalFeedback).toBeGreaterThan(0); }); it("should calculate learning metrics with baseline accuracy", () => { // First, establish baseline by getting initial metrics const initialMetrics = learningSystem.getLearningMetrics(); expect(initialMetrics.totalFeedback).toBe(0); expect(initialMetrics.accuracyImprovement).toBe(0); // Add feedback to improve accuracy for (let i = 0; i < 20; i++) { learningSystem.integrateFeedback(createMockFeedback({ correct: true })); } const improvedMetrics = learningSystem.getLearningMetrics(); expect(improvedMetrics.totalFeedback).toBe(20); expect(improvedMetrics.accuracyImprovement).toBeGreaterThanOrEqual(0); }); it("should adapt to user preferences", () => { learningSystem.adjustSensitivity("user1", BiasType.CONFIRMATION, 0.2); learningSystem.adjustSensitivity("user2", BiasType.CONFIRMATION, -0.2); const sensitivity1 = learningSystem.getUserSensitivity("user1", BiasType.CONFIRMATION); const sensitivity2 = learningSystem.getUserSensitivity("user2", BiasType.CONFIRMATION); expect(sensitivity1).toBeGreaterThan(sensitivity2); }); it("should track improvement metrics (false positive/negative reduction)", () => { // Note: Using Date.now() here because getImprovementRate compares against current time const now = Date.now(); // Early period with more false positives (8 days ago) for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: false, timestamp: new Date(now - 8 * TIME.DAY), }) ); } // Recent period with fewer false positives for (let i = 0; i < 5; i++) { learningSystem.integrateFeedback( createMockFeedback({ correct: true, timestamp: new Date(now) }) ); } const improvement = learningSystem.getImprovementRate("week"); expect(improvement).toBeGreaterThan(0); }); it("should complete feedback integration in <10ms", () => { const feedback = createMockFeedback(); const start = performance.now(); learningSystem.integrateFeedback(feedback); const duration = performance.now() - start; expect(duration).toBeLessThan(10); }); }); });