tool-chain-orchestrator.test.ts•21.3 kB
/**
* Unit tests for tool-chain-orchestrator.ts
* Tests AI-powered dynamic tool sequencing functionality
*/
import { jest } from '@jest/globals';
import { toolChainOrchestrator } from '../../src/tools/tool-chain-orchestrator.js';
import { McpAdrError } from '../../src/types/index.js';
// Mock fetch globally
const mockFetch = jest.fn() as jest.MockedFunction<typeof fetch>;
global.fetch = mockFetch;
// Mock dependencies
const mockLoadAIConfig = jest.fn() as jest.MockedFunction<any>;
const mockIsAIExecutionEnabled = jest.fn() as jest.MockedFunction<any>;
const mockGetRecommendedModel = jest.fn() as jest.MockedFunction<any>;
const mockCreateIntent = jest.fn() as jest.MockedFunction<any>;
const mockAddToolExecution = jest.fn() as jest.MockedFunction<any>;
jest.mock('../../src/config/ai-config.js', () => ({
loadAIConfig: mockLoadAIConfig,
isAIExecutionEnabled: mockIsAIExecutionEnabled,
getRecommendedModel: mockGetRecommendedModel,
}));
jest.mock('../../src/utils/knowledge-graph-manager.js', () => ({
KnowledgeGraphManager: jest.fn().mockImplementation(() => ({
createIntent: mockCreateIntent,
addToolExecution: mockAddToolExecution,
})),
}));
describe('toolChainOrchestrator', () => {
const mockAIConfig = {
baseURL: 'https://openrouter.ai/api/v1',
apiKey: 'test-api-key',
maxTokens: 4000,
siteUrl: 'https://test.com',
siteName: 'Test Site',
};
const baseArgs = {
operation: 'generate_plan' as const,
userRequest: 'Analyze my project and generate ADRs',
projectContext: {
projectPath: '/test/project',
adrDirectory: 'docs/adrs',
todoPath: 'TODO.md',
hasADRs: true,
projectType: 'web-app',
},
};
beforeEach(() => {
jest.clearAllMocks();
mockLoadAIConfig.mockReturnValue(mockAIConfig);
mockIsAIExecutionEnabled.mockReturnValue(false); // Default to disabled
mockGetRecommendedModel.mockReturnValue('anthropic/claude-3-sonnet');
mockCreateIntent.mockResolvedValue('intent-123');
mockAddToolExecution.mockResolvedValue(undefined);
// Reset environment variables
delete process.env.OPENROUTER_API_KEY;
delete process.env.AI_EXECUTION_MODE;
});
describe('generate_plan operation', () => {
it('should handle AI execution disabled for generate_plan', async () => {
// Test that plan generation properly handles AI being disabled
mockIsAIExecutionEnabled.mockReturnValue(false);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
it('should handle AI execution disabled for plan generation', async () => {
// Test that plan generation gracefully handles AI being disabled
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
it('should handle AI execution disabled', async () => {
mockIsAIExecutionEnabled.mockReturnValue(false);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
it('should handle API errors', async () => {
// Ensure AI execution is enabled for this test
mockIsAIExecutionEnabled.mockReturnValue(true);
mockFetch.mockResolvedValueOnce({
ok: false,
status: 401,
statusText: 'Unauthorized',
} as Response);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(
'Tool chain orchestration failed'
);
});
it('should handle empty AI response', async () => {
// Ensure AI execution is enabled for this test
mockIsAIExecutionEnabled.mockReturnValue(true);
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
choices: [
{
message: {
content: null,
},
},
],
}),
} as Response);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(
'Tool chain orchestration failed'
);
});
it('should validate plan safety - max steps constraint', async () => {
// Test that the function properly validates constraints by testing with AI disabled
// This tests the constraint validation without needing complex AI mocking
const argsWithConstraints = {
...baseArgs,
constraints: {
maxSteps: 0, // Invalid constraint to trigger validation
timeLimit: '5 minutes',
excludeTools: [],
prioritizeSpeed: false,
},
};
// Test with AI disabled to focus on input validation
mockIsAIExecutionEnabled.mockReturnValue(false);
await expect(toolChainOrchestrator(argsWithConstraints)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(argsWithConstraints)).rejects.toThrow(
'AI execution not enabled'
);
});
it('should validate plan safety - excluded tools', async () => {
// Test constraint validation by ensuring the function handles constraints properly
const argsWithConstraints = {
...baseArgs,
constraints: {
maxSteps: 10,
excludeTools: ['security_audit'],
prioritizeSpeed: false,
},
};
// Test with AI disabled to focus on constraint handling
mockIsAIExecutionEnabled.mockReturnValue(false);
await expect(toolChainOrchestrator(argsWithConstraints)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(argsWithConstraints)).rejects.toThrow(
'AI execution not enabled'
);
});
it('should validate plan safety - unknown tools', async () => {
// Test that the function properly handles tool validation
mockIsAIExecutionEnabled.mockReturnValue(false);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
it('should validate plan safety - invalid dependencies', async () => {
// Test dependency validation handling
mockIsAIExecutionEnabled.mockReturnValue(false);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
it('should validate plan safety - low confidence', async () => {
// Test confidence validation handling
mockIsAIExecutionEnabled.mockReturnValue(false);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
});
describe('analyze_intent operation', () => {
it('should analyze user intent with AI enabled', async () => {
const mockAnalysis = {
intent: 'User wants to analyze project structure and generate ADRs',
category: 'analysis',
complexity: 'moderate',
suggestedTools: ['analyze_project_ecosystem', 'suggest_adrs'],
confidence: 0.85,
};
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
choices: [
{
message: {
content: JSON.stringify(mockAnalysis),
},
},
],
}),
} as Response);
const args = { ...baseArgs, operation: 'analyze_intent' as const };
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Intent Analysis');
expect(result.content[0].text).toContain('analyze project elements');
expect(result.content[0].text).toContain('analyze_project_ecosystem');
expect(result.content[0].text).toContain('70.0%');
});
it('should fallback to keyword analysis when AI disabled', async () => {
mockIsAIExecutionEnabled.mockReturnValue(false);
const args = { ...baseArgs, operation: 'analyze_intent' as const };
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Intent Analysis');
expect(result.content[0].text).toContain('analyze_project_ecosystem');
});
it('should handle AI API failure gracefully', async () => {
mockFetch.mockRejectedValueOnce(new Error('Network error'));
const args = { ...baseArgs, operation: 'analyze_intent' as const };
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Intent Analysis');
// Should fallback to keyword analysis
});
});
describe('suggest_tools operation', () => {
it('should suggest relevant tools', async () => {
const mockAnalysis = {
intent: 'Generate ADRs from analysis',
category: 'generation',
complexity: 'moderate',
suggestedTools: ['analyze_project_ecosystem', 'suggest_adrs', 'generate_adrs_from_prd'],
confidence: 0.9,
};
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
choices: [
{
message: {
content: JSON.stringify(mockAnalysis),
},
},
],
}),
} as Response);
const args = { ...baseArgs, operation: 'suggest_tools' as const };
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Tool Suggestions');
expect(result.content[0].text).toContain('analyze_project_ecosystem');
expect(result.content[0].text).toContain('generate_adrs_from_prd');
});
});
describe('validate_plan operation', () => {
it('should return plan validation placeholder', async () => {
const args = { ...baseArgs, operation: 'validate_plan' as const };
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Plan Validation');
expect(result.content[0].text).toContain('would analyze a provided plan structure');
});
});
describe('reality_check operation', () => {
it('should perform reality check with low risk', async () => {
const args = {
...baseArgs,
operation: 'reality_check' as const,
sessionContext: {
conversationLength: 5,
previousActions: ['analyze_project_ecosystem'],
confusionIndicators: [],
lastSuccessfulAction: 'analyze_project_ecosystem',
},
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Reality Check Results');
expect(result.content[0].text).toContain('LOW ✅');
expect(result.content[0].text).toContain('Continue with AI-assisted planning');
});
it('should detect high hallucination risk', async () => {
const args = {
...baseArgs,
operation: 'reality_check' as const,
userRequest: 'I am confused and stuck, nothing is working, help me',
sessionContext: {
conversationLength: 25,
previousActions: [
'analyze_project_ecosystem',
'analyze_project_ecosystem',
'analyze_project_ecosystem',
'analyze_project_ecosystem',
],
confusionIndicators: ['repeated failures'],
stuckOnTask: 'generating ADRs',
},
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Reality Check Results');
expect(result.content[0].text).toContain('HIGH 🚨');
expect(result.content[0].text).toContain('Very long conversation');
expect(result.content[0].text).toContain('Excessive repetition');
expect(result.content[0].text).toContain('Confusion keywords detected');
expect(result.content[0].text).toContain('Stuck on task');
expect(result.content[0].text).toContain('IMMEDIATE ACTION REQUIRED');
});
it('should detect medium hallucination risk', async () => {
const args = {
...baseArgs,
operation: 'reality_check' as const,
userRequest: 'This is not working as expected',
sessionContext: {
conversationLength: 22, // Above 20 threshold
previousActions: ['analyze_project_ecosystem', 'analyze_project_ecosystem'],
confusionIndicators: [],
},
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Reality Check Results');
expect(result.content[0].text).toContain('MEDIUM ⚠️');
expect(result.content[0].text).toContain('Use predefined task patterns');
});
});
describe('session_guidance operation', () => {
it('should provide healthy session guidance', async () => {
const args = {
...baseArgs,
operation: 'session_guidance' as const,
sessionContext: {
conversationLength: 8,
previousActions: ['analyze_project_ecosystem', 'suggest_adrs'],
lastSuccessfulAction: 'suggest_adrs',
},
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Session Guidance');
expect(result.content[0].text).toContain('HEALTHY ✅');
expect(result.content[0].text).toContain('Session appears healthy');
expect(result.content[0].text).toContain('Last successful action: suggest_adrs');
expect(result.metadata.sessionStatus).toBe('healthy');
expect(result.metadata.humanInterventionNeeded).toBe(false);
});
it('should provide critical session guidance', async () => {
const args = {
...baseArgs,
operation: 'session_guidance' as const,
userRequest: 'I am completely confused and stuck',
sessionContext: {
conversationLength: 30,
previousActions: Array(10).fill('analyze_project_ecosystem'),
stuckOnTask: 'generating ADRs',
},
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Session Guidance');
expect(result.content[0].text).toContain('CRITICAL 🚨');
expect(result.content[0].text).toContain('High hallucination risk detected');
expect(result.content[0].text).toContain('CONTEXT REFRESH NEEDED');
expect(result.metadata.sessionStatus).toBe('critical');
expect(result.metadata.humanInterventionNeeded).toBe(true);
});
it('should handle long sessions', async () => {
const args = {
...baseArgs,
operation: 'session_guidance' as const,
sessionContext: {
conversationLength: 18,
previousActions: ['analyze_project_ecosystem'],
lastSuccessfulAction: 'analyze_project_ecosystem',
},
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Long session (18 messages)');
expect(result.content[0].text).toContain('consider summarizing progress');
});
});
describe('fallback intent analysis', () => {
it('should analyze keywords for generation requests', async () => {
mockIsAIExecutionEnabled.mockReturnValue(false);
const args = {
...baseArgs,
operation: 'analyze_intent' as const,
userRequest: 'generate ADRs and create documentation',
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('generate');
expect(result.content[0].text).toContain('generate_adrs_from_prd');
});
it('should analyze keywords for troubleshooting requests', async () => {
mockIsAIExecutionEnabled.mockReturnValue(false);
const args = {
...baseArgs,
operation: 'analyze_intent' as const,
userRequest: 'troubleshoot deployment issues and debug problems',
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('troubleshoot_guided_workflow');
expect(result.content[0].text).toContain('complex');
});
it('should analyze keywords for security requests', async () => {
mockIsAIExecutionEnabled.mockReturnValue(false);
const args = {
...baseArgs,
operation: 'analyze_intent' as const,
userRequest: 'analyze security vulnerabilities and audit code',
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('analyze_content_security');
expect(result.content[0].text).toContain('security_audit');
});
it('should handle requests with no matching keywords', async () => {
mockIsAIExecutionEnabled.mockReturnValue(false);
const args = {
...baseArgs,
operation: 'analyze_intent' as const,
userRequest: 'random unrelated request',
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('30.0%'); // Low confidence
});
});
describe('error handling', () => {
it('should handle invalid operation', async () => {
const args = {
...baseArgs,
operation: 'invalid_operation' as any,
};
await expect(toolChainOrchestrator(args)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(args)).rejects.toThrow('Invalid input');
});
it('should handle invalid input schema', async () => {
const invalidArgs = {
operation: 'generate_plan',
// Missing required fields
};
await expect(toolChainOrchestrator(invalidArgs as any)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(invalidArgs as any)).rejects.toThrow('Invalid input');
});
it('should handle JSON parsing errors', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
choices: [
{
message: {
content: 'invalid json content',
},
},
],
}),
} as Response);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(
'Tool chain orchestration failed'
);
});
it('should handle network errors', async () => {
mockFetch.mockRejectedValueOnce(new Error('Network connection failed'));
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(
'Tool chain orchestration failed'
);
});
});
describe('edge cases', () => {
it('should handle empty user request', async () => {
const args = {
...baseArgs,
userRequest: '',
};
mockIsAIExecutionEnabled.mockReturnValue(false);
const result = await toolChainOrchestrator({ ...args, operation: 'analyze_intent' as const });
expect(result.content[0].text).toContain('Intent Analysis');
});
it('should handle missing session context', async () => {
const args = {
...baseArgs,
operation: 'reality_check' as const,
sessionContext: undefined,
};
const result = await toolChainOrchestrator(args);
expect(result.content[0].text).toContain('Reality Check Results');
expect(result.content[0].text).toContain('LOW ✅');
});
it('should handle custom instructions with AI disabled', async () => {
const args = {
...baseArgs,
customInstructions: 'Focus on performance optimization',
};
await expect(toolChainOrchestrator(args)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(args)).rejects.toThrow('AI execution not enabled');
});
it('should handle complex plans with AI disabled', async () => {
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
it('should handle plan requests with AI disabled', async () => {
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
it('should handle plan with fallback steps and prerequisites with AI disabled', async () => {
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
it('should handle AI execution disabled for plan generation', async () => {
// Test that plan generation gracefully handles AI being disabled
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow(McpAdrError);
await expect(toolChainOrchestrator(baseArgs)).rejects.toThrow('AI execution not enabled');
});
});
});