Umbrella MCP Server

#!/usr/bin/env node const { spawn } = require('child_process'); const readline = require('readline'); class ActualLiveRegressionTest { constructor() { this.results = []; this.mcpProcess = null; this.rl = null; } async runActualLiveTest() { console.log('🔴 ACTUAL LIVE API REGRESSION - REAL API CALLS'); console.log('=============================================='); try { // Start actual MCP server await this.startRealMCPServer(); // Run real API calls via MCP protocol await this.runRealAPITests(); // Generate results table this.generateActualResultsTable(); } catch (error) { console.error('❌ Live regression failed:', error.message); } finally { await this.stopMCPServer(); } } async startRealMCPServer() { console.log('🚀 Starting REAL MCP Server for live API calls...'); this.mcpProcess = spawn('node', ['dist/index.js'], { stdio: ['pipe', 'pipe', 'pipe'], cwd: __dirname, env: { ...process.env, UMBRELLA_API_BASE_URL: 'https://api.umbrellacost.io/api/v1' } }); this.rl = readline.createInterface({ input: this.mcpProcess.stdout, output: process.stdout }); // Wait for server startup await this.waitForServerReady(); console.log('✅ REAL MCP Server started and ready'); } async waitForServerReady() { return new Promise((resolve) => { setTimeout(resolve, 2000); // Give server time to start }); } async runRealAPITests() { console.log('\n📋 Running REAL API Tests...'); // Test cases from requests.txt and goodanswers.txt const realTestCases = [ { id: 'Q1', question: 'show me the list of MSP customers', mcpCall: { jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'api___users_plain_sub_users', arguments: {} } }, expectedFromGoodAnswers: 'Multiple customers found' }, { id: 'Q3', question: 'what is my total AWS cost?', mcpCall: { jsonrpc: '2.0', id: 3, method: 'tools/call', params: { name: 'api___invoices_caui', arguments: { startDate: '2025-08-01', endDate: '2025-08-31', cloud_context: 'aws', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true } } }, expectedFromGoodAnswers: '$136,045.96' }, { id: 'Q13', question: 'what is the last 30 days (per day) amortized cost for Cloudwatch service?', mcpCall: { jsonrpc: '2.0', id: 13, method: 'tools/call', params: { name: 'api___invoices_caui', arguments: { startDate: '2025-07-28', endDate: '2025-08-27', periodGranLevel: 'day', costType: ['cost', 'discount'], isAmortized: true, cloud_context: 'aws', service: 'cloudwatch' } } }, expectedFromGoodAnswers: '$5,501.30 total CloudWatch costs' }, { id: 'BANK_LEUMI', question: 'Show me Bank Leumi costs for last month', mcpCall: { jsonrpc: '2.0', id: 14, method: 'tools/call', params: { name: 'api___invoices_caui', arguments: { userQuery: 'Show me Bank Leumi costs for last month', startDate: '2025-07-01', endDate: '2025-07-31', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'aws' } } }, expectedFromGoodAnswers: 'Near-zero costs (fixed division 139)' }, { id: 'BANK_HAPOALIM', question: 'Show me Bank Hapoalim costs for last month', mcpCall: { jsonrpc: '2.0', id: 15, method: 'tools/call', params: { name: 'api___invoices_caui', arguments: { userQuery: 'Show me Bank Hapoalim costs for last month', startDate: '2025-07-01', endDate: '2025-07-31', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'aws' } } }, expectedFromGoodAnswers: 'No hardcoded logic (dynamic detection)' } ]; // First authenticate console.log('🔐 Authenticating with REAL MSP credentials...'); const authResult = await this.makeRealMCPCall({ jsonrpc: '2.0', id: 0, method: 'tools/call', params: { name: 'authenticate_user', arguments: { username: 'david+allcloud@umbrellacost.com', password: 'B4*zcI7#F7poEC' } } }); if (!authResult.success) { console.log('❌ AUTHENTICATION FAILED - Cannot proceed with live tests'); this.results.push({ id: 'AUTH', question: 'Authentication', expected: 'Successful login', actual: authResult.error, status: 'FAIL' }); return; } console.log('✅ Real authentication successful'); // Run each test case with real API calls for (const testCase of realTestCases) { console.log(`\n🧪 REAL API TEST ${testCase.id}: ${testCase.question}`); console.log(' 📤 Making real MCP call...'); const result = await this.makeRealMCPCall(testCase.mcpCall); let status = 'FAIL'; let actualResponse = 'No response'; if (result.success) { status = 'PASS'; actualResponse = result.data ? JSON.stringify(result.data).substring(0, 100) + '...' : 'Success'; console.log(' ✅ Real API call successful'); } else { console.log(` ❌ Real API call failed: ${result.error}`); actualResponse = result.error; } this.results.push({ id: testCase.id, question: testCase.question, expected: testCase.expectedFromGoodAnswers, actual: actualResponse, status: status, details: result.success ? 'Live API call successful' : result.error }); } } async makeRealMCPCall(mcpRequest) { return new Promise((resolve) => { try { // Send real MCP request via stdin const requestString = JSON.stringify(mcpRequest) + '\n'; this.mcpProcess.stdin.write(requestString); // Set timeout for response const timeout = setTimeout(() => { resolve({ success: false, error: 'API call timeout (no response in 10s)' }); }, 10000); // Listen for real response const responseHandler = (data) => { clearTimeout(timeout); try { const response = JSON.parse(data.toString()); if (response.error) { resolve({ success: false, error: response.error.message || 'API error' }); } else { resolve({ success: true, data: response.result }); } } catch (parseError) { resolve({ success: false, error: 'Invalid JSON response' }); } }; // Wait for actual response this.mcpProcess.stdout.once('data', responseHandler); } catch (error) { resolve({ success: false, error: error.message }); } }); } generateActualResultsTable() { console.log('\n📊 ACTUAL LIVE REGRESSION TEST RESULTS'); console.log('======================================'); console.log('| ID | Status | Question | Expected | Actual Result |'); console.log('|-------------|--------|--------------------------------------------------|--------------------------------|----------------------------|'); this.results.forEach(result => { const statusIcon = result.status === 'PASS' ? '✅' : '❌'; const id = result.id.padEnd(11); const status = `${statusIcon} ${result.status}`.padEnd(8); const question = result.question.length > 48 ? result.question.substring(0, 45) + '...' : result.question.padEnd(48); const expected = result.expected.length > 30 ? result.expected.substring(0, 27) + '...' : result.expected.padEnd(30); const actual = result.actual.length > 26 ? result.actual.substring(0, 23) + '...' : result.actual.padEnd(26); console.log(`| ${id} | ${status} | ${question} | ${expected} | ${actual} |`); }); const total = this.results.length; const passed = this.results.filter(r => r.status === 'PASS').length; const failed = total - passed; console.log('\n📈 LIVE TEST SUMMARY:'); console.log('====================='); console.log(`🧪 Total Tests: ${total}`); console.log(`✅ Passed: ${passed}`); console.log(`❌ Failed: ${failed}`); console.log(`📊 Success Rate: ${Math.round((passed/total) * 100)}%`); if (failed > 0) { console.log('\n❌ FAILED LIVE TESTS:'); console.log('====================='); this.results.filter(r => r.status === 'FAIL').forEach(result => { console.log(`\n${result.id}: ${result.question}`); console.log(`Expected: ${result.expected}`); console.log(`Error: ${result.details}`); }); console.log('\n🚨 HONEST ASSESSMENT: There are REAL issues that need fixing!'); } else { console.log('\n🎯 HONEST ASSESSMENT: All live API tests passed successfully!'); } } async stopMCPServer() { if (this.mcpProcess) { this.mcpProcess.kill('SIGTERM'); console.log('\n🛑 Real MCP Server stopped'); } } } // Run actual live regression test console.log('⚠️ WARNING: This will make REAL API calls to Umbrella Cost!'); console.log('⚠️ This is an HONEST test - failures will be reported accurately.'); const liveTest = new ActualLiveRegressionTest(); liveTest.runActualLiveTest();