Umbrella MCP Server

#!/usr/bin/env node const { spawn } = require('child_process'); async function finalCompatibilityTest() { console.log('📊 FINAL COMPATIBILITY TEST - ALL 13 QUESTIONS'); console.log('=' .repeat(60)); console.log('Testing against goodanswers.txt baseline'); console.log(''); const server = spawn('node', ['dist/index.js'], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, UMBRELLA_API_BASE_URL: 'https://api-front.umbrellacost.io/api/v1' } }); const responses = []; let requestId = 0; server.stdout.on('data', (data) => { const lines = data.toString().split('\n').filter(line => line.trim()); for (const line of lines) { try { const parsed = JSON.parse(line); responses.push(parsed); } catch (e) {} } }); server.stderr.on('data', () => {}); // Suppress logs await new Promise(resolve => setTimeout(resolve, 2000)); // Authenticate console.log('🔐 Authenticating...\n'); server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'authenticate_user', arguments: { username: 'david+saola@umbrellacost.com', password: 'Dsamsung1!' } } }) + '\n'); await new Promise(resolve => setTimeout(resolve, 5000)); const questions = [ { id: 1, name: "MSP customers", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___customer_list', arguments: {} } }, expected: { description: "Multiple MSP customers", validation: "response" } }, { id: 2, name: "Total cost", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___invoices_caui', arguments: { startDate: '2025-08-01', endDate: '2025-08-27', periodGranLevel: 'month', costType: ['cost'], isAmortized: true, cloud_context: 'aws', accountId: '932213950603' } } }, expected: { cost: 136045.96, validation: "cost" } }, { id: 3, name: "AWS total cost", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___invoices_caui', arguments: { startDate: '2025-08-01', endDate: '2025-08-27', periodGranLevel: 'month', costType: ['cost'], isAmortized: true, cloud_context: 'aws', accountId: '932213950603' } } }, expected: { cost: 136045.96, validation: "cost" } }, { id: 4, name: "GCP total cost", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___invoices_caui', arguments: { startDate: '2025-08-01', endDate: '2025-08-27', periodGranLevel: 'month', costType: ['cost'], isAmortized: true, cloud_context: 'gcp', accountId: '932213950603' } } }, expected: { cost: 0.00, validation: "cost" } }, { id: 5, name: "Azure total cost", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___invoices_caui', arguments: { startDate: '2025-08-01', endDate: '2025-08-27', periodGranLevel: 'month', costType: ['cost'], isAmortized: true, cloud_context: 'azure', accountId: '932213950603' } } }, expected: { cost: 0.00, validation: "cost" } }, { id: 6, name: "Cost per month (2 months)", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___invoices_caui', arguments: { startDate: '2025-07-01', endDate: '2025-08-27', periodGranLevel: 'month', costType: ['cost'], isAmortized: true, cloud_context: 'aws', accountId: '932213950603' } } }, expected: { cost: 320162.54, validation: "cost" } }, { id: 7, name: "8-month AWS amortized", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___invoices_caui', arguments: { startDate: '2025-01-01', endDate: '2025-08-27', periodGranLevel: 'month', costType: ['cost'], isAmortized: true, cloud_context: 'aws', accountId: '932213950603' } } }, expected: { cost: 1099357.88, validation: "cost" } }, { id: 8, name: "All Azure accounts", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___invoices_caui', arguments: { startDate: '2025-08-01', endDate: '2025-08-27', periodGranLevel: 'month', costType: ['cost'], isAmortized: true, cloud_context: 'azure', accountId: 'all' } } }, expected: { cost: 0.00, validation: "cost" } }, { id: 9, name: "Available accounts", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___users', arguments: {} } }, expected: { description: "20 accounts", validation: "response" } }, { id: 10, name: "AWS recommendations", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___recommendations_report', arguments: { accountId: '932213950603' } } }, expected: { cost: 47239.68, validation: "cost" } }, { id: 11, name: "Savings categories", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___recommendations_report', arguments: { accountId: '932213950603' } } }, expected: { cost: 47239.68, validation: "cost" } }, { id: 12, name: "AWS anomalies", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___anomaly_list', arguments: { startDate: '2025-08-17', endDate: '2025-08-27', cloud_context: 'aws', accountId: '932213950603' } } }, expected: { cost: 127.65, validation: "cost" } }, { id: 13, name: "CloudWatch 30-day", request: { jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'api___invoices_caui', arguments: { startDate: '2025-07-28', endDate: '2025-08-27', periodGranLevel: 'day', costType: ['cost'], isAmortized: true, cloud_context: 'aws', accountId: '932213950603', service: 'CloudWatch' } } }, expected: { cost: 6218.91, recordCount: 30, validation: "cloudwatch" } } ]; // Send all requests console.log('📤 Sending all 13 questions...\n'); for (const q of questions) { server.stdin.write(JSON.stringify(q.request) + '\n'); await new Promise(resolve => setTimeout(resolve, 1500)); } // Wait for all responses await new Promise(resolve => setTimeout(resolve, 30000)); // Analyze results console.log('=' .repeat(60)); console.log('📊 RESULTS:'); console.log('=' .repeat(60)); const results = []; for (const question of questions) { const response = responses.find(r => r.id === question.request.id); let status = '❌ NO RESPONSE'; let actualValue = null; let matches = false; let details = ''; if (response?.error) { status = '❌ ERROR'; details = response.error.message; } else if (response?.result?.content?.[0]?.text) { const text = response.result.content[0].text; if (question.expected.validation === 'cost' || question.expected.validation === 'cloudwatch') { const jsonMatch = text.match(/```json\n([\s\S]*?)\n```/); if (jsonMatch) { try { const data = JSON.parse(jsonMatch[1]); if (Array.isArray(data)) { const totalCost = data.reduce((sum, item) => sum + parseFloat(item.total_cost || 0), 0); actualValue = totalCost; if (question.expected.validation === 'cloudwatch') { // Special validation for Q13 const services = new Set(); data.forEach(item => { if (item.service_name) services.add(item.service_name); if (item.group_by && item.group_by !== 'Total') services.add(item.group_by); }); const isCloudWatchOnly = services.size === 1 && services.has('AmazonCloudWatch'); const costMatch = Math.abs(totalCost - question.expected.cost) < 1; const recordMatch = data.length === question.expected.recordCount; if (isCloudWatchOnly && costMatch && recordMatch) { status = '✅ PERFECT'; matches = true; details = `CloudWatch-only, ${data.length} records`; } else { status = '⚠️ PARTIAL'; details = `Services: ${Array.from(services).join(', ')}`; } } else { // Regular cost validation const expectedCost = question.expected.cost; const diff = Math.abs(totalCost - expectedCost); const percentDiff = expectedCost > 0 ? (diff / expectedCost * 100) : 100; if (percentDiff <= 5) { status = '✅ PASS'; matches = true; details = percentDiff > 0 ? `(${percentDiff.toFixed(1)}% diff)` : ''; } else if (percentDiff <= 10) { status = '🟡 CLOSE'; details = `(${percentDiff.toFixed(1)}% diff)`; } else { status = '❌ FAIL'; details = expectedCost === 0 && totalCost > 0 ? `Expected $0, got $${totalCost.toFixed(2)}` : `(${percentDiff.toFixed(1)}% diff)`; } } } } catch (e) { status = '❌ PARSE ERROR'; } } } else { // Response validation if (text.includes('MSP') || text.includes('customer')) { status = '✅ PASS'; matches = true; } else if (text.includes('accounts')) { status = '✅ PASS'; matches = true; } else { status = '⚠️ DIFFERENT'; } } } results.push({ id: question.id, name: question.name, status, matches, expected: question.expected.cost, actual: actualValue, details }); console.log(`Q${question.id.toString().padStart(2, '0')}: ${status} - ${question.name}`); if (actualValue !== null) { console.log(` Expected: $${question.expected.cost?.toFixed(2) || 'N/A'}`); console.log(` Actual: $${actualValue.toFixed(2)} ${details}`); } else if (details) { console.log(` ${details}`); } console.log(''); } // Summary const passCount = results.filter(r => r.status.includes('✅')).length; const totalCount = results.length; const passRate = (passCount / totalCount * 100).toFixed(1); console.log('=' .repeat(60)); console.log('📈 SUMMARY:'); console.log('=' .repeat(60)); console.log(`✅ Passing: ${passCount}/${totalCount} (${passRate}%)`); console.log(`❌ Failing: ${totalCount - passCount}/${totalCount}`); // Key findings console.log('\n🎯 KEY FINDINGS:'); const q13 = results.find(r => r.id === 13); if (q13?.status.includes('✅')) { console.log('✅ Q13 CloudWatch filtering: WORKING CORRECTLY'); } else { console.log('❌ Q13 CloudWatch filtering: STILL HAS ISSUES'); } const gcpAzure = [4, 5, 8].map(id => results.find(r => r.id === id)); const gcpAzureFailing = gcpAzure.filter(r => r?.status.includes('❌')).length; if (gcpAzureFailing > 0) { console.log(`⚠️ GCP/Azure filtering: ${gcpAzureFailing}/3 failing (API may not support cloud_context)`); } const costQuestions = [2, 3, 6, 7, 10, 11].map(id => results.find(r => r.id === id)); const costPassing = costQuestions.filter(r => r?.status.includes('✅')).length; console.log(`📊 Cost calculations: ${costPassing}/6 within 5% tolerance`); server.kill(); console.log('\n🏁 Final compatibility test complete!'); } finalCompatibilityTest().catch(console.error);