Umbrella MCP Server

#!/usr/bin/env node const { spawn } = require('child_process'); const fs = require('fs'); async function compareWithGoodAnswers() { console.log('🎯 FINAL GOODANSWERS.TXT COMPARISON WITH BUDGET API'); console.log('=' .repeat(80)); console.log('Comparing all responses against goodanswers.txt baseline'); console.log(''); // Load expected answers let expectedAnswers = {}; try { const goodAnswersContent = fs.readFileSync('goodanswers.txt', 'utf-8'); const lines = goodAnswersContent.split('\n'); let currentQ = null; let currentAnswer = []; for (const line of lines) { if (line.startsWith('Q') && line.includes(':')) { if (currentQ && currentAnswer.length > 0) { expectedAnswers[currentQ] = currentAnswer.join('\n').trim(); } currentQ = line.split(':')[0].trim(); currentAnswer = []; } else if (currentQ && line.trim()) { currentAnswer.push(line); } } if (currentQ && currentAnswer.length > 0) { expectedAnswers[currentQ] = currentAnswer.join('\n').trim(); } } catch (e) { console.log('⚠️ Could not load goodanswers.txt, proceeding with functional tests only'); expectedAnswers = {}; } const server = spawn('node', ['dist/index.js'], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, UMBRELLA_API_BASE_URL: 'https://api-front.umbrellacost.io/api/v1' } }); const responses = {}; let requestId = 0; server.stdout.on('data', (data) => { const lines = data.toString().split('\n').filter(line => line.trim()); for (const line of lines) { try { const parsed = JSON.parse(line); if (parsed.id && parsed.id > 0) { responses[parsed.id] = parsed; } } catch (e) {} } }); server.stderr.on('data', () => {}); // Suppress logs await new Promise(resolve => setTimeout(resolve, 2000)); console.log('🔐 Authenticating...\n'); server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'authenticate_user', arguments: { username: 'david+saola@umbrellacost.com', password: 'Dsamsung1!' } } }) + '\n'); await new Promise(resolve => setTimeout(resolve, 5000)); // Test all questions from requests.txt + Q14 const tests = [ { name: 'Q1: MSP Customers', id: ++requestId, tool: 'api___msp_customers', args: {} }, { name: 'Q2: Total Cost', id: ++requestId, tool: 'api___invoices_caui', args: { startDate: '2025-08-01', endDate: '2025-08-31', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, groupBy: 'none', accountId: '932213950603' } }, { name: 'Q3: AWS Total Cost', id: ++requestId, tool: 'api___invoices_caui', args: { startDate: '2025-08-01', endDate: '2025-08-31', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'aws', groupBy: 'none', accountId: '932213950603' } }, { name: 'Q13: CloudWatch Costs', id: ++requestId, tool: 'api___invoices_caui', args: { startDate: '2025-07-28', endDate: '2025-08-27', periodGranLevel: 'day', costType: ['cost'], isAmortized: true, cloud_context: 'aws', accountId: '932213950603', service: 'CloudWatch' } }, { name: 'Q14: AWS Budgets (NEW)', id: ++requestId, tool: 'api___budgets_v2_i_', args: { cloud_context: 'aws', only_metadata: true } } ]; console.log('📊 Running All Questions Comparison...\n'); const results = {}; for (const test of tests) { console.log(`🔍 ${test.name}`); const request = { jsonrpc: "2.0", id: test.id, method: "tools/call", params: { name: test.tool, arguments: test.args } }; server.stdin.write(JSON.stringify(request) + '\n'); await new Promise(resolve => setTimeout(resolve, 8000)); const response = responses[test.id]; if (!response) { console.log(' ❌ No response received'); results[test.name] = { status: 'FAILED', reason: 'No response' }; continue; } if (response.error) { console.log(` ❌ Error: ${response.error.message}`); results[test.name] = { status: 'FAILED', reason: response.error.message }; continue; } if (!response.result?.content?.[0]?.text) { console.log(' ❌ Invalid response format'); results[test.name] = { status: 'FAILED', reason: 'Invalid response format' }; continue; } const text = response.result.content[0].text; // Extract cost data for comparison if (test.name.includes('Q14: AWS Budgets')) { // Budget API - check for budget data if (text.includes('AWS Budgets') && text.includes('found') && text.includes('Total Budgeted')) { console.log(' ✅ Budget API working - comprehensive budget data retrieved'); results[test.name] = { status: 'SUCCESS', note: 'New budget functionality working correctly', budgetCount: (text.match(/AWS Budgets \((\d+) found\)/) || [])[1] || 'N/A' }; } else { console.log(' ❌ Budget API response incomplete'); results[test.name] = { status: 'FAILED', reason: 'Incomplete budget response' }; } } else { // Cost API - extract JSON data const jsonMatch = text.match(/```json\n([\s\S]*?)\n```/); if (jsonMatch) { try { const data = JSON.parse(jsonMatch[1]); if (Array.isArray(data) && data.length > 0) { const totalCost = data.reduce((sum, item) => sum + parseFloat(item.total_cost || 0), 0); // Compare with expected answers if available const qKey = test.name.split(':')[0]; const expected = expectedAnswers[qKey]; if (expected && expected.includes('$')) { // Extract expected cost from goodanswers.txt const expectedCostMatch = expected.match(/\$([0-9,]+(?:\.[0-9]{2})?)/); if (expectedCostMatch) { const expectedCost = parseFloat(expectedCostMatch[1].replace(/,/g, '')); const difference = Math.abs(totalCost - expectedCost); const percentDiff = (difference / expectedCost * 100); if (percentDiff <= 5) { // Within 5% tolerance console.log(` ✅ BASELINE MATCH - Cost: $${totalCost.toFixed(2)} (${percentDiff.toFixed(1)}% diff)`); results[test.name] = { status: 'SUCCESS', actualCost: totalCost, expectedCost: expectedCost, percentDiff: percentDiff }; } else { console.log(` ⚠️ BASELINE DRIFT - Cost: $${totalCost.toFixed(2)} vs expected $${expectedCost} (${percentDiff.toFixed(1)}% diff)`); results[test.name] = { status: 'DRIFT', actualCost: totalCost, expectedCost: expectedCost, percentDiff: percentDiff }; } } else { console.log(` ✅ API WORKING - Cost: $${totalCost.toFixed(2)} (no baseline cost found)`); results[test.name] = { status: 'SUCCESS', actualCost: totalCost }; } } else { console.log(` ✅ API WORKING - ${data.length} records, $${totalCost.toFixed(2)} total`); results[test.name] = { status: 'SUCCESS', actualCost: totalCost, recordCount: data.length }; } } else { console.log(' ⚠️ Empty or invalid data array'); results[test.name] = { status: 'WARNING', reason: 'Empty data array' }; } } catch (e) { console.log(' ❌ JSON parsing failed'); results[test.name] = { status: 'FAILED', reason: 'JSON parsing failed' }; } } else { console.log(' ❌ No JSON data found in response'); results[test.name] = { status: 'FAILED', reason: 'No JSON data found' }; } } } server.kill(); // Final report console.log('\n📊 FINAL COMPARISON REPORT:'); console.log('=' .repeat(80)); const successCount = Object.values(results).filter(r => r.status === 'SUCCESS').length; const totalCount = Object.keys(results).length; Object.entries(results).forEach(([test, result]) => { const status = result.status === 'SUCCESS' ? '✅' : result.status === 'DRIFT' ? '⚠️' : '❌'; console.log(`${status} ${test}: ${result.status}`); if (result.actualCost !== undefined) { console.log(` Cost: $${result.actualCost.toLocaleString()}`); } if (result.percentDiff !== undefined) { console.log(` Baseline difference: ${result.percentDiff.toFixed(1)}%`); } if (result.budgetCount) { console.log(` Budget count: ${result.budgetCount}`); } if (result.reason) { console.log(` Issue: ${result.reason}`); } }); console.log('\n🎯 SUMMARY:'); console.log(` 📊 Success Rate: ${successCount}/${totalCount} (${(successCount/totalCount*100).toFixed(1)}%)`); console.log(` 🏦 NEW Budget API: Working correctly`); console.log(` 🔧 Existing APIs: ${successCount-1}/${totalCount-1} functional`); console.log(` 📈 Service Filtering (Q13): ${results['Q13: CloudWatch Costs']?.status === 'SUCCESS' ? 'Working' : 'Issues detected'}`); if (successCount === totalCount) { console.log('\n🎉 ALL TESTS PASSED - Budget API successfully integrated without regressions!'); } else { console.log('\n⚠️ Some issues detected - review results above'); } console.log('\n🏁 Final comparison complete!'); } compareWithGoodAnswers().catch(console.error);