Umbrella MCP Server

#!/usr/bin/env node const { spawn } = require('child_process'); async function fixedMcpRegression() { console.log('🧪 COMPREHENSIVE MCP REGRESSION TEST (FIXED)'); console.log('=' .repeat(80)); console.log('Testing ALL functionality through MCP protocol only'); console.log(''); const server = spawn('node', ['dist/index.js'], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, UMBRELLA_API_BASE_URL: 'https://api-front.umbrellacost.io/api/v1' } }); const responses = {}; let requestId = 0; let testResults = []; server.stdout.on('data', (data) => { const lines = data.toString().split('\n').filter(line => line.trim()); for (const line of lines) { try { const parsed = JSON.parse(line); if (parsed.id) responses[parsed.id] = parsed; } catch (e) {} } }); console.log('🔐 Authenticating through MCP...'); await new Promise(resolve => setTimeout(resolve, 2000)); server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'authenticate_user', arguments: { username: 'david+saola@umbrellacost.com', password: 'Dsamsung1!' } } }) + '\n'); await new Promise(resolve => setTimeout(resolve, 8000)); console.log('\n📊 CRITICAL CLOUD CONTEXT TESTS'); console.log('=' .repeat(60)); const criticalTests = [ { name: 'AWS Cost (Cloud Context)', tool: 'api___invoices_caui', args: { startDate: '2025-08-11', endDate: '2025-08-17', groupBy: 'none', periodGranLevel: 'day', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'aws' }, expectedAccount: '932213950603' }, { name: 'GCP Cost (Cloud Context)', tool: 'api___invoices_caui', args: { startDate: '2025-06-30', endDate: '2025-08-27', groupBy: 'none', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'gcp' }, expectedAccount: 'Master-59f88c' }, { name: 'Azure Cost (Cloud Context)', tool: 'api___invoices_caui', args: { startDate: '2024-07-10', endDate: '2024-08-25', groupBy: 'none', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'azure' }, expectedAccount: 'azure-pileus-9322' } ]; for (let i = 0; i < criticalTests.length; i++) { const test = criticalTests[i]; const testId = ++requestId; console.log(`\nTEST ${i + 1}: ${test.name}`); console.log('-'.repeat(50)); server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: testId, method: "tools/call", params: { name: test.tool, arguments: test.args } }) + '\n'); await new Promise(resolve => setTimeout(resolve, 10000)); const response = responses[testId]; let testResult = { name: test.name, passed: false, details: '', cloudContext: test.args.cloud_context }; if (!response) { testResult.details = '❌ No response received'; console.log('❌ FAIL: No response received'); } else if (response.error) { testResult.details = `❌ Error: ${response.error.message || response.error}`; console.log(`❌ FAIL: Error - ${response.error.message || response.error}`); } else if (response.result?.content?.[0]?.text) { const text = response.result.content[0].text; // Check for JSON data in response const jsonMatch = text.match(/```json\n([\s\S]*?)\n```/); if (jsonMatch) { try { const data = JSON.parse(jsonMatch[1]); if (Array.isArray(data) && data.length > 0) { // Analyze the data const accounts = new Set(); const services = new Set(); let totalCost = 0; data.forEach(item => { if (item.account_id) accounts.add(item.account_id); if (item.service_name) services.add(item.service_name); if (item.total_cost) totalCost += parseFloat(item.total_cost); }); const accountList = Array.from(accounts); const serviceList = Array.from(services); // Check if we got the expected account const hasExpectedAccount = accountList.includes(test.expectedAccount); if (hasExpectedAccount) { testResult.passed = true; testResult.details = `✅ PASS: ${data.length} records, $${totalCost.toFixed(2)} total, correct account: ${test.expectedAccount}`; console.log(`✅ PASS: Correct ${test.args.cloud_context.toUpperCase()} data received`); console.log(` Account: ${test.expectedAccount} ✓`); console.log(` Records: ${data.length}`); console.log(` Total Cost: $${totalCost.toFixed(2)}`); if (serviceList.length > 0) { console.log(` Sample Services: ${serviceList.slice(0, 3).join(', ')}`); } } else { testResult.details = `❌ WRONG ACCOUNT: Expected ${test.expectedAccount}, got ${accountList.join(', ')}`; console.log(`❌ FAIL: Wrong account data`); console.log(` Expected: ${test.expectedAccount}`); console.log(` Got: ${accountList.join(', ')}`); console.log(' 🚨 CLOUD CONTEXT NOT WORKING!'); } } else { testResult.details = '❌ Empty or invalid data array'; console.log('❌ FAIL: Empty or invalid data array'); } } catch (e) { testResult.details = `❌ JSON parsing error: ${e.message}`; console.log(`❌ FAIL: JSON parsing error - ${e.message}`); } } else { testResult.details = '❌ No JSON found in response'; console.log('❌ FAIL: No JSON found in response'); } } else { testResult.details = '❌ Invalid response format'; console.log('❌ FAIL: Invalid response format'); } testResults.push(testResult); } // Quick test of other functionality console.log('\n\n📊 BASIC FUNCTIONALITY TESTS'); console.log('=' .repeat(60)); const basicTests = [ { name: 'Budget API', tool: 'api___budgets_v2_i_', args: { cloud_context: 'aws', only_metadata: true } }, { name: 'Recommendations', tool: 'api___recommendations_report', args: {} } ]; for (let i = 0; i < basicTests.length; i++) { const test = basicTests[i]; const testId = ++requestId; console.log(`\nBASIC TEST ${i + 1}: ${test.name}`); server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: testId, method: "tools/call", params: { name: test.tool, arguments: test.args } }) + '\n'); await new Promise(resolve => setTimeout(resolve, 8000)); const response = responses[testId]; if (response && response.result?.content?.[0]?.text) { console.log('✅ PASS: Response received'); testResults.push({ name: test.name, passed: true, details: '✅ Working' }); } else { console.log('❌ FAIL: No proper response'); testResults.push({ name: test.name, passed: false, details: '❌ No response' }); } } server.kill(); // Final Assessment console.log('\n\n🎯 FINAL ASSESSMENT - HONEST VERDICT'); console.log('=' .repeat(80)); const totalTests = testResults.length; const passedTests = testResults.filter(t => t.passed).length; const criticalCloudTests = testResults.filter(t => t.cloudContext).length; const passedCloudTests = testResults.filter(t => t.cloudContext && t.passed).length; console.log(`\n📊 RESULTS:`); console.log(` Total Tests: ${totalTests}`); console.log(` Passed: ${passedTests}`); console.log(` Failed: ${totalTests - passedTests}`); console.log(` Success Rate: ${((passedTests/totalTests) * 100).toFixed(1)}%`); console.log(`\n🔥 CRITICAL CLOUD CONTEXT TESTS:`); console.log(` Cloud Tests: ${criticalCloudTests}`); console.log(` Cloud Tests Passed: ${passedCloudTests}`); console.log(` Cloud Success Rate: ${((passedCloudTests/criticalCloudTests) * 100).toFixed(1)}%`); console.log('\n📋 DETAILED RESULTS:'); testResults.forEach((result, index) => { const icon = result.passed ? '✅' : '❌'; const critical = result.cloudContext ? '🔥 CRITICAL' : ''; console.log(` ${icon} ${critical} ${result.name}: ${result.details}`); }); // Honest Assessment const cloudContextWorking = passedCloudTests === criticalCloudTests; console.log('\n🎯 HONEST VERDICT:'); if (cloudContextWorking && passedTests === totalTests) { console.log('✅ PERFECT: All tests pass, cloud context is working correctly'); console.log(' ✅ GCP/Azure account issue has been successfully fixed'); console.log(' ✅ No regression in existing functionality'); } else if (cloudContextWorking) { console.log('✅ GOOD: Cloud context working, minor issues in other features'); console.log(' ✅ GCP/Azure account issue has been successfully fixed'); console.log(' ⚠️ Some non-critical features have minor issues'); } else { console.log('❌ CRITICAL ISSUES: Cloud context is NOT working correctly'); console.log(' ❌ GCP/Azure account issue is NOT fully fixed'); console.log(' 🚨 This breaks the main functionality user requested'); } console.log('\n🏁 MCP Regression testing complete!'); } fixedMcpRegression().catch(console.error);