Umbrella MCP Server

#!/usr/bin/env node const { spawn } = require('child_process'); async function comprehensiveMcpRegression() { console.log('🧪 COMPREHENSIVE MCP REGRESSION TEST'); console.log('=' .repeat(80)); console.log('Testing ALL functionality through MCP protocol only'); console.log('No direct API calls - everything through MCP server'); console.log(''); const server = spawn('node', ['dist/index.js'], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, UMBRELLA_API_BASE_URL: 'https://api-front.umbrellacost.io/api/v1' } }); const responses = {}; let requestId = 0; let testResults = []; server.stdout.on('data', (data) => { const lines = data.toString().split('\n').filter(line => line.trim()); for (const line of lines) { try { const parsed = JSON.parse(line); if (parsed.id) responses[parsed.id] = parsed; } catch (e) {} } }); // Capture any errors server.stderr.on('data', (data) => { const msg = data.toString(); if (msg.includes('ERROR') || msg.includes('FAILED')) { console.log('🚨 ERROR:', msg.trim()); } }); console.log('🔐 Authenticating through MCP...\n'); await new Promise(resolve => setTimeout(resolve, 2000)); server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name: 'authenticate_user', arguments: { username: 'david+saola@umbrellacost.com', password: 'Dsamsung1!' } } }) + '\n'); await new Promise(resolve => setTimeout(resolve, 8000)); // Test Suite 1: Basic functionality console.log('📊 TEST SUITE 1: Basic Cost Queries (MCP Protocol)'); console.log('=' .repeat(60)); const tests = [ { name: 'AWS Total Cost (No Cloud Context)', tool: 'api___invoices_caui', args: { startDate: '2025-08-11', endDate: '2025-08-17', groupBy: 'none', periodGranLevel: 'day', costType: ['cost', 'discount'], isUnblended: true }, expectedType: 'cost_data' }, { name: 'AWS Total Cost (With Cloud Context)', tool: 'api___invoices_caui', args: { startDate: '2025-08-11', endDate: '2025-08-17', groupBy: 'none', periodGranLevel: 'day', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'aws' }, expectedType: 'cost_data' }, { name: 'GCP Total Cost (Cloud Context)', tool: 'api___invoices_caui', args: { startDate: '2025-06-30', endDate: '2025-08-27', groupBy: 'none', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'gcp' }, expectedType: 'cost_data' }, { name: 'Azure Total Cost (Cloud Context)', tool: 'api___invoices_caui', args: { startDate: '2024-07-10', endDate: '2024-08-25', groupBy: 'none', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'azure' }, expectedType: 'cost_data' }, { name: 'AWS Services Breakdown', tool: 'api___invoices_caui', args: { startDate: '2025-08-11', endDate: '2025-08-17', groupBy: 'service', periodGranLevel: 'day', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'aws' }, expectedType: 'cost_data' }, { name: 'GCP Services Breakdown', tool: 'api___invoices_caui', args: { startDate: '2025-06-30', endDate: '2025-08-27', groupBy: 'service', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, cloud_context: 'gcp' }, expectedType: 'cost_data' } ]; for (let i = 0; i < tests.length; i++) { const test = tests[i]; const testId = ++requestId; console.log(`\nTEST ${i + 1}: ${test.name}`); console.log('-'.repeat(40)); server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: testId, method: "tools/call", params: { name: test.tool, arguments: test.args } }) + '\n'); await new Promise(resolve => setTimeout(resolve, 8000)); const response = responses[testId]; let testResult = { name: test.name, passed: false, details: '' }; if (!response) { testResult.details = '❌ No response received'; console.log('❌ FAIL: No response received'); } else if (response.error) { testResult.details = `❌ Error: ${response.error.message || response.error}`; console.log(`❌ FAIL: Error - ${response.error.message || response.error}`); } else if (response.result?.content?.[0]?.text) { const text = response.result.content[0].text; // Check for JSON data in response const jsonMatch = text.match(/```json\n([\s\S]*?)\n```/); if (jsonMatch) { try { const data = JSON.parse(jsonMatch[1]); if (Array.isArray(data) && data.length > 0) { testResult.passed = true; // Analyze the data based on cloud context const cloudContext = test.args.cloud_context; const accounts = new Set(); const services = new Set(); let totalCost = 0; data.forEach(item => { if (item.account_id) accounts.add(item.account_id); if (item.service_name) services.add(item.service_name); if (item.total_cost) totalCost += parseFloat(item.total_cost); }); const accountList = Array.from(accounts); const serviceList = Array.from(services); testResult.details = `✅ PASS: ${data.length} records, $${totalCost.toFixed(2)} total`; // Cloud-specific validation if (cloudContext === 'gcp') { const hasGcpAccount = accountList.some(id => id.includes('Master') || id.includes('59f88c')); const hasGcpServices = serviceList.some(s => s.includes('BigQuery') || s.includes('Cloud Storage') || s.includes('Compute Engine')); if (hasGcpAccount && hasGcpServices) { testResult.details += ' | GCP data confirmed ✓'; } else { testResult.details += ' | ⚠️ GCP data validation failed'; testResult.passed = false; } } else if (cloudContext === 'azure') { const hasAzureAccount = accountList.some(id => id.includes('azure') || id.includes('pileus')); if (hasAzureAccount) { testResult.details += ' | Azure data confirmed ✓'; } else { testResult.details += ' | ⚠️ Azure data validation failed'; testResult.passed = false; } } else if (cloudContext === 'aws') { const hasAwsAccount = accountList.some(id => id.includes('932213950603')); if (hasAwsAccount) { testResult.details += ' | AWS data confirmed ✓'; } else { testResult.details += ' | ⚠️ AWS data validation failed'; testResult.passed = false; } } console.log(testResult.details); console.log(` Accounts: ${accountList.join(', ')}`); console.log(` Services: ${serviceList.slice(0, 3).join(', ')}${serviceList.length > 3 ? '...' : ''}`); } else { testResult.details = '❌ Empty or invalid data array'; console.log('❌ FAIL: Empty or invalid data array'); } } catch (e) { testResult.details = `❌ JSON parsing error: ${e.message}`; console.log(`❌ FAIL: JSON parsing error - ${e.message}`); } } else { testResult.details = '❌ No JSON found in response'; console.log('❌ FAIL: No JSON found in response'); } } else { testResult.details = '❌ Invalid response format'; console.log('❌ FAIL: Invalid response format'); } testResults.push(testResult); } // Test Suite 2: Advanced features console.log('\n\n📊 TEST SUITE 2: Advanced Features (MCP Protocol)'); console.log('=' .repeat(60)); const advancedTests = [ { name: 'Budget API', tool: 'api___budgets_v2_i_', args: { cloud_context: 'aws', only_metadata: true }, expectedType: 'budget_data' }, { name: 'Recommendations', tool: 'api___recommendations_report', args: {}, expectedType: 'recommendation_data' }, { name: 'User Accounts', tool: 'api___users', args: {}, expectedType: 'account_data' } ]; for (let i = 0; i < advancedTests.length; i++) { const test = advancedTests[i]; const testId = ++requestId; console.log(`\\nADVANCED TEST ${i + 1}: ${test.name}`); console.log('-'.repeat(40)); server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: testId, method: "tools/call", params: { name: test.tool, arguments: test.args } }) + '\n'); await new Promise(resolve => setTimeout(resolve, 10000)); const response = responses[testId]; let testResult = { name: test.name, passed: false, details: '' }; if (!response) { testResult.details = '❌ No response received'; console.log('❌ FAIL: No response received'); } else if (response.error) { testResult.details = `❌ Error: ${response.error.message || response.error}`; console.log(`❌ FAIL: Error - ${response.error.message || response.error}`); } else if (response.result?.content?.[0]?.text) { testResult.passed = true; testResult.details = '✅ PASS: Response received'; console.log('✅ PASS: Response received and processed'); } else { testResult.details = '❌ Invalid response format'; console.log('❌ FAIL: Invalid response format'); } testResults.push(testResult); } server.kill(); // Final Results console.log('\\n\\n📊 COMPREHENSIVE MCP REGRESSION RESULTS'); console.log('=' .repeat(80)); const totalTests = testResults.length; const passedTests = testResults.filter(t => t.passed).length; const failedTests = totalTests - passedTests; console.log(`\\n📋 SUMMARY:`); console.log(` Total Tests: ${totalTests}`); console.log(` Passed: ${passedTests}`); console.log(` Failed: ${failedTests}`); console.log(` Success Rate: ${((passedTests/totalTests) * 100).toFixed(1)}%`); console.log('\\n📋 DETAILED RESULTS:'); testResults.forEach((result, index) => { const icon = result.passed ? '✅' : '❌'; console.log(` ${icon} Test ${index + 1}: ${result.name}`); console.log(` ${result.details}`); }); // Critical Issues Check const criticalIssues = testResults.filter(t => !t.passed && (t.name.includes('GCP') || t.name.includes('Azure') || t.name.includes('AWS')) ); if (criticalIssues.length > 0) { console.log('\\n🚨 CRITICAL ISSUES DETECTED:'); criticalIssues.forEach(issue => { console.log(` ❌ ${issue.name}: ${issue.details}`); }); console.log('\\n⚠️ Cloud context functionality may be broken!'); } else { console.log('\\n✅ No critical issues detected - cloud context working properly'); } console.log('\\n🏁 MCP Regression testing complete!'); // Honest assessment if (passedTests === totalTests) { console.log('\\n🎯 VERDICT: All tests passed - system is working correctly'); } else if (passedTests / totalTests >= 0.8) { console.log('\\n⚠️ VERDICT: Most tests passed but some issues need attention'); } else { console.log('\\n❌ VERDICT: Significant issues detected - system needs fixes'); } } comprehensiveMcpRegression().catch(console.error);