Umbrella MCP Server

#!/usr/bin/env node /** * Simple All-in-One Endpoint Test Runner * Usage: node run-all-tests.cjs <saola_password> <allcloud_password> * * This script runs comprehensive endpoint testing for both accounts * and generates detailed reports automatically. */ const https = require('https'); const fs = require('fs'); const path = require('path'); // Parse command line arguments const args = process.argv.slice(2); if (args.length < 2) { console.log('🚀 Umbrella MCP - Simple Endpoint Test Runner'); console.log('=============================================='); console.log(''); console.log('Usage: node run-all-tests.cjs <saola_password> <allcloud_password>'); console.log(''); console.log('Example:'); console.log(' node run-all-tests.cjs "my_saola_password" "my_allcloud_password"'); console.log(''); console.log('This will test all endpoints for both SAOLA and AllCloud accounts'); console.log('and generate comprehensive reports automatically.'); process.exit(1); } const SAOLA_PASSWORD = args[0]; const ALLCLOUD_PASSWORD = args[1]; // Account configurations const ACCOUNTS = { SAOLA: { name: 'SAOLA (Direct Customer)', username: 'david+saola@umbrellacost.com', password: SAOLA_PASSWORD, baseUrl: 'api.umbrellacost.io', apiPath: '/api/v1', type: 'direct' }, ALLCLOUD: { name: 'AllCloud (MSP)', username: 'david+allcloud@umbrellacost.com', password: ALLCLOUD_PASSWORD, baseUrl: 'api.umbrellacost.io', apiPath: '/api/v1', type: 'msp' } }; // MSP customers to test const MSP_CUSTOMERS = [ { name: 'Bank Leumi', accountKey: '22676', divisionId: '139', accountId: '696314371547' }, { name: 'Bank Hapoalim', accountKey: '16185', divisionId: '1', accountId: '123456789012' } ]; // All endpoint tests to run const ENDPOINT_TESTS = [ // Cost Analysis Tests { path: '/invoices/caui', method: 'GET', name: 'Basic unblended costs', category: 'Cost Analysis', params: { groupBy: 'none', periodGranLevel: 'month', costType: JSON.stringify(['cost', 'discount']), isUnblended: 'true', startDate: '2024-01-01', endDate: '2024-12-31' } }, { path: '/invoices/caui', method: 'GET', name: 'Service breakdown (amortized)', category: 'Cost Analysis', params: { groupBy: 'service', periodGranLevel: 'month', costType: JSON.stringify(['cost', 'discount']), isAmortized: 'true', startDate: '2024-01-01', endDate: '2024-12-31' } }, { path: '/invoices/caui', method: 'GET', name: 'EC2 costs only', category: 'Cost Analysis', params: { 'filters[service]': 'Amazon Elastic Compute Cloud', groupBy: 'region', periodGranLevel: 'day', costType: JSON.stringify(['cost']), isUnblended: 'true', startDate: '2024-11-01', endDate: '2024-11-30' } }, // Budget Management Tests { path: '/budgets/v2/i/', method: 'GET', name: 'All budgets (metadata)', category: 'Budget Management', params: { only_metadata: 'true' } }, { path: '/budgets/v2/i/', method: 'GET', name: 'AWS budgets only', category: 'Budget Management', params: { cloud_context: 'aws', only_metadata: 'true' } }, // Recommendations Tests { path: '/recommendationsNew/heatmap/summary', method: 'POST', name: 'Recommendations summary', category: 'Recommendations', params: {} }, { path: '/recommendations/report', method: 'GET', name: 'Legacy recommendations', category: 'Recommendations (Legacy)', params: {} }, // User Management Tests { path: '/users', method: 'GET', name: 'User information', category: 'User Management', params: {} }, { path: '/users/plain-sub-users', method: 'GET', name: 'Customer divisions', category: 'MSP Management', params: {} }, // Anomaly Detection Tests { path: '/anomaly-detection', method: 'GET', name: 'Recent anomalies', category: 'Anomaly Detection', params: { startDate: '2024-11-01', endDate: '2024-12-31', isFull: 'true' } }, // Service Discovery { path: '/invoices/service-names/distinct', method: 'GET', name: 'Service names (limited)', category: 'Service Discovery', params: { limit: '50' } } ]; class SimpleTestRunner { constructor() { this.results = []; this.startTime = Date.now(); this.totalTests = 0; this.successfulTests = 0; this.authTokens = new Map(); } log(message, level = 'info') { const timestamp = new Date().toISOString().split('.')[0]; const emoji = { info: '📋', success: '✅', error: '❌', warning: '⚠️', test: '🧪', title: '🚀' }[level] || '📋'; console.log(`${timestamp} ${emoji} ${message}`); } async makeHttpRequest(hostname, path, method = 'GET', headers = {}, postData = null) { return new Promise((resolve, reject) => { const options = { hostname, path, method, headers: { 'Content-Type': 'application/json', 'User-Agent': 'Umbrella-Test-Runner/1.0', ...headers } }; if (postData && method === 'POST') { const data = JSON.stringify(postData); options.headers['Content-Length'] = Buffer.byteLength(data); } const req = https.request(options, (res) => { let responseBody = ''; res.on('data', (chunk) => responseBody += chunk); res.on('end', () => { try { const parsedBody = responseBody ? JSON.parse(responseBody) : null; resolve({ statusCode: res.statusCode, headers: res.headers, body: parsedBody, rawBody: responseBody }); } catch (error) { resolve({ statusCode: res.statusCode, headers: res.headers, body: null, rawBody: responseBody }); } }); }); req.on('error', reject); req.setTimeout(30000, () => { req.destroy(); reject(new Error('Request timeout')); }); if (postData && method === 'POST') { req.write(JSON.stringify(postData)); } req.end(); }); } async authenticateAccount(account) { this.log(`Authenticating ${account.name}...`); try { const response = await this.makeHttpRequest( account.baseUrl, account.apiPath + '/users/signin', 'POST', {}, { username: account.username, password: account.password } ); if (response.statusCode === 200 && response.body?.jwtToken) { const authToken = { Authorization: response.body.jwtToken, 'Content-Type': 'application/json' }; this.authTokens.set(account.name, authToken); this.log(`${account.name} authentication successful`, 'success'); return { success: true, token: authToken }; } else { this.log(`${account.name} authentication failed: ${response.statusCode}`, 'error'); return { success: false }; } } catch (error) { this.log(`${account.name} authentication error: ${error.message}`, 'error'); return { success: false }; } } async testEndpoint(account, authToken, test, customer = null) { const testName = `${account.name} - ${test.name}${customer ? ` (${customer.name})` : ''}`; this.log(`Testing: ${testName}`, 'test'); this.totalTests++; try { // Prepare parameters let params = { ...test.params }; // Add MSP customer parameters if applicable if (customer && account.type === 'msp') { params.customer_account_key = customer.accountKey; params.customer_division_id = customer.divisionId; if (test.path === '/invoices/caui') { params.accountId = customer.accountId; params.divisionId = customer.divisionId; } } // Build URL with query parameters for GET requests let requestPath = account.apiPath + test.path; if (test.method === 'GET' && Object.keys(params).length > 0) { const queryString = Object.entries(params) .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`) .join('&'); requestPath += '?' + queryString; } // Make request const startTime = Date.now(); const response = await this.makeHttpRequest( account.baseUrl, requestPath, test.method, authToken, test.method === 'POST' ? params : null ); const duration = Date.now() - startTime; // Analyze response const success = response.statusCode >= 200 && response.statusCode < 300; const hasData = response.body && ( Array.isArray(response.body) ? response.body.length > 0 : Object.keys(response.body).length > 0 ); const responseSize = response.rawBody ? response.rawBody.length : 0; if (success) { this.successfulTests++; const dataInfo = hasData ? `(${responseSize} bytes)` : '(empty)'; this.log(`✅ ${testName} - ${response.statusCode} - ${duration}ms ${dataInfo}`, 'success'); } else { const errorMsg = response.body?.message || response.body?.error || response.rawBody?.substring(0, 100) || 'Unknown error'; this.log(`❌ ${testName} - ${response.statusCode} - ${errorMsg}`, 'error'); } // Store result this.results.push({ account: account.name, customer: customer?.name || 'Direct', endpoint: test.path, testName: test.name, category: test.category, method: test.method, success, statusCode: response.statusCode, hasData, duration, responseSize, error: success ? null : (response.body?.message || response.body?.error || 'HTTP ' + response.statusCode), timestamp: new Date().toISOString() }); // Brief pause to avoid overwhelming API await new Promise(resolve => setTimeout(resolve, 200)); } catch (error) { this.log(`❌ ${testName} - Network Error: ${error.message}`, 'error'); this.results.push({ account: account.name, customer: customer?.name || 'Direct', endpoint: test.path, testName: test.name, category: test.category, method: test.method, success: false, statusCode: 0, hasData: false, duration: 0, responseSize: 0, error: error.message, timestamp: new Date().toISOString() }); } } async runAllTests() { this.log('🚀 SIMPLE ENDPOINT TEST RUNNER', 'title'); this.log('Comprehensive testing of all Umbrella API endpoints', 'info'); this.log(`Testing ${Object.keys(ACCOUNTS).length} accounts with ${ENDPOINT_TESTS.length} endpoint variations\n`); for (const [accountKey, account] of Object.entries(ACCOUNTS)) { this.log(`\n${'='.repeat(50)}`, 'info'); this.log(`Testing ${account.name}`, 'title'); this.log(`${'='.repeat(50)}`, 'info'); // Authenticate const { success, token } = await this.authenticateAccount(account); if (!success) { this.log(`Skipping ${account.name} due to authentication failure`, 'warning'); continue; } // Test all endpoints for (const test of ENDPOINT_TESTS) { if (account.type === 'msp') { // Test with each MSP customer for (const customer of MSP_CUSTOMERS) { await this.testEndpoint(account, token, test, customer); } } else { // Test direct customer await this.testEndpoint(account, token, test); } } } this.generateReport(); } generateReport() { const duration = Date.now() - this.startTime; this.log('\n' + '='.repeat(60), 'info'); this.log('📊 COMPREHENSIVE TEST RESULTS', 'title'); this.log('='.repeat(60), 'info'); this.log(`Total Tests: ${this.totalTests}`, 'info'); this.log(`Successful: ${this.successfulTests} (${((this.successfulTests/this.totalTests)*100).toFixed(1)}%)`, 'success'); this.log(`Failed: ${this.totalTests - this.successfulTests} (${(((this.totalTests - this.successfulTests)/this.totalTests)*100).toFixed(1)}%)`, 'error'); this.log(`Duration: ${(duration/1000).toFixed(1)}s`, 'info'); // Account performance this.log('\n📊 ACCOUNT PERFORMANCE:', 'info'); const accountGroups = {}; this.results.forEach(r => { if (!accountGroups[r.account]) { accountGroups[r.account] = { total: 0, success: 0, withData: 0, totalSize: 0 }; } accountGroups[r.account].total++; if (r.success) accountGroups[r.account].success++; if (r.hasData) accountGroups[r.account].withData++; accountGroups[r.account].totalSize += r.responseSize; }); Object.entries(accountGroups).forEach(([account, stats]) => { const successRate = ((stats.success/stats.total)*100).toFixed(1); const dataRate = ((stats.withData/stats.total)*100).toFixed(1); const dataSize = (stats.totalSize/1024).toFixed(1); this.log(`${account}:`, 'info'); this.log(` Success: ${stats.success}/${stats.total} (${successRate}%)`, 'info'); this.log(` With Data: ${stats.withData}/${stats.total} (${dataRate}%)`, 'info'); this.log(` Data Retrieved: ${dataSize}KB`, 'info'); }); // Category performance this.log('\n📂 CATEGORY PERFORMANCE:', 'info'); const categoryGroups = {}; this.results.forEach(r => { if (!categoryGroups[r.category]) { categoryGroups[r.category] = { total: 0, success: 0 }; } categoryGroups[r.category].total++; if (r.success) categoryGroups[r.category].success++; }); Object.entries(categoryGroups).forEach(([category, stats]) => { const rate = ((stats.success/stats.total)*100).toFixed(1); this.log(`${category}: ${stats.success}/${stats.total} (${rate}%)`, 'info'); }); // Top working endpoints const working = this.results.filter(r => r.success && r.hasData); if (working.length > 0) { this.log('\n✅ TOP WORKING ENDPOINTS:', 'success'); working.slice(0, 10).forEach(r => { const customerInfo = r.customer !== 'Direct' ? ` (${r.customer})` : ''; this.log(` ${r.endpoint} - ${r.testName}${customerInfo}`, 'info'); }); } // Failed endpoints const failed = this.results.filter(r => !r.success); if (failed.length > 0) { this.log('\n❌ FAILED ENDPOINTS:', 'error'); failed.slice(0, 10).forEach(r => { const customerInfo = r.customer !== 'Direct' ? ` (${r.customer})` : ''; this.log(` ${r.endpoint}${customerInfo} - ${r.error}`, 'warning'); }); } // Save reports this.saveReports(); } saveReports() { const timestamp = Date.now(); // Summary report const summary = { testRun: { timestamp: new Date().toISOString(), duration: Date.now() - this.startTime, totalTests: this.totalTests, successfulTests: this.successfulTests, successRate: ((this.successfulTests/this.totalTests)*100).toFixed(1) }, results: this.results }; const summaryPath = path.join(__dirname, `test-summary-${timestamp}.json`); fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2)); this.log(`\nSummary Report: ${summaryPath}`, 'success'); // Detailed results const detailedPath = path.join(__dirname, `test-results-${timestamp}.json`); fs.writeFileSync(detailedPath, JSON.stringify(this.results, null, 2)); this.log(`Detailed Results: ${detailedPath}`, 'success'); // CSV report const csvHeaders = [ 'Account', 'Customer', 'Endpoint', 'TestName', 'Category', 'Method', 'Success', 'StatusCode', 'HasData', 'Duration(ms)', 'ResponseSize(bytes)', 'Error' ]; const csvRows = this.results.map(r => [ r.account, r.customer, r.endpoint, r.testName, r.category, r.method, r.success, r.statusCode, r.hasData, r.duration, r.responseSize, r.error || '' ]); const csvContent = [csvHeaders.join(','), ...csvRows.map(row => row.map(cell => `"${cell}"`).join(',') )].join('\n'); const csvPath = path.join(__dirname, `test-results-${timestamp}.csv`); fs.writeFileSync(csvPath, csvContent); this.log(`CSV Export: ${csvPath}`, 'success'); this.log('\n🎉 All endpoint testing completed!', 'title'); this.log('Check the generated report files for detailed analysis.', 'info'); } } // Main execution async function main() { console.log('🧪 Umbrella MCP - Simple All-in-One Test Runner'); console.log('================================================'); const runner = new SimpleTestRunner(); await runner.runAllTests(); } if (require.main === module) { main().catch(error => { console.error('❌ Test runner failed:', error); process.exit(1); }); } module.exports = { SimpleTestRunner };