Umbrella MCP Server

#!/usr/bin/env node /** * Direct API Comprehensive Test Script * Tests all Umbrella API endpoints directly with various parameter combinations * for both SAOLA and AllCloud accounts automatically */ const https = require('https'); const querystring = require('querystring'); const fs = require('fs'); const path = require('path'); // Configuration for different account types const ACCOUNT_CONFIGS = { SAOLA: { name: 'SAOLA (Direct Customer)', username: process.env.SAOLA_USERNAME || 'david+saola@umbrellacost.com', password: process.env.SAOLA_PASSWORD || '', accountType: 'direct', baseUrl: 'saola.umbrellacost.com' }, ALLCLOUD: { name: 'AllCloud (MSP Customer)', username: process.env.ALLCLOUD_USERNAME || 'david+allcloud@umbrellacost.com', password: process.env.ALLCLOUD_PASSWORD || '', accountType: 'msp', baseUrl: 'umbrellacost.com' } }; // MSP customer test variations for AllCloud account const MSP_CUSTOMERS = [ { name: 'Bank Leumi', customer_account_key: '22676', customer_division_id: '139', accountId: '696314371547' }, { name: 'Bank Hapoalim', customer_account_key: '16185', customer_division_id: '1', accountId: '123456789012' // example } ]; // All available endpoints with their parameter variations const ENDPOINTS = [ { path: '/invoices/caui', method: 'GET', category: 'Cost Analysis', variations: [ // Basic cost queries { name: 'Basic unblended costs', params: { groupBy: 'none', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, startDate: '2024-01-01', endDate: '2024-12-31' } }, { name: 'Amortized costs', params: { groupBy: 'service', periodGranLevel: 'month', costType: ['cost', 'discount'], isAmortized: true, startDate: '2024-01-01', endDate: '2024-12-31' } }, { name: 'Net amortized costs', params: { groupBy: 'account', periodGranLevel: 'month', costType: ['cost', 'discount'], isNetAmortized: true, startDate: '2024-01-01', endDate: '2024-12-31' } }, // Service-specific queries { name: 'EC2 costs', params: { 'filters[service]': 'Amazon Elastic Compute Cloud', groupBy: 'service', periodGranLevel: 'day', costType: ['cost'], isUnblended: true, startDate: '2024-11-01', endDate: '2024-11-30' } }, { name: 'S3 costs', params: { 'filters[service]': 'Amazon Simple Storage Service', groupBy: 'region', periodGranLevel: 'day', costType: ['cost', 'discount'], isUnblended: true, startDate: '2024-11-01', endDate: '2024-11-30' } }, // Different time periods { name: 'Daily breakdown current month', params: { groupBy: 'service', periodGranLevel: 'day', costType: ['cost', 'discount'], isUnblended: true, startDate: '2024-12-01', endDate: '2024-12-31' } } ] }, { path: '/budgets/v2/i/', method: 'GET', category: 'Budget Management', variations: [ { name: 'All budgets metadata', params: { only_metadata: true } }, { name: 'Full budget details', params: { only_metadata: false } }, { name: 'AWS budgets only', params: { cloud_context: 'aws', only_metadata: true } }, { name: 'Azure budgets only', params: { cloud_context: 'azure', only_metadata: true } }, { name: 'GCP budgets only', params: { cloud_context: 'gcp', only_metadata: true } } ] }, { path: '/recommendationsNew/heatmap/summary', method: 'POST', category: 'Recommendations', variations: [ { name: 'All recommendations summary', params: {} } ] }, { path: '/recommendations/report', method: 'GET', category: 'Recommendations (Legacy)', variations: [ { name: 'Legacy recommendations report', params: {} } ] }, { path: '/anomaly-detection', method: 'GET', category: 'Anomaly Detection', variations: [ { name: 'All anomalies', params: { isFull: true } }, { name: 'Recent anomalies', params: { startDate: '2024-11-01', endDate: '2024-12-31', isFull: true } }, { name: 'Only alerted anomalies', params: { alerted: true, isFull: true } }, { name: 'AWS anomalies', params: { cloud_context: 'aws', isFull: true } } ] }, { path: '/anomalies/stats', method: 'GET', category: 'Anomaly Statistics', variations: [ { name: 'All cloud anomaly stats', params: {} }, { name: 'AWS anomaly stats', params: { cloud_context: 'aws' } } ] }, { path: '/users', method: 'GET', category: 'User Management', variations: [ { name: 'User information', params: {} } ] }, { path: '/users/plain-sub-users', method: 'GET', category: 'MSP Management', variations: [ { name: 'All customer divisions', params: {} } ] }, { path: '/user-management/accounts', method: 'GET', category: 'Account Management', variations: [ { name: 'Cloud accounts info', params: {} } ] }, { path: '/invoices/service-names/distinct', method: 'GET', category: 'Service Discovery', variations: [ { name: 'All service names', params: {} }, { name: 'Limited service names', params: { limit: 100 } } ] }, { path: '/usage/rds/instance-costs', method: 'GET', category: 'RDS Usage Analysis', variations: [ { name: 'RDS costs last 3 months', params: { startDate: '2024-10-01', endDate: '2024-12-31' } } ] }, { path: '/usage/s3/bucket-costs', method: 'GET', category: 'S3 Usage Analysis', variations: [ { name: 'S3 costs last 3 months', params: { startDate: '2024-10-01', endDate: '2024-12-31' } } ] }, { path: '/usage/resource-explorer/distinct', method: 'GET', category: 'Resource Explorer', variations: [ { name: 'Distinct resource IDs', params: {} } ] }, { path: '/dashboards', method: 'GET', category: 'Dashboards', variations: [ { name: 'Available dashboards', params: {} } ] }, { path: '/commitment', method: 'GET', category: 'Commitment Analysis', variations: [ { name: 'All commitments', params: {} }, { name: 'Reserved instances only', params: { type: 'reserved-instances' } }, { name: 'Savings plans only', params: { type: 'savings-plans' } } ] }, { path: '/kubernetes', method: 'GET', category: 'Kubernetes', variations: [ { name: 'All Kubernetes costs', params: { startDate: '2024-11-01', endDate: '2024-12-31' } } ] }, { path: '/msp/customers', method: 'GET', category: 'MSP Customer Management', variations: [ { name: 'MSP customers list', params: {} } ] }, { path: '/mspcustomer', method: 'GET', category: 'MSP Customer Management (Alt)', variations: [ { name: 'MSP customers (alternate)', params: {} } ] }, { path: '/mspbilling-rulesv2', method: 'GET', category: 'MSP Billing Management', variations: [ { name: 'MSP billing rules', params: {} } ] }, { path: '/divisions/customers/awscredit', method: 'GET', category: 'MSP Credit Management', variations: [ { name: 'MSP customer credits', params: {} } ] } ]; class DirectApiTester { constructor() { this.results = []; this.totalTests = 0; this.successfulTests = 0; this.failedTests = 0; this.startTime = Date.now(); this.authTokens = new Map(); } log(message, level = 'info') { const timestamp = new Date().toISOString(); const levelEmoji = { info: '📋', success: '✅', error: '❌', warning: '⚠️', test: '🧪' }[level] || '📋'; console.log(`${timestamp} ${levelEmoji} ${message}`); } async makeHttpRequest(hostname, path, method = 'GET', headers = {}, postData = null) { return new Promise((resolve, reject) => { const options = { hostname, path, method, headers: { 'Content-Type': 'application/json', 'User-Agent': 'Umbrella-API-Tester/1.0', ...headers } }; if (postData && method === 'POST') { const data = JSON.stringify(postData); options.headers['Content-Length'] = Buffer.byteLength(data); } const req = https.request(options, (res) => { let responseBody = ''; res.on('data', (chunk) => { responseBody += chunk; }); res.on('end', () => { try { const parsedBody = responseBody ? JSON.parse(responseBody) : null; resolve({ statusCode: res.statusCode, headers: res.headers, body: parsedBody, rawBody: responseBody }); } catch (error) { // If not valid JSON, return raw body resolve({ statusCode: res.statusCode, headers: res.headers, body: null, rawBody: responseBody }); } }); }); req.on('error', (error) => { reject(error); }); req.setTimeout(30000, () => { req.destroy(); reject(new Error('Request timeout')); }); if (postData && method === 'POST') { req.write(JSON.stringify(postData)); } req.end(); }); } async authenticateAccount(accountConfig) { this.log(`Authenticating ${accountConfig.name}...`, 'info'); try { const authData = { username: accountConfig.username, password: accountConfig.password }; const response = await this.makeHttpRequest( accountConfig.baseUrl, '/auth/login', 'POST', {}, authData ); if (response.statusCode === 200 && response.body?.token) { const authToken = { Authorization: `Bearer ${response.body.token}`, apikey: response.body.userApiKey || '' }; this.authTokens.set(accountConfig.name, authToken); this.log(`${accountConfig.name} authentication successful`, 'success'); return { success: true, token: authToken }; } else { this.log(`${accountConfig.name} authentication failed: ${response.statusCode} - ${response.rawBody}`, 'error'); return { success: false }; } } catch (error) { this.log(`${accountConfig.name} authentication error: ${error.message}`, 'error'); return { success: false }; } } async testEndpoint(accountConfig, authToken, endpoint, variation, customerConfig = null) { const testName = `${accountConfig.name} - ${endpoint.category} - ${variation.name}${customerConfig ? ` (${customerConfig.name})` : ''}`; this.log(`Testing: ${testName}`, 'test'); this.totalTests++; try { // Prepare parameters let params = { ...variation.params }; // Add MSP customer parameters if applicable if (customerConfig && accountConfig.accountType === 'msp') { // Add customer context for MSP accounts params.customer_account_key = customerConfig.customer_account_key; params.customer_division_id = customerConfig.customer_division_id; // Add account ID for certain endpoints if (endpoint.path === '/invoices/caui' && customerConfig.accountId) { params.accountId = customerConfig.accountId; params.divisionId = customerConfig.customer_division_id; } } // Convert boolean and array parameters for URL encoding const urlParams = {}; for (const [key, value] of Object.entries(params)) { if (Array.isArray(value)) { urlParams[key] = JSON.stringify(value); } else if (typeof value === 'boolean') { urlParams[key] = value.toString(); } else { urlParams[key] = value; } } // Build request path let requestPath = endpoint.path; if (endpoint.method === 'GET' && Object.keys(urlParams).length > 0) { requestPath += '?' + querystring.stringify(urlParams); } // Prepare headers const headers = { ...authToken }; // Make request const startTime = Date.now(); const response = await this.makeHttpRequest( accountConfig.baseUrl, requestPath, endpoint.method, headers, endpoint.method === 'POST' ? params : null ); const duration = Date.now() - startTime; // Analyze result const success = response.statusCode >= 200 && response.statusCode < 300; const hasData = response.body && (Array.isArray(response.body) ? response.body.length > 0 : Object.keys(response.body).length > 0); const responseSize = response.rawBody ? response.rawBody.length : 0; if (success) { this.successfulTests++; this.log(`✅ ${testName} - ${response.statusCode} - ${duration}ms - ${responseSize}bytes ${hasData ? '(with data)' : '(empty)'}`, 'success'); } else { this.failedTests++; const errorMsg = response.body?.message || response.body?.error || response.rawBody?.substring(0, 200) || 'Unknown error'; this.log(`❌ ${testName} - ${response.statusCode} - ${duration}ms - ${errorMsg}`, 'error'); } // Store result this.results.push({ account: accountConfig.name, customer: customerConfig?.name || 'Direct', endpoint: endpoint.path, category: endpoint.category, variation: variation.name, method: endpoint.method, params, success, statusCode: response.statusCode, hasData, duration, responseSize, errorMessage: success ? null : (response.body?.message || response.body?.error || response.rawBody?.substring(0, 500)), timestamp: new Date().toISOString(), sampleData: hasData && response.body ? JSON.stringify(response.body).substring(0, 1000) : null }); // Small delay to avoid overwhelming the API await new Promise(resolve => setTimeout(resolve, 200)); } catch (error) { this.failedTests++; this.log(`❌ ${testName} - Network Error: ${error.message}`, 'error'); this.results.push({ account: accountConfig.name, customer: customerConfig?.name || 'Direct', endpoint: endpoint.path, category: endpoint.category, variation: variation.name, method: endpoint.method, params, success: false, statusCode: 0, hasData: false, duration: 0, responseSize: 0, errorMessage: error.message, timestamp: new Date().toISOString(), sampleData: null }); } } async runComprehensiveTest() { this.log('🚀 Starting Comprehensive Direct API Test', 'info'); this.log(`Testing ${ENDPOINTS.length} endpoints with multiple variations`, 'info'); for (const [accountType, accountConfig] of Object.entries(ACCOUNT_CONFIGS)) { this.log(`\n🏢 Testing ${accountConfig.name}...`, 'info'); // Authenticate const { success, token } = await this.authenticateAccount(accountConfig); if (!success) { this.log(`Skipping ${accountConfig.name} due to authentication failure`, 'warning'); continue; } // Test endpoints for (const endpoint of ENDPOINTS) { this.log(`\n📂 Testing ${endpoint.category} (${endpoint.path})`, 'info'); for (const variation of endpoint.variations) { if (accountConfig.accountType === 'msp') { // Test with different MSP customers for (const customer of MSP_CUSTOMERS) { await this.testEndpoint(accountConfig, token, endpoint, variation, customer); } } else { // Test direct customer await this.testEndpoint(accountConfig, token, endpoint, variation); } } } } this.generateReport(); } generateReport() { const endTime = Date.now(); const totalDuration = endTime - this.startTime; this.log('\n📊 COMPREHENSIVE API TEST RESULTS', 'info'); this.log(`Total Tests: ${this.totalTests}`, 'info'); this.log(`Successful: ${this.successfulTests} (${((this.successfulTests/this.totalTests)*100).toFixed(1)}%)`, 'success'); this.log(`Failed: ${this.failedTests} (${((this.failedTests/this.totalTests)*100).toFixed(1)}%)`, 'error'); this.log(`Total Duration: ${(totalDuration/1000).toFixed(1)}s`, 'info'); // Generate detailed reports this.generateSummaryReport(); this.generateDetailedReport(); this.generateAccountComparison(); this.generateEndpointAnalysis(); this.generateErrorAnalysis(); } generateSummaryReport() { const summaryReport = { testRun: { startTime: new Date(this.startTime).toISOString(), endTime: new Date().toISOString(), duration: Date.now() - this.startTime, totalTests: this.totalTests, successfulTests: this.successfulTests, failedTests: this.failedTests, successRate: ((this.successfulTests/this.totalTests)*100).toFixed(1) }, accountResults: {}, endpointResults: {}, categoryResults: {} }; // Account results for (const accountType of Object.keys(ACCOUNT_CONFIGS)) { const accountName = ACCOUNT_CONFIGS[accountType].name; const accountTests = this.results.filter(r => r.account === accountName); summaryReport.accountResults[accountName] = { totalTests: accountTests.length, successful: accountTests.filter(r => r.success).length, failed: accountTests.filter(r => !r.success).length, withData: accountTests.filter(r => r.hasData).length, avgDuration: accountTests.reduce((sum, r) => sum + r.duration, 0) / accountTests.length, totalResponseSize: accountTests.reduce((sum, r) => sum + r.responseSize, 0), statusCodes: this.getStatusCodeDistribution(accountTests) }; } // Endpoint results for (const endpoint of ENDPOINTS) { const endpointTests = this.results.filter(r => r.endpoint === endpoint.path); summaryReport.endpointResults[endpoint.path] = { category: endpoint.category, method: endpoint.method, totalTests: endpointTests.length, successful: endpointTests.filter(r => r.success).length, failed: endpointTests.filter(r => !r.success).length, withData: endpointTests.filter(r => r.hasData).length, avgDuration: endpointTests.reduce((sum, r) => sum + r.duration, 0) / endpointTests.length, statusCodes: this.getStatusCodeDistribution(endpointTests) }; } // Category results const categories = [...new Set(ENDPOINTS.map(e => e.category))]; for (const category of categories) { const categoryTests = this.results.filter(r => r.category === category); summaryReport.categoryResults[category] = { totalTests: categoryTests.length, successful: categoryTests.filter(r => r.success).length, failed: categoryTests.filter(r => !r.success).length, withData: categoryTests.filter(r => r.hasData).length, avgDuration: categoryTests.reduce((sum, r) => sum + r.duration, 0) / categoryTests.length }; } // Save summary report const summaryPath = path.join(__dirname, `api-test-summary-${Date.now()}.json`); fs.writeFileSync(summaryPath, JSON.stringify(summaryReport, null, 2)); this.log(`Summary report saved: ${summaryPath}`, 'success'); } generateDetailedReport() { // Save detailed results const detailedPath = path.join(__dirname, `api-test-results-detailed-${Date.now()}.json`); fs.writeFileSync(detailedPath, JSON.stringify(this.results, null, 2)); this.log(`Detailed results saved: ${detailedPath}`, 'success'); // Generate CSV for easy analysis const csvHeaders = [ 'Account', 'Customer', 'Endpoint', 'Category', 'Method', 'Variation', 'Success', 'StatusCode', 'HasData', 'Duration(ms)', 'ResponseSize(bytes)', 'ErrorMessage', 'Timestamp' ]; const csvRows = this.results.map(r => [ r.account, r.customer, r.endpoint, r.category, r.method, r.variation, r.success, r.statusCode, r.hasData, r.duration, r.responseSize, (r.errorMessage || '').replace(/"/g, '""'), r.timestamp ]); const csvContent = [csvHeaders.join(','), ...csvRows.map(row => row.map(cell => `"${cell}"`).join(',') )].join('\n'); const csvPath = path.join(__dirname, `api-test-results-${Date.now()}.csv`); fs.writeFileSync(csvPath, csvContent); this.log(`CSV results saved: ${csvPath}`, 'success'); } generateAccountComparison() { this.log('\n🔄 ACCOUNT COMPARISON', 'info'); for (const [accountType, accountConfig] of Object.entries(ACCOUNT_CONFIGS)) { const accountTests = this.results.filter(r => r.account === accountConfig.name); const successful = accountTests.filter(r => r.success).length; const withData = accountTests.filter(r => r.hasData).length; const avgResponseTime = accountTests.reduce((sum, r) => sum + r.duration, 0) / accountTests.length; this.log(`${accountConfig.name}:`, 'info'); this.log(` Tests: ${accountTests.length}`, 'info'); this.log(` Success: ${successful}/${accountTests.length} (${((successful/accountTests.length)*100).toFixed(1)}%)`, 'info'); this.log(` With Data: ${withData}/${accountTests.length} (${((withData/accountTests.length)*100).toFixed(1)}%)`, 'info'); this.log(` Avg Response Time: ${Math.round(avgResponseTime)}ms`, 'info'); // Show status code distribution const statusCodes = this.getStatusCodeDistribution(accountTests); this.log(` Status Codes: ${Object.entries(statusCodes).map(([code, count]) => `${code}(${count})`).join(', ')}`, 'info'); } } generateEndpointAnalysis() { this.log('\n🔍 ENDPOINT ANALYSIS', 'info'); // Group by endpoint const endpointGroups = {}; for (const result of this.results) { if (!endpointGroups[result.endpoint]) { endpointGroups[result.endpoint] = []; } endpointGroups[result.endpoint].push(result); } // Analyze each endpoint const endpointAnalysis = []; for (const [endpoint, tests] of Object.entries(endpointGroups)) { const successful = tests.filter(r => r.success).length; const withData = tests.filter(r => r.hasData).length; const avgDuration = tests.reduce((sum, r) => sum + r.duration, 0) / tests.length; const avgResponseSize = tests.reduce((sum, r) => sum + r.responseSize, 0) / tests.length; endpointAnalysis.push({ endpoint, category: tests[0].category, method: tests[0].method, totalTests: tests.length, successRate: ((successful/tests.length)*100).toFixed(1), dataRate: ((withData/tests.length)*100).toFixed(1), avgDuration: Math.round(avgDuration), avgResponseSize: Math.round(avgResponseSize), statusCodes: this.getStatusCodeDistribution(tests), commonErrors: this.getCommonErrors(tests) }); } // Sort by success rate endpointAnalysis.sort((a, b) => parseFloat(b.successRate) - parseFloat(a.successRate)); this.log('Top performing endpoints:', 'success'); endpointAnalysis.slice(0, 10).forEach(ep => { this.log(` ${ep.endpoint} (${ep.category}): ${ep.successRate}% success, ${ep.dataRate}% with data, ~${ep.avgResponseSize}b`, 'info'); }); this.log('\nProblematic endpoints:', 'error'); endpointAnalysis.slice(-10).forEach(ep => { this.log(` ${ep.endpoint} (${ep.category}): ${ep.successRate}% success`, 'info'); if (ep.commonErrors.length > 0) { this.log(` Common errors: ${ep.commonErrors[0]}`, 'warning'); } }); // Save endpoint analysis const analysisPath = path.join(__dirname, `api-endpoint-analysis-${Date.now()}.json`); fs.writeFileSync(analysisPath, JSON.stringify(endpointAnalysis, null, 2)); this.log(`\nEndpoint analysis saved: ${analysisPath}`, 'success'); } generateErrorAnalysis() { this.log('\n🚨 ERROR ANALYSIS', 'info'); const errors = this.results.filter(r => !r.success); const errorsByStatus = {}; const errorsByEndpoint = {}; errors.forEach(error => { // Group by status code if (!errorsByStatus[error.statusCode]) { errorsByStatus[error.statusCode] = []; } errorsByStatus[error.statusCode].push(error); // Group by endpoint if (!errorsByEndpoint[error.endpoint]) { errorsByEndpoint[error.endpoint] = []; } errorsByEndpoint[error.endpoint].push(error); }); this.log(`Total Errors: ${errors.length}`, 'error'); // Most common error codes this.log('\nMost common error codes:', 'warning'); Object.entries(errorsByStatus) .sort(([,a], [,b]) => b.length - a.length) .slice(0, 5) .forEach(([statusCode, errorList]) => { this.log(` ${statusCode}: ${errorList.length} occurrences`, 'info'); if (errorList[0].errorMessage) { this.log(` Example: ${errorList[0].errorMessage.substring(0, 100)}...`, 'warning'); } }); // Most problematic endpoints this.log('\nMost problematic endpoints:', 'warning'); Object.entries(errorsByEndpoint) .sort(([,a], [,b]) => b.length - a.length) .slice(0, 5) .forEach(([endpoint, errorList]) => { this.log(` ${endpoint}: ${errorList.length} errors`, 'info'); }); } getStatusCodeDistribution(tests) { const distribution = {}; tests.forEach(test => { distribution[test.statusCode] = (distribution[test.statusCode] || 0) + 1; }); return distribution; } getCommonErrors(tests) { const errors = tests.filter(t => !t.success && t.errorMessage); const errorCounts = {}; errors.forEach(error => { const truncatedError = error.errorMessage.substring(0, 100); errorCounts[truncatedError] = (errorCounts[truncatedError] || 0) + 1; }); return Object.entries(errorCounts) .sort(([,a], [,b]) => b - a) .slice(0, 3) .map(([error]) => error); } } // Main execution async function main() { console.log('🧪 Comprehensive Direct API Tester'); console.log('==================================='); // Check environment variables let missingEnvVars = []; if (!process.env.SAOLA_PASSWORD) missingEnvVars.push('SAOLA_PASSWORD'); if (!process.env.ALLCLOUD_PASSWORD) missingEnvVars.push('ALLCLOUD_PASSWORD'); if (missingEnvVars.length > 0) { console.log('⚠️ Missing environment variables:'); missingEnvVars.forEach(envVar => { console.log(` export ${envVar}="your_password_here"`); }); console.log('\nYou can still run the test by setting passwords inline or modifying the script.'); } const tester = new DirectApiTester(); await tester.runComprehensiveTest(); console.log('\n🎉 Comprehensive direct API testing completed!'); console.log('Check the generated reports for detailed analysis.'); } // Run if called directly if (require.main === module) { main().catch(error => { console.error('❌ Test runner error:', error); process.exit(1); }); } module.exports = { DirectApiTester, ENDPOINTS, ACCOUNT_CONFIGS, MSP_CUSTOMERS };