Umbrella MCP Server

#!/usr/bin/env node /** * Comprehensive Endpoint Test Script * Tests all MCP server endpoints with various parameter combinations * for both SAOLA and AllCloud accounts automatically */ const fs = require('fs'); const path = require('path'); // Configuration for different account types const ACCOUNT_CONFIGS = { SAOLA: { name: 'SAOLA (Direct Customer)', username: process.env.SAOLA_USERNAME || 'david+saola@umbrellacost.com', password: process.env.SAOLA_PASSWORD || '', accountType: 'direct' }, ALLCLOUD: { name: 'AllCloud (MSP Customer)', username: process.env.ALLCLOUD_USERNAME || 'david+allcloud@umbrellacost.com', password: process.env.ALLCLOUD_PASSWORD || '', accountType: 'msp' } }; // MSP customer test variations for AllCloud account const MSP_CUSTOMERS = [ { name: 'Bank Leumi', customer_account_key: '22676', customer_division_id: '139', accountId: '696314371547' }, { name: 'Bank Hapoalim', customer_account_key: '16185', customer_division_id: '1', accountId: '123456789012' // example } ]; // All available endpoints with their parameter variations const ENDPOINTS = [ { path: '/invoices/caui', method: 'GET', category: 'Cost Analysis', variations: [ // Basic cost queries { name: 'Basic unblended costs', params: { groupBy: 'none', periodGranLevel: 'month', costType: ['cost', 'discount'], isUnblended: true, startDate: '2024-01-01', endDate: '2024-12-31' } }, { name: 'Amortized costs', params: { groupBy: 'service', periodGranLevel: 'month', costType: ['cost', 'discount'], isAmortized: true, startDate: '2024-01-01', endDate: '2024-12-31' } }, { name: 'Net amortized costs', params: { groupBy: 'account', periodGranLevel: 'month', costType: ['cost', 'discount'], isNetAmortized: true, startDate: '2024-01-01', endDate: '2024-12-31' } }, // Service-specific queries { name: 'EC2 costs', params: { service: 'Amazon Elastic Compute Cloud', groupBy: 'none', periodGranLevel: 'day', costType: ['cost'], isUnblended: true, startDate: '2024-11-01', endDate: '2024-11-30' } }, { name: 'S3 costs', params: { service: 'Amazon Simple Storage Service', groupBy: 'region', periodGranLevel: 'day', costType: ['cost', 'discount'], isUnblended: true, startDate: '2024-11-01', endDate: '2024-11-30' } }, // Different time periods { name: 'Daily breakdown current month', params: { groupBy: 'service', periodGranLevel: 'day', costType: ['cost', 'discount'], isUnblended: true, startDate: '2024-12-01', endDate: '2024-12-31' } } ] }, { path: '/budgets/v2/i/', method: 'GET', category: 'Budget Management', variations: [ { name: 'All budgets metadata', params: { only_metadata: true } }, { name: 'Full budget details', params: { only_metadata: false } }, { name: 'AWS budgets only', params: { cloud_context: 'aws', only_metadata: true } }, { name: 'Azure budgets only', params: { cloud_context: 'azure', only_metadata: true } }, { name: 'GCP budgets only', params: { cloud_context: 'gcp', only_metadata: true } } ] }, { path: '/recommendationsNew/heatmap/summary', method: 'POST', category: 'Recommendations', variations: [ { name: 'All recommendations summary', params: {} } ] }, { path: '/recommendations/report', method: 'GET', category: 'Recommendations (Legacy)', variations: [ { name: 'Legacy recommendations report', params: {} } ] }, { path: '/anomaly-detection', method: 'GET', category: 'Anomaly Detection', variations: [ { name: 'All anomalies', params: { isFull: true } }, { name: 'Recent anomalies', params: { startDate: '2024-11-01', endDate: '2024-12-31', isFull: true } }, { name: 'Only alerted anomalies', params: { alerted: true, isFull: true } }, { name: 'AWS anomalies', params: { cloud_context: 'aws', isFull: true } } ] }, { path: '/anomalies/stats', method: 'GET', category: 'Anomaly Statistics', variations: [ { name: 'All cloud anomaly stats', params: {} }, { name: 'AWS anomaly stats', params: { cloud_context: 'aws' } } ] }, { path: '/users', method: 'GET', category: 'User Management', variations: [ { name: 'User information', params: {} } ] }, { path: '/users/plain-sub-users', method: 'GET', category: 'MSP Management', variations: [ { name: 'All customer divisions', params: {} }, { name: 'Bank Leumi customer search', params: { userQuery: 'Bank Leumi' } }, { name: 'Bank Hapoalim customer search', params: { userQuery: 'Bank Hapoalim' } } ] }, { path: '/user-management/accounts', method: 'GET', category: 'Account Management', variations: [ { name: 'Cloud accounts info', params: {} } ] }, { path: '/invoices/service-names/distinct', method: 'GET', category: 'Service Discovery', variations: [ { name: 'All service names', params: {} }, { name: 'Limited service names', params: { limit: 100 } } ] }, { path: '/usage/rds/instance-costs', method: 'GET', category: 'RDS Usage Analysis', variations: [ { name: 'RDS costs last 3 months', params: { startDate: '2024-10-01', endDate: '2024-12-31' } } ] }, { path: '/usage/s3/bucket-costs', method: 'GET', category: 'S3 Usage Analysis', variations: [ { name: 'S3 costs last 3 months', params: { startDate: '2024-10-01', endDate: '2024-12-31' } } ] }, { path: '/usage/resource-explorer/distinct', method: 'GET', category: 'Resource Explorer', variations: [ { name: 'Distinct resource IDs', params: {} } ] }, { path: '/dashboards', method: 'GET', category: 'Dashboards', variations: [ { name: 'Available dashboards', params: {} } ] }, { path: '/commitment', method: 'GET', category: 'Commitment Analysis', variations: [ { name: 'All commitments', params: {} }, { name: 'Reserved instances only', params: { type: 'reserved-instances' } }, { name: 'Savings plans only', params: { type: 'savings-plans' } } ] }, { path: '/kubernetes', method: 'GET', category: 'Kubernetes', variations: [ { name: 'All Kubernetes costs', params: { startDate: '2024-11-01', endDate: '2024-12-31' } } ] }, { path: '/msp/customers', method: 'GET', category: 'MSP Customer Management', variations: [ { name: 'MSP customers list', params: {} } ] }, { path: '/mspcustomer', method: 'GET', category: 'MSP Customer Management (Alt)', variations: [ { name: 'MSP customers (alternate)', params: {} } ] }, { path: '/mspbilling-rulesv2', method: 'GET', category: 'MSP Billing Management', variations: [ { name: 'MSP billing rules', params: {} } ] }, { path: '/divisions/customers/awscredit', method: 'GET', category: 'MSP Credit Management', variations: [ { name: 'MSP customer credits', params: {} } ] } ]; class EndpointTester { constructor() { this.results = []; this.totalTests = 0; this.successfulTests = 0; this.failedTests = 0; this.startTime = Date.now(); } log(message, level = 'info') { const timestamp = new Date().toISOString(); const levelEmoji = { info: '📋', success: '✅', error: '❌', warning: '⚠️', test: '🧪' }[level] || '📋'; console.log(`${timestamp} ${levelEmoji} ${message}`); } async authenticateAccount(accountConfig) { this.log(`Authenticating ${accountConfig.name}...`, 'info'); try { // Import the MCP client helper const { createMcpClient } = await import('../../test-helpers/mcp-client.mjs'); const client = await createMcpClient(); // Authenticate const authResult = await client.callTool('authenticate_user', { username: accountConfig.username, password: accountConfig.password }); if (authResult.content?.[0]?.text?.includes('✅')) { this.log(`${accountConfig.name} authentication successful`, 'success'); return { client, success: true }; } else { this.log(`${accountConfig.name} authentication failed: ${authResult.content?.[0]?.text}`, 'error'); return { client: null, success: false }; } } catch (error) { this.log(`${accountConfig.name} authentication error: ${error.message}`, 'error'); return { client: null, success: false }; } } async testEndpoint(client, accountConfig, endpoint, variation, customerConfig = null) { const testName = `${accountConfig.name} - ${endpoint.category} - ${variation.name}${customerConfig ? ` (${customerConfig.name})` : ''}`; this.log(`Testing: ${testName}`, 'test'); this.totalTests++; try { // Prepare parameters let params = { ...variation.params }; // Add MSP customer parameters if applicable if (customerConfig && accountConfig.accountType === 'msp') { // Add customer context for MSP accounts params.customer_account_key = customerConfig.customer_account_key; params.customer_division_id = customerConfig.customer_division_id; // Add user query for customer detection if (endpoint.path.includes('recommendations') || endpoint.path === '/invoices/caui') { params.userQuery = `Show me ${customerConfig.name} ${endpoint.category.toLowerCase()}`; } // Add account ID for certain endpoints if (endpoint.path === '/invoices/caui' && customerConfig.accountId) { params.accountId = customerConfig.accountId; params.divisionId = customerConfig.customer_division_id; } } // Create the tool name from the endpoint path const toolName = `api__${endpoint.path.replace(/^\//, '').replace(/\//g, '_').replace(/[-]/g, '_')}`; // Call the endpoint const startTime = Date.now(); const result = await client.callTool(toolName, params); const duration = Date.now() - startTime; // Analyze result const success = !result.content?.[0]?.text?.includes('❌'); const responseText = result.content?.[0]?.text || ''; const hasData = responseText.includes('```json') || responseText.includes('Results:') || responseText.includes('Budget'); if (success) { this.successfulTests++; this.log(`✅ ${testName} - ${duration}ms ${hasData ? '(with data)' : '(empty)'}`, 'success'); } else { this.failedTests++; this.log(`❌ ${testName} - ${duration}ms - ${responseText.substring(0, 200)}...`, 'error'); } // Store result this.results.push({ account: accountConfig.name, customer: customerConfig?.name || 'Direct', endpoint: endpoint.path, category: endpoint.category, variation: variation.name, toolName, params, success, hasData, duration, responseLength: responseText.length, errorMessage: success ? null : responseText.substring(0, 500), timestamp: new Date().toISOString() }); // Small delay to avoid overwhelming the API await new Promise(resolve => setTimeout(resolve, 100)); } catch (error) { this.failedTests++; this.log(`❌ ${testName} - Error: ${error.message}`, 'error'); this.results.push({ account: accountConfig.name, customer: customerConfig?.name || 'Direct', endpoint: endpoint.path, category: endpoint.category, variation: variation.name, toolName: `api__${endpoint.path.replace(/^\//, '').replace(/\//g, '_').replace(/[-]/g, '_')}`, params, success: false, hasData: false, duration: 0, responseLength: 0, errorMessage: error.message, timestamp: new Date().toISOString() }); } } async runComprehensiveTest() { this.log('🚀 Starting Comprehensive Endpoint Test', 'info'); this.log(`Testing ${ENDPOINTS.length} endpoints with multiple variations`, 'info'); for (const [accountType, accountConfig] of Object.entries(ACCOUNT_CONFIGS)) { this.log(`\n🏢 Testing ${accountConfig.name}...`, 'info'); // Authenticate const { client, success } = await this.authenticateAccount(accountConfig); if (!success) { this.log(`Skipping ${accountConfig.name} due to authentication failure`, 'warning'); continue; } // Test endpoints for (const endpoint of ENDPOINTS) { this.log(`\n📂 Testing ${endpoint.category} (${endpoint.path})`, 'info'); for (const variation of endpoint.variations) { if (accountConfig.accountType === 'msp') { // Test with different MSP customers for (const customer of MSP_CUSTOMERS) { await this.testEndpoint(client, accountConfig, endpoint, variation, customer); } } else { // Test direct customer await this.testEndpoint(client, accountConfig, endpoint, variation); } } } // Logout try { await client.callTool('logout', {}); this.log(`${accountConfig.name} logged out`, 'info'); } catch (error) { this.log(`${accountConfig.name} logout error: ${error.message}`, 'warning'); } } this.generateReport(); } generateReport() { const endTime = Date.now(); const totalDuration = endTime - this.startTime; this.log('\n📊 COMPREHENSIVE TEST RESULTS', 'info'); this.log(`Total Tests: ${this.totalTests}`, 'info'); this.log(`Successful: ${this.successfulTests} (${((this.successfulTests/this.totalTests)*100).toFixed(1)}%)`, 'success'); this.log(`Failed: ${this.failedTests} (${((this.failedTests/this.totalTests)*100).toFixed(1)}%)`, 'error'); this.log(`Total Duration: ${(totalDuration/1000).toFixed(1)}s`, 'info'); // Generate detailed reports this.generateSummaryReport(); this.generateDetailedReport(); this.generateAccountComparison(); this.generateEndpointAnalysis(); } generateSummaryReport() { const summaryReport = { testRun: { startTime: new Date(this.startTime).toISOString(), endTime: new Date().toISOString(), duration: Date.now() - this.startTime, totalTests: this.totalTests, successfulTests: this.successfulTests, failedTests: this.failedTests, successRate: ((this.successfulTests/this.totalTests)*100).toFixed(1) }, accountResults: {}, endpointResults: {}, categoryResults: {} }; // Account results for (const accountType of Object.keys(ACCOUNT_CONFIGS)) { const accountName = ACCOUNT_CONFIGS[accountType].name; const accountTests = this.results.filter(r => r.account === accountName); summaryReport.accountResults[accountName] = { totalTests: accountTests.length, successful: accountTests.filter(r => r.success).length, failed: accountTests.filter(r => !r.success).length, withData: accountTests.filter(r => r.hasData).length, avgDuration: accountTests.reduce((sum, r) => sum + r.duration, 0) / accountTests.length }; } // Endpoint results for (const endpoint of ENDPOINTS) { const endpointTests = this.results.filter(r => r.endpoint === endpoint.path); summaryReport.endpointResults[endpoint.path] = { category: endpoint.category, totalTests: endpointTests.length, successful: endpointTests.filter(r => r.success).length, failed: endpointTests.filter(r => !r.success).length, withData: endpointTests.filter(r => r.hasData).length, avgDuration: endpointTests.reduce((sum, r) => sum + r.duration, 0) / endpointTests.length }; } // Category results const categories = [...new Set(ENDPOINTS.map(e => e.category))]; for (const category of categories) { const categoryTests = this.results.filter(r => r.category === category); summaryReport.categoryResults[category] = { totalTests: categoryTests.length, successful: categoryTests.filter(r => r.success).length, failed: categoryTests.filter(r => !r.success).length, withData: categoryTests.filter(r => r.hasData).length, avgDuration: categoryTests.reduce((sum, r) => sum + r.duration, 0) / categoryTests.length }; } // Save summary report const summaryPath = path.join(__dirname, `test-summary-${Date.now()}.json`); fs.writeFileSync(summaryPath, JSON.stringify(summaryReport, null, 2)); this.log(`Summary report saved: ${summaryPath}`, 'success'); } generateDetailedReport() { // Save detailed results const detailedPath = path.join(__dirname, `test-results-detailed-${Date.now()}.json`); fs.writeFileSync(detailedPath, JSON.stringify(this.results, null, 2)); this.log(`Detailed results saved: ${detailedPath}`, 'success'); // Generate CSV for easy analysis const csvHeaders = [ 'Account', 'Customer', 'Endpoint', 'Category', 'Variation', 'Success', 'HasData', 'Duration(ms)', 'ResponseLength', 'ErrorMessage', 'Timestamp' ]; const csvRows = this.results.map(r => [ r.account, r.customer, r.endpoint, r.category, r.variation, r.success, r.hasData, r.duration, r.responseLength, (r.errorMessage || '').replace(/"/g, '""'), r.timestamp ]); const csvContent = [csvHeaders.join(','), ...csvRows.map(row => row.map(cell => `"${cell}"`).join(',') )].join('\n'); const csvPath = path.join(__dirname, `test-results-${Date.now()}.csv`); fs.writeFileSync(csvPath, csvContent); this.log(`CSV results saved: ${csvPath}`, 'success'); } generateAccountComparison() { this.log('\n🔄 ACCOUNT COMPARISON', 'info'); for (const [accountType, accountConfig] of Object.entries(ACCOUNT_CONFIGS)) { const accountTests = this.results.filter(r => r.account === accountConfig.name); const successful = accountTests.filter(r => r.success).length; const withData = accountTests.filter(r => r.hasData).length; this.log(`${accountConfig.name}:`, 'info'); this.log(` Tests: ${accountTests.length}`, 'info'); this.log(` Success: ${successful}/${accountTests.length} (${((successful/accountTests.length)*100).toFixed(1)}%)`, 'info'); this.log(` With Data: ${withData}/${accountTests.length} (${((withData/accountTests.length)*100).toFixed(1)}%)`, 'info'); } } generateEndpointAnalysis() { this.log('\n🔍 ENDPOINT ANALYSIS', 'info'); // Group by endpoint const endpointGroups = {}; for (const result of this.results) { if (!endpointGroups[result.endpoint]) { endpointGroups[result.endpoint] = []; } endpointGroups[result.endpoint].push(result); } // Analyze each endpoint const endpointAnalysis = []; for (const [endpoint, tests] of Object.entries(endpointGroups)) { const successful = tests.filter(r => r.success).length; const withData = tests.filter(r => r.hasData).length; const avgDuration = tests.reduce((sum, r) => sum + r.duration, 0) / tests.length; endpointAnalysis.push({ endpoint, category: tests[0].category, totalTests: tests.length, successRate: ((successful/tests.length)*100).toFixed(1), dataRate: ((withData/tests.length)*100).toFixed(1), avgDuration: Math.round(avgDuration), issues: tests.filter(r => !r.success).map(r => r.errorMessage).slice(0, 3) }); } // Sort by success rate endpointAnalysis.sort((a, b) => parseFloat(b.successRate) - parseFloat(a.successRate)); this.log('Top performing endpoints:', 'success'); endpointAnalysis.slice(0, 10).forEach(ep => { this.log(` ${ep.endpoint} (${ep.category}): ${ep.successRate}% success, ${ep.dataRate}% with data`, 'info'); }); this.log('\nProblematic endpoints:', 'error'); endpointAnalysis.slice(-10).forEach(ep => { this.log(` ${ep.endpoint} (${ep.category}): ${ep.successRate}% success`, 'info'); if (ep.issues.length > 0) { this.log(` Issues: ${ep.issues[0]}`, 'warning'); } }); // Save endpoint analysis const analysisPath = path.join(__dirname, `endpoint-analysis-${Date.now()}.json`); fs.writeFileSync(analysisPath, JSON.stringify(endpointAnalysis, null, 2)); this.log(`\nEndpoint analysis saved: ${analysisPath}`, 'success'); } } // Main execution async function main() { console.log('🧪 Comprehensive MCP Endpoint Tester'); console.log('====================================='); // Check environment variables let missingEnvVars = []; if (!process.env.SAOLA_PASSWORD) missingEnvVars.push('SAOLA_PASSWORD'); if (!process.env.ALLCLOUD_PASSWORD) missingEnvVars.push('ALLCLOUD_PASSWORD'); if (missingEnvVars.length > 0) { console.log('⚠️ Missing environment variables:'); missingEnvVars.forEach(envVar => { console.log(` export ${envVar}="your_password_here"`); }); console.log('\nYou can still run the test by setting passwords inline or modifying the script.'); } const tester = new EndpointTester(); await tester.runComprehensiveTest(); console.log('\n🎉 Comprehensive endpoint testing completed!'); console.log('Check the generated reports for detailed analysis.'); } // Run if called directly if (require.main === module) { main().catch(error => { console.error('❌ Test runner error:', error); process.exit(1); }); } module.exports = { EndpointTester, ENDPOINTS, ACCOUNT_CONFIGS, MSP_CUSTOMERS };