Umbrella MCP Server

#!/usr/bin/env node /** * MCP Endpoint Test Script * Tests all MCP server endpoints with various parameter combinations * Uses the existing built MCP server via stdio transport */ const { spawn } = require('child_process'); const path = require('path'); const fs = require('fs'); // Test account configurations (you need to set the passwords) const ACCOUNTS = { SAOLA: { name: 'SAOLA', username: 'david+saola@umbrellacost.com', password: process.env.SAOLA_PASSWORD || '' }, ALLCLOUD: { name: 'AllCloud', username: 'david+allcloud@umbrellacost.com', password: process.env.ALLCLOUD_PASSWORD || '' } }; // MSP customer variations const MSP_CUSTOMERS = [ { name: 'Bank Leumi', accountKey: '22676', divisionId: '139' }, { name: 'Bank Hapoalim', accountKey: '16185', divisionId: '1' } ]; // Endpoint test variations const ENDPOINT_TESTS = [ // Cost Analysis { tool: 'api__invoices_caui', name: 'Basic costs - unblended', params: { groupBy: 'none', periodGranLevel: 'month', costType: 'cost,discount', isUnblended: 'true', startDate: '2024-01-01', endDate: '2024-12-31' } }, { tool: 'api__invoices_caui', name: 'Service breakdown - amortized', params: { groupBy: 'service', periodGranLevel: 'month', costType: 'cost,discount', isAmortized: 'true', startDate: '2024-01-01', endDate: '2024-12-31' } }, { tool: 'api__invoices_caui', name: 'EC2 costs only', params: { service: 'Amazon Elastic Compute Cloud', groupBy: 'region', periodGranLevel: 'day', costType: 'cost', isUnblended: 'true', startDate: '2024-11-01', endDate: '2024-11-30' } }, // Budget Management { tool: 'api__budgets_v2_i_', name: 'All budgets metadata', params: { only_metadata: 'true' } }, { tool: 'api__budgets_v2_i_', name: 'AWS budgets only', params: { cloud_context: 'aws', only_metadata: 'true' } }, // Recommendations { tool: 'api__recommendationsNew_heatmap_summary', name: 'Recommendations summary', params: {} }, // User Management { tool: 'api__users', name: 'User information', params: {} }, { tool: 'api__users_plain_sub_users', name: 'Customer divisions', params: {} }, // Anomaly Detection { tool: 'api__anomaly_detection', name: 'Recent anomalies', params: { startDate: '2024-11-01', endDate: '2024-12-31', isFull: 'true' } }, // Service Discovery { tool: 'api__invoices_service_names_distinct', name: 'Available services (limited)', params: { limit: '50' } } ]; class McpTester { constructor() { this.results = []; this.totalTests = 0; this.successCount = 0; this.failCount = 0; this.startTime = Date.now(); } log(message, level = 'info') { const timestamp = new Date().toISOString().split('.')[0]; const emoji = { info: '📋', success: '✅', error: '❌', test: '🧪', warning: '⚠️' }[level] || '📋'; console.log(`${timestamp} ${emoji} ${message}`); } async createMcpClient() { return new Promise((resolve, reject) => { // Start MCP server const serverProcess = spawn('node', ['dist/index.js'], { stdio: ['pipe', 'pipe', 'pipe'], cwd: process.cwd() }); let buffer = ''; const pendingRequests = new Map(); let requestId = 1; let initialized = false; // Handle server responses serverProcess.stdout.on('data', (data) => { buffer += data.toString(); const lines = buffer.split('\n'); buffer = lines.pop() || ''; lines.forEach(line => { if (line.trim()) { try { const response = JSON.parse(line); if (response.id && pendingRequests.has(response.id)) { const { resolve: resolveReq, reject: rejectReq } = pendingRequests.get(response.id); pendingRequests.delete(response.id); if (response.error) { rejectReq(new Error(response.error.message || JSON.stringify(response.error))); } else { resolveReq(response.result); } } } catch (e) { // Ignore malformed JSON } } }); }); // Handle server errors serverProcess.stderr.on('data', (data) => { if (process.env.DEBUG) { console.error('[MCP-SERVER]', data.toString()); } }); const sendRequest = (method, params = {}) => { return new Promise((resolveReq, rejectReq) => { const id = requestId++; const request = { jsonrpc: '2.0', id, method, params }; pendingRequests.set(id, { resolve: resolveReq, reject: rejectReq }); serverProcess.stdin.write(JSON.stringify(request) + '\n'); // Timeout after 30 seconds setTimeout(() => { if (pendingRequests.has(id)) { pendingRequests.delete(id); rejectReq(new Error(`Request timeout for ${method}`)); } }, 30000); }); }; // Initialize connection setTimeout(async () => { try { await sendRequest('initialize', { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'test-client', version: '1.0.0' } }); await sendRequest('initialized'); initialized = true; const client = { callTool: async (name, args) => sendRequest('tools/call', { name, arguments: args }), listTools: async () => sendRequest('tools/list'), close: () => serverProcess.kill() }; resolve(client); } catch (error) { reject(error); } }, 1000); // Handle process exit serverProcess.on('exit', (code) => { if (!initialized) { reject(new Error(`MCP server exited with code ${code}`)); } }); }); } async testAccount(accountName, username, password) { this.log(`\n🏢 Testing ${accountName} account...`); try { // Create MCP client const client = await this.createMcpClient(); // Authenticate this.log(`Authenticating ${accountName}...`); const authResult = await client.callTool('authenticate_user', { username, password }); if (!authResult.content?.[0]?.text?.includes('✅')) { this.log(`Authentication failed for ${accountName}: ${authResult.content?.[0]?.text}`, 'error'); client.close(); return; } this.log(`Authentication successful for ${accountName}`, 'success'); // Test endpoints for (const test of ENDPOINT_TESTS) { await this.testEndpoint(client, accountName, test); // If this is an MSP account (AllCloud), test with different customers if (accountName === 'AllCloud') { for (const customer of MSP_CUSTOMERS) { const customerTest = { ...test, name: `${test.name} (${customer.name})`, params: { ...test.params, customer_account_key: customer.accountKey, customer_division_id: customer.divisionId, userQuery: `Show me ${customer.name} data` } }; await this.testEndpoint(client, accountName, customerTest, customer.name); } } } // Logout await client.callTool('logout', {}); client.close(); } catch (error) { this.log(`Failed to test ${accountName}: ${error.message}`, 'error'); } } async testEndpoint(client, accountName, test, customerName = null) { const testName = `${accountName} - ${test.name}${customerName ? ` (${customerName})` : ''}`; this.log(`Testing: ${testName}`, 'test'); this.totalTests++; try { const startTime = Date.now(); const result = await client.callTool(test.tool, test.params); const duration = Date.now() - startTime; const responseText = result.content?.[0]?.text || ''; const success = !responseText.includes('❌'); const hasData = responseText.includes('```json') || responseText.includes('Results:') || responseText.match(/\d+\s+(items?|results?|recommendations?|budgets?|accounts?)/i); if (success) { this.successCount++; this.log(`✅ ${testName} - ${duration}ms ${hasData ? '(with data)' : '(empty)'}`, 'success'); } else { this.failCount++; const errorMsg = responseText.substring(0, 200); this.log(`❌ ${testName} - ${duration}ms - ${errorMsg}...`, 'error'); } // Store result this.results.push({ account: accountName, customer: customerName || 'Direct', testName: test.name, tool: test.tool, params: test.params, success, hasData, duration, responseLength: responseText.length, error: success ? null : errorMsg, timestamp: new Date().toISOString() }); // Brief pause between tests await new Promise(resolve => setTimeout(resolve, 100)); } catch (error) { this.failCount++; this.log(`❌ ${testName} - Error: ${error.message}`, 'error'); this.results.push({ account: accountName, customer: customerName || 'Direct', testName: test.name, tool: test.tool, params: test.params, success: false, hasData: false, duration: 0, responseLength: 0, error: error.message, timestamp: new Date().toISOString() }); } } async runAllTests() { this.log('🚀 Starting MCP Endpoint Tests', 'info'); // Check for passwords let canTest = false; for (const [key, account] of Object.entries(ACCOUNTS)) { if (account.password) { canTest = true; await this.testAccount(account.name, account.username, account.password); } else { this.log(`Skipping ${account.name} - no password set`, 'warning'); } } if (!canTest) { this.log('No passwords provided. Set environment variables:', 'warning'); this.log(' export SAOLA_PASSWORD="your_password"', 'info'); this.log(' export ALLCLOUD_PASSWORD="your_password"', 'info'); } this.generateReport(); } generateReport() { const duration = Date.now() - this.startTime; this.log('\n📊 TEST SUMMARY', 'info'); this.log(`Total Tests: ${this.totalTests}`, 'info'); this.log(`Successful: ${this.successCount} (${((this.successCount/this.totalTests)*100).toFixed(1)}%)`, 'success'); this.log(`Failed: ${this.failCount} (${((this.failCount/this.totalTests)*100).toFixed(1)}%)`, 'error'); this.log(`Duration: ${(duration/1000).toFixed(1)}s`, 'info'); // Account breakdown this.log('\n📋 ACCOUNT RESULTS:', 'info'); const accountGroups = {}; this.results.forEach(r => { if (!accountGroups[r.account]) { accountGroups[r.account] = { total: 0, success: 0, withData: 0 }; } accountGroups[r.account].total++; if (r.success) accountGroups[r.account].success++; if (r.hasData) accountGroups[r.account].withData++; }); Object.entries(accountGroups).forEach(([account, stats]) => { this.log(`${account}: ${stats.success}/${stats.total} successful (${((stats.success/stats.total)*100).toFixed(1)}%), ${stats.withData} with data`, 'info'); }); // Top successful tests const successful = this.results.filter(r => r.success && r.hasData); if (successful.length > 0) { this.log('\n✅ WORKING ENDPOINTS:', 'success'); successful.slice(0, 10).forEach(r => { this.log(` ${r.tool} (${r.account}${r.customer !== 'Direct' ? ` - ${r.customer}` : ''})`, 'info'); }); } // Failed tests const failed = this.results.filter(r => !r.success); if (failed.length > 0) { this.log('\n❌ FAILED ENDPOINTS:', 'error'); failed.slice(0, 10).forEach(r => { this.log(` ${r.tool} (${r.account}): ${r.error?.substring(0, 100)}`, 'warning'); }); } // Save detailed results const resultsPath = path.join(__dirname, `mcp-test-results-${Date.now()}.json`); fs.writeFileSync(resultsPath, JSON.stringify(this.results, null, 2)); this.log(`\nDetailed results: ${resultsPath}`, 'success'); // Generate CSV const csvHeaders = ['Account', 'Customer', 'Test', 'Tool', 'Success', 'HasData', 'Duration(ms)', 'Error']; const csvRows = this.results.map(r => [ r.account, r.customer, r.testName, r.tool, r.success, r.hasData, r.duration, r.error || '' ]); const csvContent = [csvHeaders.join(','), ...csvRows.map(row => row.map(cell => `"${cell}"`).join(','))].join('\n'); const csvPath = path.join(__dirname, `mcp-test-results-${Date.now()}.csv`); fs.writeFileSync(csvPath, csvContent); this.log(`CSV results: ${csvPath}`, 'success'); } } // Main execution async function main() { console.log('🧪 MCP Endpoint Comprehensive Tester'); console.log('===================================='); const tester = new McpTester(); await tester.runAllTests(); console.log('\n🎉 Testing completed!'); } if (require.main === module) { main().catch(error => { console.error('❌ Test failed:', error); process.exit(1); }); }