Umbrella MCP Server

#!/usr/bin/env node import dotenv from 'dotenv'; import { UmbrellaAuth } from './auth.js'; import { UmbrellaApiClient } from './api-client.js'; import { getTestCredentialSets } from './auth-helper.js'; import { UMBRELLA_ENDPOINTS } from './types.js'; dotenv.config(); interface TestResult { endpoint: string; method: string; description: string; category: string; success: boolean; error?: string; data?: any; responseTime: number; requiredParams?: string[]; providedParams?: Record<string, any>; statusCode?: number; } interface TestSummary { totalEndpoints: number; successCount: number; failureCount: number; successRate: number; testDuration: number; errorBreakdown: Record<string, number>; categoryResults: Record<string, { success: number; total: number; rate: number }>; } // Based on frontend analysis, these are the key parameters needed for different endpoint types const COMMON_PARAMETERS = { // Date parameters for cost queries date: { startDate: '2024-01-01', endDate: '2024-01-31', }, // Pagination parameters pagination: { limit: 10, offset: 0, }, // Cost analysis parameters cost: { costType: ['unblended'], groupBy: ['service'], periodGranLevel: 'daily', }, // Account parameters (would need to be discovered dynamically) account: { // These would be populated after initial auth/user discovery accountId: '', organizationId: '', }, }; class ComprehensiveApiTester { private auth: UmbrellaAuth; private apiClient: UmbrellaApiClient; private baseURL: string; private credentials: { username: string; password: string } | null = null; private results: TestResult[] = []; private accountIds: string[] = []; constructor(baseURL: string) { this.baseURL = baseURL; this.auth = new UmbrellaAuth(baseURL); this.apiClient = new UmbrellaApiClient(baseURL); } async authenticate(credentials: { username: string; password: string }): Promise<boolean> { try { const authResult = await this.auth.authenticate(credentials); const authHeaders = this.auth.getAuthHeaders(); this.apiClient.setAuthToken(authHeaders); this.credentials = credentials; // Try to discover user accounts for proper parameterization await this.discoverUserAccounts(); return true; } catch (error: any) { console.error(`❌ Authentication failed for ${credentials.username}: ${error.message}`); return false; } } private async discoverUserAccounts(): Promise<void> { try { // Try to get user info to discover available accounts const userResponse = await this.apiClient.makeRequest('/users', {}); if (userResponse.success && userResponse.data) { // Extract account information if available console.log('✅ User data discovery successful'); } } catch (error) { console.log('⚠️ Could not discover user accounts for better parameterization'); } } private getTestParameters(endpoint: any): Record<string, any> { const params: Record<string, any> = {}; // Add common date parameters for cost-related endpoints if (endpoint.path.includes('cost') || endpoint.path.includes('usage') || endpoint.path.includes('invoice')) { Object.assign(params, COMMON_PARAMETERS.date); } // Add pagination for list endpoints if (endpoint.path.includes('list') || endpoint.description.toLowerCase().includes('list')) { Object.assign(params, COMMON_PARAMETERS.pagination); } // Add specific parameters based on endpoint analysis switch (true) { case endpoint.path.includes('/caui'): // Cost and Usage Interface needs account context Object.assign(params, COMMON_PARAMETERS.date, COMMON_PARAMETERS.cost); if (this.accountIds.length > 0) { params.accountId = this.accountIds[0]; } break; case endpoint.path.includes('/service-names/distinct'): // This endpoint works without extra params params.limit = 10; break; case endpoint.path.includes('/recommendations'): // Recommendations might need account/org context if (this.accountIds.length > 0) { params.accountId = this.accountIds[0]; } break; case endpoint.path.includes('/budget'): // Budget endpoints need account context if (this.accountIds.length > 0) { params.accountId = this.accountIds[0]; } break; } // Add any endpoint-specific parameters defined in our types if (endpoint.parameters) { Object.entries(endpoint.parameters).forEach(([key, description]) => { if (!params[key]) { // Provide reasonable defaults based on parameter name patterns if (key.includes('Id') && key !== 'accountId') { params[key] = 'test-id'; } else if (key.includes('Name')) { params[key] = 'test-name'; } else if (key.includes('Type')) { params[key] = 'unblended'; } } }); } return params; } private categorizeError(error: string): string { if (error.includes('not found') || error.includes('404')) return 'Not Found'; if (error.includes('required') || error.includes('mandatory')) return 'Missing Required Parameters'; if (error.includes('Access denied') || error.includes('403')) return 'Access Denied'; if (error.includes('500')) return 'Server Error'; if (error.includes('400')) return 'Bad Request'; if (error.includes('401')) return 'Authentication Error'; if (error.includes('timeout')) return 'Timeout'; if (error.includes('network') || error.includes('ECONNREFUSED')) return 'Network Error'; return 'Other'; } async testEndpoint(endpoint: any): Promise<TestResult> { const startTime = Date.now(); const testParams = this.getTestParameters(endpoint); try { console.log(` 🔍 ${endpoint.method} ${endpoint.path}`); console.log(` 📝 ${endpoint.description}`); if (Object.keys(testParams).length > 0) { console.log(` 🔧 Params: ${Object.keys(testParams).join(', ')}`); } const response = await this.apiClient.makeRequest(endpoint.path, testParams); const responseTime = Date.now() - startTime; if (response.success) { let dataInfo = 'No data'; if (response.data) { if (Array.isArray(response.data)) { dataInfo = `${response.data.length} array items`; if (response.data.length > 0) { const sample = JSON.stringify(response.data[0]).substring(0, 100); console.log(` 📊 Sample: ${sample}${sample.length === 100 ? '...' : ''}`); } } else if (typeof response.data === 'object') { dataInfo = `object with ${Object.keys(response.data).length} keys`; console.log(` 📊 Keys: ${Object.keys(response.data).join(', ')}`); } else { dataInfo = `${typeof response.data}: ${response.data}`; } } console.log(` ✅ SUCCESS: ${dataInfo} (${responseTime}ms)`); return { endpoint: endpoint.path, method: endpoint.method, description: endpoint.description, category: endpoint.category, success: true, data: response.data, responseTime, providedParams: testParams, statusCode: 200 }; } else { console.log(` ❌ FAILED: ${response.error} (${responseTime}ms)`); return { endpoint: endpoint.path, method: endpoint.method, description: endpoint.description, category: endpoint.category, success: false, error: response.error, responseTime, providedParams: testParams, statusCode: 400 }; } } catch (error: any) { const responseTime = Date.now() - startTime; console.log(` 💥 ERROR: ${error.message} (${responseTime}ms)`); return { endpoint: endpoint.path, method: endpoint.method, description: endpoint.description, category: endpoint.category, success: false, error: error.message, responseTime, providedParams: testParams, statusCode: 500 }; } } async runFullTest(credentials: { username: string; password: string }): Promise<TestSummary> { const testStartTime = Date.now(); console.log(`\n${'='.repeat(100)}`); console.log(`🧪 COMPREHENSIVE API TEST: ${credentials.username}`); console.log(`📡 Base URL: ${this.baseURL}`); console.log(`📊 Total Endpoints: ${UMBRELLA_ENDPOINTS.length}`); console.log(`${'='.repeat(100)}`); // Authenticate first console.log('\n🔐 AUTHENTICATION'); const authSuccess = await this.authenticate(credentials); if (!authSuccess) { throw new Error('Authentication failed'); } console.log(`✅ Authentication successful for ${credentials.username}`); // Test all endpoints this.results = []; console.log(`\n🧪 TESTING ALL ${UMBRELLA_ENDPOINTS.length} ENDPOINTS`); for (let i = 0; i < UMBRELLA_ENDPOINTS.length; i++) { const endpoint = UMBRELLA_ENDPOINTS[i]; console.log(`\n[${i + 1}/${UMBRELLA_ENDPOINTS.length}] ${endpoint.category}`); const result = await this.testEndpoint(endpoint); this.results.push(result); // Brief pause to avoid overwhelming the server await new Promise(resolve => setTimeout(resolve, 100)); } const testDuration = Date.now() - testStartTime; return this.generateSummary(testDuration); } private generateSummary(testDuration: number): TestSummary { const successCount = this.results.filter(r => r.success).length; const failureCount = this.results.length - successCount; const successRate = (successCount / this.results.length) * 100; // Error breakdown const errorBreakdown: Record<string, number> = {}; this.results .filter(r => !r.success) .forEach(result => { const errorType = this.categorizeError(result.error || ''); errorBreakdown[errorType] = (errorBreakdown[errorType] || 0) + 1; }); // Category breakdown const categoryResults: Record<string, { success: number; total: number; rate: number }> = {}; this.results.forEach(result => { if (!categoryResults[result.category]) { categoryResults[result.category] = { success: 0, total: 0, rate: 0 }; } categoryResults[result.category].total++; if (result.success) { categoryResults[result.category].success++; } }); // Calculate rates Object.keys(categoryResults).forEach(category => { const cat = categoryResults[category]; cat.rate = (cat.success / cat.total) * 100; }); return { totalEndpoints: this.results.length, successCount, failureCount, successRate, testDuration, errorBreakdown, categoryResults }; } printDetailedResults(summary: TestSummary): void { console.log(`\n${'='.repeat(100)}`); console.log(`📈 DETAILED TEST RESULTS`); console.log(`${'='.repeat(100)}`); console.log(`\n🎯 OVERALL SUMMARY:`); console.log(` ✅ Successful: ${summary.successCount}/${summary.totalEndpoints} (${summary.successRate.toFixed(1)}%)`); console.log(` ❌ Failed: ${summary.failureCount}/${summary.totalEndpoints}`); console.log(` ⏱️ Test Duration: ${(summary.testDuration / 1000).toFixed(1)}s`); console.log(` 📊 Avg Response Time: ${Math.round(this.results.reduce((sum, r) => sum + r.responseTime, 0) / this.results.length)}ms`); // Working endpoints const working = this.results.filter(r => r.success); if (working.length > 0) { console.log(`\n✅ WORKING ENDPOINTS (${working.length}):`); working.forEach(result => { const dataInfo = Array.isArray(result.data) ? `${result.data.length} items` : typeof result.data === 'object' && result.data ? `object` : 'data available'; console.log(` ${result.method} ${result.endpoint} - ${dataInfo} (${result.responseTime}ms)`); }); } // Failed endpoints by error type if (summary.failureCount > 0) { console.log(`\n❌ FAILED ENDPOINTS (${summary.failureCount}):`); Object.entries(summary.errorBreakdown).forEach(([errorType, count]) => { console.log(`\n 📋 ${errorType} (${count} endpoints):`); this.results .filter(r => !r.success && this.categorizeError(r.error || '') === errorType) .forEach(result => { console.log(` ${result.method} ${result.endpoint} - ${result.error}`); }); }); } // Category breakdown console.log(`\n📊 RESULTS BY CATEGORY:`); Object.entries(summary.categoryResults) .sort(([,a], [,b]) => b.rate - a.rate) .forEach(([category, stats]) => { const status = stats.rate === 100 ? '✅' : stats.rate > 0 ? '⚠️' : '❌'; console.log(` ${status} ${category}: ${stats.success}/${stats.total} (${stats.rate.toFixed(1)}%)`); }); // Recommendations console.log(`\n💡 RECOMMENDATIONS:`); if (summary.successRate < 50) { console.log(` 🔴 LOW SUCCESS RATE: Only ${summary.successRate.toFixed(1)}% of endpoints are working`); console.log(` 📋 Priority: Focus on fixing "Missing Required Parameters" errors first`); } if (summary.errorBreakdown['Missing Required Parameters']) { console.log(` 🔧 ${summary.errorBreakdown['Missing Required Parameters']} endpoints need required parameters`); console.log(` 📋 Solution: Implement parameter discovery or provide account context`); } if (summary.errorBreakdown['Access Denied']) { console.log(` 🔒 ${summary.errorBreakdown['Access Denied']} endpoints have permission issues`); console.log(` 📋 Solution: Check user roles and permissions in Umbrella Cost`); } console.log(`\n🎉 COMPREHENSIVE API TEST COMPLETED!`); } } async function main() { console.log('🧪 COMPREHENSIVE UMBRELLA MCP API TESTING'); console.log('=========================================='); console.log(`📅 Test Time: ${new Date().toISOString()}`); const baseURL = process.env.UMBRELLA_API_BASE_URL || 'https://api.umbrellacost.io/api/v1'; const credentialSets = getTestCredentialSets(); if (!credentialSets) { console.error('❌ No test credentials available'); process.exit(1); } const allResults: Array<{ credName: string; summary: TestSummary }> = []; for (let i = 0; i < credentialSets.length; i++) { const creds = credentialSets[i]; const credName = `Account ${i + 1} (${creds.username})`; try { const tester = new ComprehensiveApiTester(baseURL); const summary = await tester.runFullTest(creds); tester.printDetailedResults(summary); allResults.push({ credName, summary }); } catch (error: any) { console.error(`❌ Test failed for ${credName}: ${error.message}`); } // Pause between different accounts if (i < credentialSets.length - 1) { console.log('\n⏳ Pausing before testing next account...'); await new Promise(resolve => setTimeout(resolve, 3000)); } } // Final cross-analysis if (allResults.length > 1) { console.log(`\n${'='.repeat(100)}`); console.log(`🔄 CROSS-ACCOUNT ANALYSIS`); console.log(`${'='.repeat(100)}`); const avgSuccessRate = allResults.reduce((sum, r) => sum + r.summary.successRate, 0) / allResults.length; console.log(`\n📊 Average Success Rate: ${avgSuccessRate.toFixed(1)}%`); allResults.forEach(result => { console.log(` ${result.credName}: ${result.summary.successRate.toFixed(1)}% (${result.summary.successCount}/${result.summary.totalEndpoints})`); }); } console.log(`\n🏁 ALL TESTING COMPLETED!`); } // Run main function if this file is executed directly if (import.meta.url === `file://${process.argv[1]}`) { main().catch(console.error); } export { ComprehensiveApiTester, TestResult, TestSummary };