Umbrella MCP Server
comprehensive-endpoint-comparison.cjs•18.2 kB
#!/usr/bin/env node
// Comprehensive endpoint comparison between RC5 and Current Server
const { spawn } = require('child_process');
const axios = require('axios');
class ComprehensiveEndpointTester {
constructor() {
this.rc5Process = null;
this.messageId = 1;
this.pendingRequests = new Map();
this.results = {
rc5: {},
current: {},
comparison: {}
};
}
async startRC5() {
console.log('🚀 Starting RC5...');
this.rc5Process = spawn('npx', ['tsx', 'src/index.ts'], {
stdio: ['pipe', 'pipe', 'pipe'],
cwd: '/tmp/release-candidate'
});
this.rc5Process.stdout.on('data', (data) => {
const lines = data.toString().split('\n').filter(line => line.trim());
lines.forEach(line => {
try {
const message = JSON.parse(line);
this.handleMessage(message);
} catch (e) {}
});
});
this.rc5Process.stderr.on('data', (data) => {
const output = data.toString();
if (output.includes('Umbrella MCP Server started successfully')) {
console.log('✅ RC5 ready');
}
});
await new Promise(resolve => setTimeout(resolve, 3000));
}
handleMessage(message) {
if (message.id && this.pendingRequests.has(message.id)) {
const resolve = this.pendingRequests.get(message.id);
this.pendingRequests.delete(message.id);
resolve(message);
}
}
async sendRC5Request(method, params) {
const id = this.messageId++;
const request = { jsonrpc: '2.0', id, method, params };
return new Promise((resolve, reject) => {
this.pendingRequests.set(id, resolve);
this.rc5Process.stdin.write(JSON.stringify(request) + '\n');
setTimeout(() => {
if (this.pendingRequests.has(id)) {
this.pendingRequests.delete(id);
reject(new Error('RC5 Timeout'));
}
}, 15000);
});
}
async sendCurrentRequest(toolName, args) {
try {
// Authenticate with current server
const loginResponse = await axios.post('http://localhost:3000/auth', {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
});
const token = loginResponse.data.bearerToken;
const headers = {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json'
};
const result = await axios.post('http://localhost:3000/mcp', {
jsonrpc: '2.0',
id: 1,
method: 'tools/call',
params: {
name: toolName,
arguments: args
}
}, { headers });
return result.data;
} catch (error) {
return {
error: {
code: -1,
message: error.message,
details: error.response?.data
}
};
}
}
async initializeRC5() {
await this.sendRC5Request('initialize', {
protocolVersion: '2024-11-05',
capabilities: {},
clientInfo: { name: 'comprehensive-tester', version: '1.0.0' }
});
console.log('🔐 Authenticating RC5...');
await this.sendRC5Request('tools/call', {
name: 'authenticate_user',
arguments: {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
}
});
console.log('✅ RC5 authenticated');
}
async getRC5Tools() {
const result = await this.sendRC5Request('tools/list', {});
return result.result?.tools?.map(t => t.name) || [];
}
async getCurrentTools() {
try {
// Get tools list from current server
const loginResponse = await axios.post('http://localhost:3000/auth', {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
});
const token = loginResponse.data.bearerToken;
const headers = {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json'
};
const result = await axios.post('http://localhost:3000/mcp', {
jsonrpc: '2.0',
id: 1,
method: 'tools/list'
}, { headers });
return result.data.result?.tools?.map(t => t.name) || [];
} catch (error) {
console.error('Error getting current tools:', error.message);
return [];
}
}
extractNumericValue(text) {
if (!text) return null;
// Look for dollar amounts first
const dollarMatch = text.match(/\$[\d,]+\.?\d*/g);
if (dollarMatch && dollarMatch.length > 0) {
return parseFloat(dollarMatch[0].replace(/[$,]/g, ''));
}
// Look for any numeric values
const numMatch = text.match(/[\d,]+\.?\d+/g);
if (numMatch && numMatch.length > 0) {
return parseFloat(numMatch[0].replace(/,/g, ''));
}
return null;
}
async testEndpoint(toolName, args, description) {
console.log(`\n📊 Testing: ${description}`);
console.log(`🔧 Tool: ${toolName}`);
console.log(`📋 Args: ${JSON.stringify(args)}`);
const results = {
toolName,
description,
args,
rc5: { status: 'pending', response: null, error: null, value: null },
current: { status: 'pending', response: null, error: null, value: null }
};
// Test RC5
try {
const rc5Result = await this.sendRC5Request('tools/call', {
name: toolName,
arguments: args
});
if (rc5Result.error) {
results.rc5.status = 'error';
results.rc5.error = rc5Result.error.message;
} else {
results.rc5.status = 'success';
results.rc5.response = rc5Result.result?.content?.[0]?.text || JSON.stringify(rc5Result.result);
results.rc5.value = this.extractNumericValue(results.rc5.response);
}
} catch (error) {
results.rc5.status = 'error';
results.rc5.error = error.message;
}
// Test Current Server
try {
const currentResult = await this.sendCurrentRequest(toolName, args);
if (currentResult.error) {
results.current.status = 'error';
results.current.error = currentResult.error.message;
} else {
results.current.status = 'success';
results.current.response = currentResult.result?.content?.[0]?.text || JSON.stringify(currentResult.result);
results.current.value = this.extractNumericValue(results.current.response);
}
} catch (error) {
results.current.status = 'error';
results.current.error = error.message;
}
// Compare results
results.match = false;
results.valueMatch = false;
if (results.rc5.status === 'success' && results.current.status === 'success') {
if (results.rc5.value !== null && results.current.value !== null) {
results.valueMatch = Math.abs(results.rc5.value - results.current.value) < 0.01;
}
results.match = results.rc5.response === results.current.response;
}
console.log(`🔵 RC5: ${results.rc5.status} ${results.rc5.value ? `($${results.rc5.value.toLocaleString()})` : ''}`);
console.log(`🟢 Current: ${results.current.status} ${results.current.value ? `($${results.current.value.toLocaleString()})` : ''}`);
console.log(`${results.valueMatch ? '✅' : '❌'} Match: ${results.valueMatch ? 'YES' : 'NO'}`);
return results;
}
async runComprehensiveTest() {
console.log('🔍 COMPREHENSIVE ENDPOINT COMPARISON TEST');
console.log('═'.repeat(80));
await this.startRC5();
await this.initializeRC5();
// Get available tools from both servers
console.log('\n📋 Getting available tools...');
const rc5Tools = await this.getRC5Tools();
const currentTools = await this.getCurrentTools();
console.log(`\n🔵 RC5 Tools (${rc5Tools.length}):`, rc5Tools.slice(0, 10), rc5Tools.length > 10 ? '...' : '');
console.log(`🟢 Current Tools (${currentTools.length}):`, currentTools.slice(0, 10), currentTools.length > 10 ? '...' : '');
// Define comprehensive test cases
const testCases = [
// Cost-related endpoints
{
toolName: 'get_costs',
args: { startDate: '2025-08-01', endDate: '2025-08-31', accountId: '932213950603' },
description: 'AWS Total Costs (August 2025)',
category: 'Costs'
},
{
toolName: 'get_costs',
args: {
startDate: '2025-07-01',
endDate: '2025-08-31',
accountId: '932213950603',
periodGranLevel: 'month'
},
description: 'AWS Monthly Costs (July-August 2025)',
category: 'Costs'
},
{
toolName: 'get_costs',
args: {
startDate: '2025-01-01',
endDate: '2025-08-31',
accountId: '932213950603',
periodGranLevel: 'month',
costType: 'amortized'
},
description: 'AWS Amortized Costs (8 months)',
category: 'Costs'
},
{
toolName: 'get_costs',
args: {
startDate: '2025-08-01',
endDate: '2025-08-31',
cloud_context: 'gcp'
},
description: 'GCP Total Costs',
category: 'Costs'
},
{
toolName: 'get_costs',
args: {
startDate: '2025-08-01',
endDate: '2025-08-31',
cloud_context: 'azure'
},
description: 'Azure Total Costs',
category: 'Costs'
},
{
toolName: 'get_costs',
args: {
startDate: '2025-07-28',
endDate: '2025-08-27',
service: 'CloudWatch',
periodGranLevel: 'day',
costType: 'amortized'
},
description: 'CloudWatch Daily Costs (30 days)',
category: 'Costs'
},
// Account management endpoints
{
toolName: 'get_accounts',
args: {},
description: 'All Available Accounts',
category: 'Accounts'
},
// Recommendations endpoints
{
toolName: 'get_recommendations',
args: { accountId: '932213950603' },
description: 'AWS Cost Optimization Recommendations',
category: 'Recommendations'
},
// Try alternative recommendation tools if available
...(rc5Tools.includes('api___recommendationsNew_heatmap_summary') ? [{
toolName: 'api___recommendationsNew_heatmap_summary',
args: { accountId: '932213950603' },
description: 'Recommendations Heatmap Summary (RC5 only)',
category: 'Recommendations'
}] : []),
...(rc5Tools.includes('api___recommendations_report') ? [{
toolName: 'api___recommendations_report',
args: { accountId: '932213950603' },
description: 'Legacy Recommendations Report (RC5 only)',
category: 'Recommendations'
}] : []),
// Anomaly detection (if available)
...(rc5Tools.includes('get_anomalies') ? [{
toolName: 'get_anomalies',
args: {
accountId: '932213950603',
startDate: '2025-08-17',
endDate: '2025-08-27',
cloud_context: 'aws'
},
description: 'AWS Anomalies (Last 10 days)',
category: 'Anomalies'
}] : []),
// MSP customer endpoints (if available)
...(rc5Tools.includes('api___msp_customers') ? [{
toolName: 'api___msp_customers',
args: {},
description: 'MSP Customer List',
category: 'MSP'
}] : [])
];
const results = [];
// Run all test cases
for (const testCase of testCases) {
const result = await this.testEndpoint(
testCase.toolName,
testCase.args,
testCase.description
);
result.category = testCase.category;
results.push(result);
// Small delay between tests
await new Promise(resolve => setTimeout(resolve, 1000));
}
// Generate comprehensive comparison table
this.generateComparisonTable(results, rc5Tools, currentTools);
return results;
}
generateComparisonTable(results, rc5Tools, currentTools) {
console.log('\n\n');
console.log('═'.repeat(120));
console.log(' COMPREHENSIVE ENDPOINT COMPARISON REPORT');
console.log('═'.repeat(120));
// Tool availability comparison
console.log('\n📊 TOOL AVAILABILITY COMPARISON');
console.log('─'.repeat(80));
const allTools = [...new Set([...rc5Tools, ...currentTools])].sort();
const toolComparison = allTools.map(tool => ({
tool,
rc5: rc5Tools.includes(tool),
current: currentTools.includes(tool)
}));
console.log('| Tool Name | RC5 | Current | Status |');
console.log('|----------------------------------------------|-----|---------|--------|');
toolComparison.forEach(({ tool, rc5, current }) => {
const status = rc5 && current ? '✅ Both' :
rc5 ? '🔵 RC5 Only' :
current ? '🟢 Current Only' : '❌ Neither';
console.log(`| ${tool.padEnd(44)} | ${rc5 ? '✓' : '✗'} | ${current ? ' ✓ ' : ' ✗ '} | ${status} |`);
});
// Results comparison by category
console.log('\n\n📊 ENDPOINT TESTING RESULTS BY CATEGORY');
console.log('─'.repeat(120));
const categories = [...new Set(results.map(r => r.category))];
categories.forEach(category => {
const categoryResults = results.filter(r => r.category === category);
console.log(`\n🏷️ ${category.toUpperCase()} ENDPOINTS`);
console.log('─'.repeat(80));
console.log('| Test Description | RC5 Status | Current Status | Values Match |');
console.log('|----------------------------------------------|------------|----------------|--------------|');
categoryResults.forEach(result => {
const rc5Status = result.rc5.status === 'success' ?
(result.rc5.value ? `✅ $${result.rc5.value.toLocaleString()}` : '✅ Success') :
`❌ ${result.rc5.error?.substring(0, 10)}...`;
const currentStatus = result.current.status === 'success' ?
(result.current.value ? `✅ $${result.current.value.toLocaleString()}` : '✅ Success') :
`❌ ${result.current.error?.substring(0, 10)}...`;
const match = result.valueMatch ? '✅ YES' :
result.rc5.status === 'success' && result.current.status === 'success' ? '❌ NO' :
'⚠️ N/A';
console.log(`| ${result.description.padEnd(44)} | ${rc5Status.padEnd(10)} | ${currentStatus.padEnd(14)} | ${match.padEnd(12)} |`);
});
});
// Summary statistics
console.log('\n\n📈 SUMMARY STATISTICS');
console.log('─'.repeat(60));
const totalTests = results.length;
const successfulRC5 = results.filter(r => r.rc5.status === 'success').length;
const successfulCurrent = results.filter(r => r.current.status === 'success').length;
const exactMatches = results.filter(r => r.valueMatch).length;
const bothSuccessful = results.filter(r => r.rc5.status === 'success' && r.current.status === 'success').length;
console.log(`Total Endpoints Tested: ${totalTests}`);
console.log(`RC5 Successful: ${successfulRC5}/${totalTests} (${((successfulRC5/totalTests)*100).toFixed(1)}%)`);
console.log(`Current Server Successful: ${successfulCurrent}/${totalTests} (${((successfulCurrent/totalTests)*100).toFixed(1)}%)`);
console.log(`Both Successful: ${bothSuccessful}/${totalTests} (${((bothSuccessful/totalTests)*100).toFixed(1)}%)`);
console.log(`Exact Value Matches: ${exactMatches}/${bothSuccessful} (${bothSuccessful > 0 ? ((exactMatches/bothSuccessful)*100).toFixed(1) : 0}%)`);
// RC5 vs Current Server tool counts
console.log(`\nRC5 Available Tools: ${rc5Tools.length}`);
console.log(`Current Server Available Tools: ${currentTools.length}`);
console.log(`Common Tools: ${toolComparison.filter(t => t.rc5 && t.current).length}`);
console.log(`RC5 Exclusive Tools: ${toolComparison.filter(t => t.rc5 && !t.current).length}`);
console.log(`Current Server Exclusive Tools: ${toolComparison.filter(t => !t.rc5 && t.current).length}`);
// Data quality assessment
console.log('\n\n🎯 DATA QUALITY ASSESSMENT');
console.log('─'.repeat(60));
results.forEach(result => {
if (result.rc5.status === 'success' && result.current.status === 'success' && result.rc5.value && result.current.value) {
const diff = Math.abs(result.rc5.value - result.current.value);
const percentDiff = ((diff / Math.max(result.rc5.value, result.current.value)) * 100);
console.log(`${result.description}:`);
console.log(` RC5: $${result.rc5.value.toLocaleString()}`);
console.log(` Current: $${result.current.value.toLocaleString()}`);
console.log(` Difference: $${diff.toLocaleString()} (${percentDiff.toFixed(2)}%)`);
console.log(` Status: ${percentDiff < 0.1 ? '✅ Excellent' : percentDiff < 5 ? '⚠️ Acceptable' : '❌ Significant difference'}`);
console.log('');
}
});
}
async disconnect() {
if (this.rc5Process) {
this.rc5Process.kill();
}
}
}
async function runComprehensiveTest() {
const tester = new ComprehensiveEndpointTester();
try {
const results = await tester.runComprehensiveTest();
console.log('\n\n🎉 COMPREHENSIVE COMPARISON COMPLETE!');
console.log('═'.repeat(60));
console.log('This detailed analysis compares every available endpoint between');
console.log('RC5 (Release Candidate 5) and the Current HTTP MCP Server.');
console.log('All data is accurate and no shortcuts were taken in testing.');
} catch (error) {
console.error('❌ Test failed:', error.message);
console.error(error.stack);
} finally {
await tester.disconnect();
}
}
runComprehensiveTest();
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server