Umbrella MCP Server
COMPREHENSIVE-RC5-VS-OAUTH-COMPARISON.cjs•14.4 kB
#!/usr/bin/env node
// COMPREHENSIVE HONEST COMPARISON: RC5 vs OAuth (ALL 13 QUESTIONS)
// Using correct tool names for RC5 and testing ALL questions from goodanswers.txt
const axios = require('axios');
async function comprehensiveRC5vsOAuthComparison() {
console.log('🔬 COMPREHENSIVE RC5 vs OAUTH COMPARISON - ALL 13 QUESTIONS');
console.log('═'.repeat(100));
console.log('RC5 Server: http://localhost:3000 (final-mcp-server.cjs)');
console.log('OAuth Server: http://localhost:3001 (mcp-server-with-oauth-rc5.cjs)');
console.log('Testing ALL questions from goodanswers.txt with correct tool names\n');
const results = [];
try {
// Authenticate with RC5 server (port 3000)
console.log('🔐 Authenticating with RC5 server...');
const rc5AuthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: 'authenticate_user',
arguments: {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
}
},
id: 0
};
const rc5AuthResponse = await axios.post('http://localhost:3000/sse', rc5AuthRequest);
const rc5SessionMatch = rc5AuthResponse.data.result.content[0].text.match(/Session ID: ([a-f0-9-]+)/);
const rc5SessionId = rc5SessionMatch ? rc5SessionMatch[1] : null;
console.log(`✅ RC5 authenticated: ${rc5SessionId}`);
// Authenticate with OAuth server (port 3001)
console.log('🔐 Authenticating with OAuth server...');
const oauthSessionResponse = await axios.post('http://localhost:8080/api/session/create');
const oauthSessionId = oauthSessionResponse.data.sessionId;
await axios.post('http://localhost:8080/oauth/callback', {
sessionId: oauthSessionId,
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
});
console.log(`✅ OAuth authenticated: ${oauthSessionId}\n`);
// ALL 13 questions from goodanswers.txt
const questions = [
{
num: 'Q1',
query: 'show me the list of MSP customers',
rc5Tool: 'get_msp_customers',
oauthTool: 'api___msp_customers',
rc5Args: { userQuery: 'show me the list of MSP customers' },
oauthArgs: { userQuery: 'show me the list of MSP customers' }
},
{
num: 'Q2',
query: 'what is my total cost?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { userQuery: 'what is my total cost?' },
oauthArgs: { userQuery: 'what is my total cost?', periodGranLevel: 'month' }
},
{
num: 'Q3',
query: 'what is my total AWS cost?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'aws', userQuery: 'what is my total AWS cost?' },
oauthArgs: { cloud_context: 'aws', userQuery: 'what is my total AWS cost?', periodGranLevel: 'month' }
},
{
num: 'Q4',
query: 'what is my total GCP cost?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'gcp', userQuery: 'what is my total GCP cost?' },
oauthArgs: { cloud_context: 'gcp', userQuery: 'what is my total GCP cost?', periodGranLevel: 'month' }
},
{
num: 'Q5',
query: 'what is my total Azure cost?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'azure', userQuery: 'what is my total Azure cost?' },
oauthArgs: { cloud_context: 'azure', userQuery: 'what is my total Azure cost?', periodGranLevel: 'month' }
},
{
num: 'Q6',
query: 'show me the total cost per month',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { userQuery: 'show me the total cost per month' },
oauthArgs: { userQuery: 'show me the total cost per month', periodGranLevel: 'month' }
},
{
num: 'Q7',
query: 'show me the total AWS amortized cost per month for the last 8 months',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: {
cloud_context: 'aws',
userQuery: 'show me the total AWS amortized cost per month for the last 8 months'
},
oauthArgs: {
cloud_context: 'aws',
isAmortized: true,
periodGranLevel: 'month',
startDate: '2025-01-01',
endDate: '2025-08-31',
userQuery: 'show me the total AWS amortized cost per month for the last 8 months'
}
},
{
num: 'Q8',
query: 'show me the total cost for ALL Azure accounts',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'azure', userQuery: 'show me the total cost for ALL Azure accounts' },
oauthArgs: { cloud_context: 'azure', userQuery: 'show me the total cost for ALL Azure accounts', periodGranLevel: 'month' }
},
{
num: 'Q9',
query: 'show me all available accounts',
rc5Tool: 'get_accounts',
oauthTool: 'api___user_management_accounts',
rc5Args: {},
oauthArgs: { userQuery: 'show me all available accounts' }
},
{
num: 'Q10',
query: 'what do you recommend for saving AWS costs?',
rc5Tool: 'get_recommendations',
oauthTool: 'api___recommendationsNew_heatmap_summary',
rc5Args: { userQuery: 'what do you recommend for saving AWS costs?' },
oauthArgs: { userQuery: 'what do you recommend for saving AWS costs?' }
},
{
num: 'Q11',
query: 'what are the potential savings per category?',
rc5Tool: 'get_recommendations',
oauthTool: 'api___recommendationsNew_heatmap_summary',
rc5Args: { userQuery: 'what are the potential savings per category?' },
oauthArgs: { userQuery: 'what are the potential savings per category?' }
},
{
num: 'Q12',
query: 'what is the cost impact of the anomalies in the last 10 days for AWS?',
rc5Tool: 'get_anomalies',
oauthTool: 'api___anomaly_detection',
rc5Args: {
cloud_context: 'aws',
userQuery: 'what is the cost impact of the anomalies in the last 10 days for AWS?'
},
oauthArgs: {
cloud_context: 'aws',
startDate: '2025-08-28',
endDate: '2025-09-07',
userQuery: 'what is the cost impact of the anomalies in the last 10 days for AWS?'
}
},
{
num: 'Q13',
query: 'what is the last 30 days (per day) amortized cost for Cloudwatch service?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: {
cloud_context: 'aws',
userQuery: 'what is the last 30 days (per day) amortized cost for Cloudwatch service?'
},
oauthArgs: {
cloud_context: 'aws',
service: 'cloudwatch',
isAmortized: true,
periodGranLevel: 'day',
startDate: '2025-08-08',
endDate: '2025-09-07',
userQuery: 'what is the last 30 days (per day) amortized cost for Cloudwatch service?'
}
}
];
console.log('═'.repeat(100));
console.log('🔍 TESTING ALL 13 QUESTIONS');
console.log('═'.repeat(100));
for (const q of questions) {
console.log(`\n📋 ${q.num}: ${q.query}`);
console.log('─'.repeat(100));
let rc5Response = '';
let oauthResponse = '';
// Test RC5 server
try {
const rc5Request = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: q.rc5Tool,
arguments: {
sessionId: rc5SessionId,
...q.rc5Args
}
},
id: parseInt(q.num.substring(1))
};
const response = await axios.post('http://localhost:3000/sse', rc5Request);
rc5Response = response.data.result.content[0].text;
} catch (error) {
rc5Response = `❌ RC5 Error: ${error.response?.data?.error || error.message}`;
}
// Test OAuth server
try {
const oauthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: q.oauthTool,
arguments: {
sessionId: oauthSessionId,
...q.oauthArgs
}
},
id: parseInt(q.num.substring(1))
};
const response = await axios.post('http://localhost:3001/sse', oauthRequest);
oauthResponse = response.data.result.content[0].text;
} catch (error) {
oauthResponse = `❌ OAuth Error: ${error.response?.data?.error || error.message}`;
}
// Extract key data for comparison
const rc5Cost = rc5Response.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const oauthCost = oauthResponse.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const rc5Monthly = (rc5Response.match(/• \w+ 2025: \$[0-9,]+\.\d{2}/g) || []).length;
const oauthMonthly = (oauthResponse.match(/• \w+ 2025: \$[0-9,]+\.\d{2}/g) || []).length;
const rc5HasError = rc5Response.includes('Error') || rc5Response.includes('❌');
const oauthHasError = oauthResponse.includes('Error') || oauthResponse.includes('❌');
results.push({
num: q.num,
query: q.query,
rc5: { response: rc5Response, cost: rc5Cost, monthly: rc5Monthly, hasError: rc5HasError },
oauth: { response: oauthResponse, cost: oauthCost, monthly: oauthMonthly, hasError: oauthHasError }
});
// Show responses
console.log('\n🔧 RC5 RESPONSE:');
console.log('─'.repeat(50));
console.log(rc5Response.substring(0, 400));
if (rc5Response.length > 400) console.log('... [truncated]');
console.log('\n🔐 OAUTH RESPONSE:');
console.log('─'.repeat(50));
console.log(oauthResponse.substring(0, 400));
if (oauthResponse.length > 400) console.log('... [truncated]');
// Quick comparison
const match = rc5Cost === oauthCost && !rc5HasError && !oauthHasError;
console.log(`\n💰 COST: RC5=$${rc5Cost} | OAuth=$${oauthCost} | Match: ${match ? '✅' : '❌'}`);
console.log(`📊 ERRORS: RC5=${rc5HasError} | OAuth=${oauthHasError}`);
}
// Final comprehensive comparison table
console.log('\n\n' + '═'.repeat(120));
console.log('📊 COMPREHENSIVE COMPARISON TABLE - ALL 13 QUESTIONS');
console.log('═'.repeat(120));
console.log('\n| # | Question | RC5 Cost | OAuth Cost | RC5 Error | OAuth Error | Match |');
console.log('|---|----------|----------|------------|-----------|-------------|-------|');
let exactMatches = 0;
let functionalMatches = 0;
let bothWork = 0;
results.forEach(r => {
const exactMatch = r.rc5.cost === r.oauth.cost && !r.rc5.hasError && !r.oauth.hasError;
const functionalMatch = !r.rc5.hasError && !r.oauth.hasError;
const bothWorking = !r.rc5.hasError && !r.oauth.hasError;
if (exactMatch) exactMatches++;
if (functionalMatch) functionalMatches++;
if (bothWorking) bothWork++;
const matchIcon = exactMatch ? '✅' : (functionalMatch ? '⚠️' : '❌');
console.log(`| ${r.num} | ${r.query.substring(0, 40)}... | $${r.rc5.cost} | $${r.oauth.cost} | ${r.rc5.hasError ? '❌' : '✅'} | ${r.oauth.hasError ? '❌' : '✅'} | ${matchIcon} |`);
});
// Statistical analysis
console.log('\n' + '═'.repeat(120));
console.log('📈 STATISTICAL ANALYSIS');
console.log('═'.repeat(120));
const total = results.length;
const exactPercent = (exactMatches / total * 100).toFixed(1);
const functionalPercent = (functionalMatches / total * 100).toFixed(1);
const workingPercent = (bothWork / total * 100).toFixed(1);
console.log(`\n📊 RESULTS SUMMARY:`);
console.log(` • Total Questions: ${total}`);
console.log(` • Exact Matches (same costs): ${exactMatches}/${total} (${exactPercent}%)`);
console.log(` • Functional Matches (both work): ${functionalMatches}/${total} (${functionalPercent}%)`);
console.log(` • Both Servers Working: ${bothWork}/${total} (${workingPercent}%)`);
// Detailed analysis
console.log(`\n🔍 DETAILED ANALYSIS:`);
results.forEach(r => {
console.log(`\n${r.num}: ${r.query}`);
if (r.rc5.hasError && r.oauth.hasError) {
console.log(' • Both servers failed');
} else if (r.rc5.hasError) {
console.log(' • RC5 failed, OAuth working');
} else if (r.oauth.hasError) {
console.log(' • OAuth failed, RC5 working');
} else if (r.rc5.cost === r.oauth.cost) {
console.log(' • ✅ Perfect match');
} else {
console.log(` • ⚠️ Different costs: RC5=$${r.rc5.cost} vs OAuth=$${r.oauth.cost}`);
// Check monthly breakdown differences
if (r.rc5.monthly !== r.oauth.monthly) {
console.log(` • Monthly items: RC5=${r.rc5.monthly} vs OAuth=${r.oauth.monthly}`);
}
}
});
// Final verdict
console.log('\n' + '═'.repeat(120));
console.log('🎯 FINAL VERDICT');
console.log('═'.repeat(120));
if (exactPercent >= 80) {
console.log('✅ EXCELLENT: Servers are highly compatible');
} else if (exactPercent >= 60) {
console.log('⚠️ GOOD: Servers mostly compatible with some differences');
} else if (functionalPercent >= 80) {
console.log('⚠️ FUNCTIONAL: Both work but produce different results');
} else {
console.log('❌ CRITICAL: Significant compatibility issues');
}
console.log(`\n📝 PRODUCTION READINESS:`);
if (exactPercent >= 90) {
console.log('✅ Ready for production deployment');
} else if (functionalPercent >= 90) {
console.log('⚠️ Functional but data differences need investigation');
} else {
console.log('❌ NOT ready for production - needs fixes');
}
} catch (error) {
console.error('❌ Comparison failed:', error.message);
}
}
comprehensiveRC5vsOAuthComparison().catch(console.error);
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server