Umbrella MCP Server
FIXED-RC5-VS-OAUTH-COMPARISON.cjs•14.5 kB
#!/usr/bin/env node
// FIXED RC5 vs OAuth comparison with proper authentication flow
// RC5 uses authenticate_user then tool calls, OAuth uses session-based auth
const axios = require('axios');
async function fixedRC5vsOAuthComparison() {
console.log('🔬 FIXED RC5 vs OAUTH COMPARISON - PROPER AUTHENTICATION');
console.log('═'.repeat(100));
console.log('RC5 Server: http://localhost:3000 (authenticate_user → tool calls)');
console.log('OAuth Server: http://localhost:3001 (OAuth session → tool calls)');
console.log('Testing ALL 13 questions with CORRECT authentication flow\n');
const results = [];
try {
// Authenticate with OAuth server (via external OAuth service)
console.log('🔐 Authenticating with OAuth server...');
const oauthSessionResponse = await axios.post('http://localhost:8080/api/session/create');
const oauthSessionId = oauthSessionResponse.data.sessionId;
await axios.post('http://localhost:8080/oauth/callback', {
sessionId: oauthSessionId,
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
});
console.log(`✅ OAuth authenticated: ${oauthSessionId}`);
// For RC5, we authenticate once then make tool calls directly (no session passing)
console.log('🔐 Authenticating with RC5 server...');
const rc5AuthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: 'authenticate_user',
arguments: {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
}
},
id: 0
};
const rc5AuthResponse = await axios.post('http://localhost:3000/sse', rc5AuthRequest);
console.log(`✅ RC5 authenticated: ${rc5AuthResponse.data.result.content[0].text.substring(0, 50)}...\n`);
// Test questions with proper tool mappings
const questions = [
{
num: 'Q1',
query: 'show me the list of MSP customers',
rc5Tool: 'get_msp_customers',
oauthTool: 'api___msp_customers',
rc5Args: {},
oauthArgs: {}
},
{
num: 'Q2',
query: 'what is my total cost?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: {},
oauthArgs: { periodGranLevel: 'month' }
},
{
num: 'Q3',
query: 'what is my total AWS cost?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'aws' },
oauthArgs: { cloud_context: 'aws', periodGranLevel: 'month' }
},
{
num: 'Q4',
query: 'what is my total GCP cost?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'gcp' },
oauthArgs: { cloud_context: 'gcp', periodGranLevel: 'month' }
},
{
num: 'Q5',
query: 'what is my total Azure cost?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'azure' },
oauthArgs: { cloud_context: 'azure', periodGranLevel: 'month' }
},
{
num: 'Q6',
query: 'show me the total cost per month',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: {},
oauthArgs: { periodGranLevel: 'month' }
},
{
num: 'Q7',
query: 'show me the total AWS amortized cost per month for the last 8 months',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'aws' },
oauthArgs: {
cloud_context: 'aws',
isAmortized: true,
periodGranLevel: 'month',
startDate: '2025-01-01',
endDate: '2025-08-31'
}
},
{
num: 'Q8',
query: 'show me the total cost for ALL Azure accounts',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'azure' },
oauthArgs: { cloud_context: 'azure', periodGranLevel: 'month' }
},
{
num: 'Q9',
query: 'show me all available accounts',
rc5Tool: 'get_accounts',
oauthTool: 'api___user_management_accounts',
rc5Args: {},
oauthArgs: {}
},
{
num: 'Q10',
query: 'what do you recommend for saving AWS costs?',
rc5Tool: 'get_recommendations',
oauthTool: 'api___recommendationsNew_heatmap_summary',
rc5Args: {},
oauthArgs: {}
},
{
num: 'Q11',
query: 'what are the potential savings per category?',
rc5Tool: 'get_recommendations',
oauthTool: 'api___recommendationsNew_heatmap_summary',
rc5Args: {},
oauthArgs: {}
},
{
num: 'Q12',
query: 'what is the cost impact of the anomalies in the last 10 days for AWS?',
rc5Tool: 'get_anomalies',
oauthTool: 'api___anomaly_detection',
rc5Args: { cloud_context: 'aws' },
oauthArgs: {
cloud_context: 'aws',
startDate: '2025-08-28',
endDate: '2025-09-07'
}
},
{
num: 'Q13',
query: 'what is the last 30 days (per day) amortized cost for Cloudwatch service?',
rc5Tool: 'get_costs',
oauthTool: 'api___invoices_caui',
rc5Args: { cloud_context: 'aws' },
oauthArgs: {
cloud_context: 'aws',
service: 'cloudwatch',
isAmortized: true,
periodGranLevel: 'day',
startDate: '2025-08-08',
endDate: '2025-09-07'
}
}
];
console.log('═'.repeat(100));
console.log('🔍 TESTING ALL 13 QUESTIONS WITH CORRECT AUTHENTICATION');
console.log('═'.repeat(100));
for (const q of questions) {
console.log(`\n📋 ${q.num}: ${q.query}`);
console.log('─'.repeat(100));
let rc5Response = '';
let oauthResponse = '';
// Test RC5 server (no sessionId needed after authentication)
try {
const rc5Request = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: q.rc5Tool,
arguments: {
userQuery: q.query,
...q.rc5Args
}
},
id: parseInt(q.num.substring(1))
};
const response = await axios.post('http://localhost:3000/sse', rc5Request);
rc5Response = response.data.result.content[0].text;
} catch (error) {
rc5Response = `❌ RC5 Error: ${error.response?.data?.error || error.message}`;
}
// Test OAuth server (with sessionId)
try {
const oauthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: q.oauthTool,
arguments: {
sessionId: oauthSessionId,
userQuery: q.query,
...q.oauthArgs
}
},
id: parseInt(q.num.substring(1))
};
const response = await axios.post('http://localhost:3001/sse', oauthRequest);
oauthResponse = response.data.result.content[0].text;
} catch (error) {
oauthResponse = `❌ OAuth Error: ${error.response?.data?.error || error.message}`;
}
// Extract key comparison data
const rc5Cost = rc5Response.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const oauthCost = oauthResponse.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const rc5HasError = rc5Response.includes('Error') || rc5Response.includes('❌');
const oauthHasError = oauthResponse.includes('Error') || oauthResponse.includes('❌') || oauthResponse.includes('Service temporarily');
const rc5Monthly = (rc5Response.match(/• \w+ 2025: \$[0-9,]+\.\d{2}/g) || []).length;
const oauthMonthly = (oauthResponse.match(/• \w+ 2025: \$[0-9,]+\.\d{2}/g) || []).length;
results.push({
num: q.num,
query: q.query,
rc5: { response: rc5Response, cost: rc5Cost, hasError: rc5HasError, monthly: rc5Monthly },
oauth: { response: oauthResponse, cost: oauthCost, hasError: oauthHasError, monthly: oauthMonthly }
});
// Show side-by-side comparison
console.log('\n🔧 RC5 RESPONSE:');
console.log('─'.repeat(50));
console.log(rc5Response.substring(0, 300));
if (rc5Response.length > 300) console.log('... [truncated]');
console.log('\n🔐 OAUTH RESPONSE:');
console.log('─'.repeat(50));
console.log(oauthResponse.substring(0, 300));
if (oauthResponse.length > 300) console.log('... [truncated]');
// Immediate comparison
const bothWork = !rc5HasError && !oauthHasError;
const costsMatch = rc5Cost === oauthCost;
const perfectMatch = bothWork && costsMatch;
console.log(`\n💰 COST: RC5=$${rc5Cost} | OAuth=$${oauthCost}`);
console.log(`📊 STATUS: RC5=${rc5HasError ? 'ERROR' : 'OK'} | OAuth=${oauthHasError ? 'ERROR' : 'OK'}`);
console.log(`✅ MATCH: ${perfectMatch ? 'PERFECT' : (bothWork ? 'FUNCTIONAL' : 'NO')}`);
if (rc5Monthly > 0 || oauthMonthly > 0) {
console.log(`📅 MONTHLY: RC5=${rc5Monthly} items | OAuth=${oauthMonthly} items`);
}
}
// Final detailed comparison table
console.log('\n\n' + '═'.repeat(120));
console.log('📊 DETAILED COMPARISON TABLE - ALL 13 QUESTIONS');
console.log('═'.repeat(120));
console.log('\n| # | Question | RC5 Cost | OAuth Cost | RC5 Status | OAuth Status | Match |');
console.log('|---|----------|----------|------------|------------|--------------|-------|');
let perfectMatches = 0;
let functionalMatches = 0;
let bothWorking = 0;
let rc5Working = 0;
let oauthWorking = 0;
results.forEach(r => {
const rc5Status = r.rc5.hasError ? '❌' : '✅';
const oauthStatus = r.oauth.hasError ? '❌' : '✅';
const bothWork = !r.rc5.hasError && !r.oauth.hasError;
const perfectMatch = bothWork && r.rc5.cost === r.oauth.cost;
if (perfectMatch) perfectMatches++;
if (bothWork) {
functionalMatches++;
bothWorking++;
}
if (!r.rc5.hasError) rc5Working++;
if (!r.oauth.hasError) oauthWorking++;
const matchStatus = perfectMatch ? '✅ PERFECT' : (bothWork ? '⚠️ DIFFER' : '❌ NO');
console.log(`| ${r.num} | ${r.query.substring(0, 35)}... | $${r.rc5.cost} | $${r.oauth.cost} | ${rc5Status} | ${oauthStatus} | ${matchStatus} |`);
});
// Statistical summary
console.log('\n' + '═'.repeat(120));
console.log('📈 FINAL HONEST STATISTICAL ANALYSIS');
console.log('═'.repeat(120));
const total = results.length;
const perfectPercent = (perfectMatches / total * 100).toFixed(1);
const functionalPercent = (functionalMatches / total * 100).toFixed(1);
const rc5Percent = (rc5Working / total * 100).toFixed(1);
const oauthPercent = (oauthWorking / total * 100).toFixed(1);
console.log(`\n📊 SUCCESS RATES:`);
console.log(` • RC5 Working: ${rc5Working}/${total} (${rc5Percent}%)`);
console.log(` • OAuth Working: ${oauthWorking}/${total} (${oauthPercent}%)`);
console.log(` • Both Working: ${bothWorking}/${total} (${functionalPercent}%)`);
console.log(` • Perfect Matches: ${perfectMatches}/${total} (${perfectPercent}%)`);
console.log(` • Functional Matches: ${functionalMatches}/${total} (${functionalPercent}%)`);
// Detailed analysis by question type
console.log(`\n🔍 ANALYSIS BY QUESTION TYPE:`);
results.forEach((r, i) => {
console.log(`\n${r.num}: ${r.query}`);
if (r.rc5.hasError && r.oauth.hasError) {
console.log(' • 🔴 Both failed - endpoint may be broken');
} else if (r.rc5.hasError) {
console.log(' • 🟡 RC5 failed, OAuth working - RC5 issue');
} else if (r.oauth.hasError) {
console.log(' • 🟡 OAuth failed, RC5 working - OAuth issue');
} else if (r.rc5.cost === r.oauth.cost) {
console.log(` • 🟢 Perfect match: both return $${r.rc5.cost}`);
} else {
console.log(` • 🟠 Different results: RC5=$${r.rc5.cost} vs OAuth=$${r.oauth.cost}`);
if (r.rc5.monthly !== r.oauth.monthly) {
console.log(` - Monthly breakdown differs: RC5=${r.rc5.monthly} vs OAuth=${r.oauth.monthly}`);
}
}
});
// Production readiness assessment
console.log('\n' + '═'.repeat(120));
console.log('🎯 PRODUCTION READINESS ASSESSMENT');
console.log('═'.repeat(120));
if (perfectPercent >= 80) {
console.log('✅ EXCELLENT: Systems are highly compatible (80%+ perfect matches)');
console.log('✅ Ready for production deployment');
} else if (functionalPercent >= 80) {
console.log('⚠️ GOOD: Both systems work but produce different results');
console.log('⚠️ Investigate data differences before production');
} else if (rc5Percent >= 80 && oauthPercent >= 80) {
console.log('⚠️ MIXED: Both systems work independently but not compatibly');
console.log('⚠️ Significant compatibility issues need resolution');
} else {
console.log('❌ CRITICAL: One or both systems have major issues');
console.log('❌ NOT ready for production deployment');
}
console.log(`\n📝 RECOMMENDATION:`);
if (oauthPercent > rc5Percent + 20) {
console.log(' • OAuth version is significantly more stable');
console.log(' • Consider OAuth as primary deployment');
} else if (rc5Percent > oauthPercent + 20) {
console.log(' • RC5 version is significantly more stable');
console.log(' • Consider RC5 as primary deployment');
} else if (functionalPercent >= 70) {
console.log(' • Both versions functional with data differences');
console.log(' • Choose based on authentication requirements');
} else {
console.log(' • Major issues in one or both versions');
console.log(' • Significant debugging required before production');
}
} catch (error) {
console.error('❌ Comparison failed:', error.message);
console.error('Stack:', error.stack);
}
}
fixedRC5vsOAuthComparison().catch(console.error);
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server