Umbrella MCP Server
FINAL-GITHUB-VS-OAUTH-COMPARISON.cjs•14.9 kB
#!/usr/bin/env node
// FINAL GITHUB vs OAUTH COMPARISON - With proper session handling and timing
// This handles the GitHub version's session management issues correctly
const axios = require('axios');
async function finalGithubVsOauthComparison() {
console.log('🔬 FINAL GITHUB vs OAUTH COMPARISON - WITH PROPER SESSION HANDLING');
console.log('═'.repeat(100));
console.log('GitHub Version: http://localhost:3000 (commit 3a3d0f8) - 4 tools with session issues');
console.log('OAuth Version: http://localhost:3001 (OAuth implementation) - 27 tools with session IDs');
console.log('Testing 8 questions mapped to available GitHub tools\n');
const results = [];
try {
// Authenticate with OAuth version
console.log('🔐 Authenticating with OAuth version...');
const oauthSessionResponse = await axios.post('http://localhost:8080/api/session/create');
const oauthSessionId = oauthSessionResponse.data.sessionId;
await axios.post('http://localhost:8080/oauth/callback', {
sessionId: oauthSessionId,
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
});
console.log(`✅ OAuth authenticated: ${oauthSessionId}`);
// Test questions that GitHub version can handle
const questions = [
{
num: 'Q1',
query: 'what is my total cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { userQuery: 'what is my total cost?' },
oauthArgs: { periodGranLevel: 'month', userQuery: 'what is my total cost?' }
},
{
num: 'Q2',
query: 'what is my total AWS cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'aws', userQuery: 'what is my total AWS cost?' },
oauthArgs: { cloud_context: 'aws', periodGranLevel: 'month', userQuery: 'what is my total AWS cost?' }
},
{
num: 'Q3',
query: 'what is my total GCP cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'gcp', userQuery: 'what is my total GCP cost?' },
oauthArgs: { cloud_context: 'gcp', periodGranLevel: 'month', userQuery: 'what is my total GCP cost?' }
},
{
num: 'Q4',
query: 'what is my total Azure cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'azure', userQuery: 'what is my total Azure cost?' },
oauthArgs: { cloud_context: 'azure', periodGranLevel: 'month', userQuery: 'what is my total Azure cost?' }
},
{
num: 'Q5',
query: 'show me all available accounts',
githubTool: 'get_accounts',
oauthTool: 'api___user_management_accounts',
githubArgs: { userQuery: 'show me all available accounts' },
oauthArgs: { userQuery: 'show me all available accounts' }
},
{
num: 'Q6',
query: 'what do you recommend for saving AWS costs?',
githubTool: 'get_recommendations',
oauthTool: 'api___recommendationsNew_heatmap_summary',
githubArgs: { userQuery: 'what do you recommend for saving AWS costs?' },
oauthArgs: { userQuery: 'what do you recommend for saving AWS costs?' }
},
{
num: 'Q7',
query: 'show me the total cost per month',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { userQuery: 'show me the total cost per month' },
oauthArgs: { periodGranLevel: 'month', userQuery: 'show me the total cost per month' }
},
{
num: 'Q8',
query: 'show me AWS amortized costs for the last 8 months',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'aws', userQuery: 'show me AWS amortized costs for the last 8 months' },
oauthArgs: {
cloud_context: 'aws',
isAmortized: true,
periodGranLevel: 'month',
startDate: '2025-01-01',
endDate: '2025-08-31',
userQuery: 'show me AWS amortized costs for the last 8 months'
}
}
];
console.log('═'.repeat(100));
console.log('🔍 TESTING ALL QUESTIONS WITH PROPER GITHUB SESSION MANAGEMENT');
console.log('═'.repeat(100));
// Helper function to authenticate and call GitHub tool with proper timing
async function callGithubToolWithPropperTiming(toolName, args) {
try {
// Authenticate first
console.log('🔐 Authenticating GitHub for this call...');
const authRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: 'authenticate_user',
arguments: {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
}
},
id: 1
};
const authResponse = await axios.post('http://localhost:3000/sse', authRequest);
console.log('✅ GitHub authentication successful');
// Wait a bit for session to be established
await new Promise(resolve => setTimeout(resolve, 1000));
// Then call the tool
const toolRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: toolName,
arguments: args
},
id: 2
};
const response = await axios.post('http://localhost:3000/sse', toolRequest);
return response.data.result.content[0].text;
} catch (error) {
console.log(`❌ GitHub ${toolName} error:`, error.response?.data?.error || error.message);
return `❌ GitHub Error: ${error.response?.data?.error || error.message}`;
}
}
for (const q of questions) {
console.log(`\n📋 ${q.num}: ${q.query}`);
console.log('─'.repeat(100));
// Test GitHub version with proper session handling
console.log('🐙 Testing GitHub version (with proper session timing)...');
const githubResponse = await callGithubToolWithPropperTiming(q.githubTool, q.githubArgs);
// Test OAuth version
console.log('🔐 Testing OAuth version...');
let oauthResponse = '';
try {
const oauthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: q.oauthTool,
arguments: {
sessionId: oauthSessionId,
...q.oauthArgs
}
},
id: parseInt(q.num.substring(1))
};
const response = await axios.post('http://localhost:3001/sse', oauthRequest);
oauthResponse = response.data.result.content[0].text;
} catch (error) {
oauthResponse = `❌ OAuth Error: ${error.response?.data?.error || error.message}`;
}
// Extract comparison data
const githubCost = githubResponse.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const oauthCost = oauthResponse.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const githubHasError = githubResponse.includes('Error') || githubResponse.includes('❌');
const oauthHasError = oauthResponse.includes('Error') || oauthResponse.includes('❌') || oauthResponse.includes('Service temporarily');
results.push({
num: q.num,
query: q.query,
github: { response: githubResponse, cost: githubCost, hasError: githubHasError },
oauth: { response: oauthResponse, cost: oauthCost, hasError: oauthHasError }
});
// Show responses
console.log('\n🐙 GITHUB RESPONSE:');
console.log('─'.repeat(50));
console.log(githubResponse.substring(0, 200));
if (githubResponse.length > 200) console.log('... [truncated]');
console.log('\n🔐 OAUTH RESPONSE:');
console.log('─'.repeat(50));
console.log(oauthResponse.substring(0, 200));
if (oauthResponse.length > 200) console.log('... [truncated]');
// Comparison
const bothWork = !githubHasError && !oauthHasError;
const costsMatch = githubCost === oauthCost;
const perfectMatch = bothWork && costsMatch;
console.log(`\n💰 COST: GitHub=$${githubCost} | OAuth=$${oauthCost}`);
console.log(`📊 STATUS: GitHub=${githubHasError ? 'ERROR' : 'OK'} | OAuth=${oauthHasError ? 'ERROR' : 'OK'}`);
console.log(`✅ MATCH: ${perfectMatch ? 'PERFECT' : (bothWork ? 'FUNCTIONAL' : 'NO')}`);
}
// Final comparison table
console.log('\n\n' + '═'.repeat(120));
console.log('📊 FINAL HONEST COMPARISON: GITHUB (3a3d0f8) vs OAUTH VERSION');
console.log('═'.repeat(120));
console.log('\n| # | Question | GitHub Cost | OAuth Cost | GitHub Status | OAuth Status | Match |');
console.log('|---|----------|-------------|------------|---------------|--------------|-------|');
let perfectMatches = 0;
let functionalMatches = 0;
let githubWorking = 0;
let oauthWorking = 0;
results.forEach(r => {
const githubStatus = r.github.hasError ? '❌' : '✅';
const oauthStatus = r.oauth.hasError ? '❌' : '✅';
const bothWork = !r.github.hasError && !r.oauth.hasError;
const perfectMatch = bothWork && r.github.cost === r.oauth.cost;
if (perfectMatch) perfectMatches++;
if (bothWork) functionalMatches++;
if (!r.github.hasError) githubWorking++;
if (!r.oauth.hasError) oauthWorking++;
const matchStatus = perfectMatch ? '✅ PERFECT' : (bothWork ? '⚠️ DIFFER' : '❌ NO');
console.log(`| ${r.num} | ${r.query.substring(0, 30)}... | $${r.github.cost} | $${r.oauth.cost} | ${githubStatus} | ${oauthStatus} | ${matchStatus} |`);
});
// Statistical analysis
console.log('\n' + '═'.repeat(120));
console.log('📈 FINAL HONEST STATISTICAL ANALYSIS');
console.log('═'.repeat(120));
const total = results.length;
const perfectPercent = (perfectMatches / total * 100).toFixed(1);
const functionalPercent = (functionalMatches / total * 100).toFixed(1);
const githubPercent = (githubWorking / total * 100).toFixed(1);
const oauthPercent = (oauthWorking / total * 100).toFixed(1);
console.log(`\n📊 SUCCESS RATES:`);
console.log(` • GitHub (3a3d0f8) Working: ${githubWorking}/${total} (${githubPercent}%)`);
console.log(` • OAuth Version Working: ${oauthWorking}/${total} (${oauthPercent}%)`);
console.log(` • Both Working: ${functionalMatches}/${total} (${functionalPercent}%)`);
console.log(` • Perfect Matches: ${perfectMatches}/${total} (${perfectPercent}%)`);
console.log(`\n🔍 KEY ARCHITECTURAL DIFFERENCES:`);
console.log(` • GitHub: 4 tools (authenticate_user, get_costs, get_accounts, get_recommendations)`);
console.log(` • OAuth: 27 tools (comprehensive RC5 endpoint coverage)`);
console.log(` • GitHub: Session management issues - needs auth per request sequence`);
console.log(` • OAuth: Proper session-based authentication with persistent sessions`);
console.log(` • GitHub: Simplified response formatting`);
console.log(` • OAuth: RC5-compatible response formatting with monthly breakdowns`);
// Question analysis
console.log(`\n🔍 DETAILED ANALYSIS BY QUESTION:`);
results.forEach((r, i) => {
console.log(`\n${r.num}: ${r.query}`);
if (r.github.hasError && r.oauth.hasError) {
console.log(' • 🔴 Both failed - endpoint/authentication issue');
} else if (r.github.hasError) {
console.log(' • 🟡 GitHub failed, OAuth working - GitHub session/auth issue');
} else if (r.oauth.hasError) {
console.log(' • 🟡 OAuth failed, GitHub working - OAuth endpoint issue');
} else if (r.github.cost === r.oauth.cost) {
console.log(` • 🟢 Perfect match: both return $${r.github.cost}`);
} else {
console.log(` • 🟠 Different results: GitHub=$${r.github.cost} vs OAuth=$${r.oauth.cost}`);
console.log(` - Could be due to different data processing or API parameters`);
}
});
// Final verdict
console.log('\n' + '═'.repeat(120));
console.log('🎯 FINAL PRODUCTION ASSESSMENT');
console.log('═'.repeat(120));
console.log(`\n🏗️ ARCHITECTURAL COMPARISON:`);
if (githubPercent >= 75 && oauthPercent >= 75) {
console.log('✅ BOTH FUNCTIONAL: Both versions work but have different architectures');
} else if (oauthPercent >= 75) {
console.log('✅ OAUTH SUPERIOR: OAuth version more stable and feature-complete');
} else if (githubPercent >= 75) {
console.log('⚠️ GITHUB FUNCTIONAL: But limited to 4 tools vs OAuth 27 tools');
} else {
console.log('❌ BOTH HAVE ISSUES: Neither version fully stable');
}
console.log(`\n📝 PRODUCTION RECOMMENDATION:`);
if (oauthPercent >= 80) {
console.log('✅ RECOMMEND OAUTH VERSION:');
console.log(' • 27 comprehensive tools vs GitHub 4 tools');
console.log(' • Better session management');
console.log(' • RC5-compatible response formatting');
console.log(' • More stable authentication flow');
} else if (githubPercent >= 80 && functionalPercent < 50) {
console.log('⚠️ CONSIDER GITHUB VERSION IF:');
console.log(' • Only basic functionality needed (costs, accounts, recommendations)');
console.log(' • Simpler architecture preferred');
console.log(' • But fix session management issues first');
} else {
console.log('⚠️ BOTH VERSIONS NEED WORK:');
console.log(' • GitHub: Fix session management and add more tools');
console.log(' • OAuth: Investigate endpoint stability issues');
}
console.log(`\n🔧 GITHUB VERSION ISSUES IDENTIFIED:`);
console.log(' • Session management race conditions');
console.log(' • "Most recent session" lookup is fragile');
console.log(' • No proper session cleanup');
console.log(' • Limited to 4 tools vs comprehensive coverage needed');
console.log(`\n⭐ OAUTH VERSION ADVANTAGES:`);
console.log(' • Complete RC5 tool coverage (27 endpoints)');
console.log(' • Proper session-based authentication');
console.log(' • Better response formatting and error handling');
console.log(' • External OAuth service for security');
} catch (error) {
console.error('❌ Comparison failed:', error.message);
console.error('Stack:', error.stack);
}
}
finalGithubVsOauthComparison().catch(console.error);
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server