Umbrella MCP Server
CORRECTED-GITHUB-VS-OAUTH-COMPARISON.cjs•11 kB
#!/usr/bin/env node
// CORRECTED GITHUB LATEST VERSION vs OAUTH COMPARISON
// Testing GitHub version (3a3d0f8) with its correct 4 tools vs OAuth version
const axios = require('axios');
async function correctedGithubVsOauthComparison() {
console.log('🔬 CORRECTED GITHUB LATEST VERSION (3a3d0f8) vs OAUTH COMPARISON');
console.log('═'.repeat(100));
console.log('GitHub Latest: http://localhost:3000 (final-mcp-server.cjs) - 4 tools');
console.log('OAuth Version: http://localhost:3001 (mcp-server-with-oauth-rc5.cjs) - 27 tools');
console.log('Using CORRECT tool names for each version\n');
const results = [];
try {
// Authenticate with OAuth version
console.log('🔐 Authenticating with OAuth version...');
const oauthSessionResponse = await axios.post('http://localhost:8080/api/session/create');
const oauthSessionId = oauthSessionResponse.data.sessionId;
await axios.post('http://localhost:8080/oauth/callback', {
sessionId: oauthSessionId,
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
});
console.log(`✅ OAuth authenticated: ${oauthSessionId}\n`);
// Test questions using CORRECT tools for each version
const tests = [
{
num: 'Q1',
query: 'what is my total cost?',
githubTool: 'get_costs', // GitHub has get_costs
oauthTool: 'api___invoices_caui', // OAuth has api___invoices_caui
githubArgs: { userQuery: 'what is my total cost?' },
oauthArgs: { periodGranLevel: 'month', userQuery: 'what is my total cost?' }
},
{
num: 'Q2',
query: 'what is my total AWS cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'aws', userQuery: 'what is my total AWS cost?' },
oauthArgs: { cloud_context: 'aws', periodGranLevel: 'month', userQuery: 'what is my total AWS cost?' }
},
{
num: 'Q3',
query: 'what is my total GCP cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'gcp', userQuery: 'what is my total GCP cost?' },
oauthArgs: { cloud_context: 'gcp', periodGranLevel: 'month', userQuery: 'what is my total GCP cost?' }
},
{
num: 'Q4',
query: 'show me all available accounts',
githubTool: 'get_accounts', // GitHub has get_accounts
oauthTool: 'api___user_management_accounts', // OAuth has api___user_management_accounts
githubArgs: { userQuery: 'show me all available accounts' },
oauthArgs: { userQuery: 'show me all available accounts' }
},
{
num: 'Q5',
query: 'what do you recommend for saving AWS costs?',
githubTool: 'get_recommendations', // GitHub has get_recommendations
oauthTool: 'api___recommendationsNew_heatmap_summary', // OAuth has api___recommendationsNew_heatmap_summary
githubArgs: { userQuery: 'what do you recommend for saving AWS costs?' },
oauthArgs: { userQuery: 'what do you recommend for saving AWS costs?' }
}
];
console.log('═'.repeat(100));
console.log('TESTING EACH QUESTION WITH CORRECT TOOLS');
console.log('═'.repeat(100));
for (const test of tests) {
console.log(`\n🔍 ${test.num}: ${test.query}`);
console.log('─'.repeat(100));
// Test GitHub latest version with its correct tools
let githubResponse = '';
try {
// First authenticate with GitHub
const githubAuthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: 'authenticate_user',
arguments: {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
}
},
id: 1
};
await axios.post('http://localhost:3000/sse', githubAuthRequest);
// Wait for session
await new Promise(resolve => setTimeout(resolve, 500));
// Then call the tool
const githubRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: test.githubTool,
arguments: test.githubArgs
},
id: 2
};
const response = await axios.post('http://localhost:3000/sse', githubRequest);
githubResponse = response.data.result.content[0].text;
} catch (error) {
githubResponse = `❌ GitHub Error: ${error.response?.data?.error || error.message}`;
}
// Test OAuth version
let oauthResponse = '';
try {
const oauthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: test.oauthTool,
arguments: {
sessionId: oauthSessionId,
...test.oauthArgs
}
},
id: parseInt(test.num.substring(1))
};
const response = await axios.post('http://localhost:3001/sse', oauthRequest);
oauthResponse = response.data.result.content[0].text;
} catch (error) {
oauthResponse = `❌ OAuth Error: ${error.response?.data?.error || error.message}`;
}
// Extract costs for comparison
const githubCost = githubResponse.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const oauthCost = oauthResponse.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const githubHasError = githubResponse.includes('Error') || githubResponse.includes('❌');
const oauthHasError = oauthResponse.includes('Error') || oauthResponse.includes('❌') || oauthResponse.includes('Service temporarily');
results.push({
num: test.num,
query: test.query,
github: { response: githubResponse, cost: githubCost, hasError: githubHasError },
oauth: { response: oauthResponse, cost: oauthCost, hasError: oauthHasError }
});
// Show responses
console.log('\n📊 GITHUB LATEST VERSION RESPONSE (port 3000):');
console.log('─'.repeat(50));
console.log(githubResponse.substring(0, 300));
if (githubResponse.length > 300) console.log('... [truncated]');
console.log('\n🔧 OAUTH VERSION RESPONSE (port 3001):');
console.log('─'.repeat(50));
console.log(oauthResponse.substring(0, 300));
if (oauthResponse.length > 300) console.log('... [truncated]');
// Comparison
const bothWork = !githubHasError && !oauthHasError;
const costsMatch = githubCost === oauthCost;
const perfectMatch = bothWork && costsMatch;
console.log(`\n💰 COST COMPARISON:`);
console.log(`GitHub Latest: $${githubCost} | OAuth: $${oauthCost} | Match: ${perfectMatch ? '✅' : '❌'}`);
}
// Final comparison table
console.log('\n\n' + '═'.repeat(120));
console.log('📊 FINAL HONEST COMPARISON: GITHUB LATEST (3a3d0f8) vs OAUTH VERSION');
console.log('═'.repeat(120));
console.log('\n| Question | GitHub Latest Version | OAuth Version | Match |');
console.log('|----------|----------------------|---------------|-------|');
let perfectMatches = 0;
let functionalMatches = 0;
let githubWorking = 0;
let oauthWorking = 0;
results.forEach(r => {
const githubStatus = r.github.hasError ? '❌' : '✅';
const oauthStatus = r.oauth.hasError ? '❌' : '✅';
const bothWork = !r.github.hasError && !r.oauth.hasError;
const perfectMatch = bothWork && r.github.cost === r.oauth.cost;
if (perfectMatch) perfectMatches++;
if (bothWork) functionalMatches++;
if (!r.github.hasError) githubWorking++;
if (!r.oauth.hasError) oauthWorking++;
const matchStatus = perfectMatch ? '✅ PERFECT' : (bothWork ? '⚠️ DIFFER' : '❌ NO');
console.log(`| ${r.num}: ${r.query.substring(0, 35)}... | $${r.github.cost} (${githubStatus}) | $${r.oauth.cost} (${oauthStatus}) | ${matchStatus} |`);
});
// Statistical analysis
console.log('\n' + '═'.repeat(120));
console.log('📈 HONEST STATISTICAL ANALYSIS');
console.log('═'.repeat(120));
const total = results.length;
const perfectPercent = (perfectMatches / total * 100).toFixed(1);
const functionalPercent = (functionalMatches / total * 100).toFixed(1);
const githubPercent = (githubWorking / total * 100).toFixed(1);
const oauthPercent = (oauthWorking / total * 100).toFixed(1);
console.log(`\n📊 SUCCESS RATES:`);
console.log(` • GitHub Latest (3a3d0f8) Working: ${githubWorking}/${total} (${githubPercent}%)`);
console.log(` • OAuth Version Working: ${oauthWorking}/${total} (${oauthPercent}%)`);
console.log(` • Both Working: ${functionalMatches}/${total} (${functionalPercent}%)`);
console.log(` • Perfect Matches: ${perfectMatches}/${total} (${perfectPercent}%)`);
console.log(`\n🏗️ ARCHITECTURAL DIFFERENCES:`);
console.log(` • GitHub Latest: 4 basic tools (authenticate_user, get_costs, get_accounts, get_recommendations)`);
console.log(` • OAuth Version: 27 comprehensive tools (full RC5 API coverage)`);
console.log(` • GitHub Latest: Simplified session management (has issues)`);
console.log(` • OAuth Version: Proper OAuth session-based authentication`);
// Final assessment
console.log('\n' + '═'.repeat(120));
console.log('🎯 FINAL PRODUCTION ASSESSMENT');
console.log('═'.repeat(120));
if (perfectPercent >= 80) {
console.log('✅ HIGH COMPATIBILITY: Both versions produce similar results');
} else if (functionalPercent >= 60) {
console.log('⚠️ FUNCTIONAL BUT DIFFERENT: Both work but produce different results');
} else if (oauthPercent >= 80) {
console.log('✅ OAUTH SUPERIOR: OAuth version significantly more stable');
} else if (githubPercent >= 80) {
console.log('⚠️ GITHUB FUNCTIONAL: But limited compared to OAuth comprehensive coverage');
} else {
console.log('❌ BOTH NEED WORK: Neither version fully functional');
}
console.log(`\n📝 PRODUCTION RECOMMENDATION:`);
if (oauthPercent >= 80 && functionalPercent >= 60) {
console.log('✅ DEPLOY OAUTH VERSION: More stable and comprehensive tool coverage');
} else if (githubPercent >= 80 && oauthPercent < 60) {
console.log('⚠️ CONSIDER GITHUB VERSION: Simpler but limited functionality');
} else {
console.log('⚠️ MORE WORK NEEDED: Both versions require improvements');
}
} catch (error) {
console.error('❌ Comparison failed:', error.message);
console.error('Stack:', error.stack);
}
}
correctedGithubVsOauthComparison().catch(console.error);
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server