Umbrella MCP Server
FIXED-GITHUB-COMPARISON.cjs•12.7 kB
#!/usr/bin/env node
// FIXED GitHub (3a3d0f8) vs OAuth comparison with proper session handling
const axios = require('axios');
async function fixedGithubComparison() {
console.log('🔬 FIXED GITHUB (3a3d0f8) vs OAUTH COMPARISON');
console.log('═'.repeat(80));
console.log('Properly testing GitHub version with session awareness');
console.log('GitHub has 4 tools: authenticate_user, get_costs, get_accounts, get_recommendations\n');
const results = [];
try {
// Authenticate with OAuth version
console.log('🔐 Authenticating with OAuth version...');
const oauthSessionResponse = await axios.post('http://localhost:8080/api/session/create');
const oauthSessionId = oauthSessionResponse.data.sessionId;
await axios.post('http://localhost:8080/oauth/callback', {
sessionId: oauthSessionId,
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
});
console.log(`✅ OAuth authenticated: ${oauthSessionId}`);
// For GitHub - we need to authenticate for each request (session issue)
console.log('🔐 GitHub version requires auth per request due to session issue\n');
// Define test questions mapped to GitHub's 4 tools
const questions = [
{
num: 'Q1',
query: 'what is my total cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { userQuery: 'what is my total cost?' },
oauthArgs: { userQuery: 'what is my total cost?', periodGranLevel: 'month' }
},
{
num: 'Q2',
query: 'what is my total AWS cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'aws', userQuery: 'what is my total AWS cost?' },
oauthArgs: { cloud_context: 'aws', userQuery: 'what is my total AWS cost?', periodGranLevel: 'month' }
},
{
num: 'Q3',
query: 'what is my total GCP cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'gcp', userQuery: 'what is my total GCP cost?' },
oauthArgs: { cloud_context: 'gcp', userQuery: 'what is my total GCP cost?', periodGranLevel: 'month' }
},
{
num: 'Q4',
query: 'what is my total Azure cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'azure', userQuery: 'what is my total Azure cost?' },
oauthArgs: { cloud_context: 'azure', userQuery: 'what is my total Azure cost?', periodGranLevel: 'month' }
},
{
num: 'Q5',
query: 'show me all available accounts',
githubTool: 'get_accounts',
oauthTool: 'api___user_management_accounts',
githubArgs: { userQuery: 'show me all available accounts' },
oauthArgs: { userQuery: 'show me all available accounts' }
},
{
num: 'Q6',
query: 'what do you recommend for saving AWS costs?',
githubTool: 'get_recommendations',
oauthTool: 'api___recommendationsNew_heatmap_summary',
githubArgs: { userQuery: 'what do you recommend for saving AWS costs?' },
oauthArgs: { userQuery: 'what do you recommend for saving AWS costs?' }
},
{
num: 'Q7',
query: 'show me the total cost per month',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { userQuery: 'show me the total cost per month' },
oauthArgs: { userQuery: 'show me the total cost per month', periodGranLevel: 'month' }
},
{
num: 'Q8',
query: 'show me AWS amortized costs for the last 8 months',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
githubArgs: { cloud_context: 'aws', userQuery: 'show me AWS amortized costs for the last 8 months' },
oauthArgs: {
cloud_context: 'aws',
isAmortized: true,
periodGranLevel: 'month',
startDate: '2025-01-01',
endDate: '2025-08-31',
userQuery: 'show me AWS amortized costs for the last 8 months'
}
}
];
console.log('═'.repeat(80));
console.log('🔍 TESTING GITHUB vs OAUTH WITH PROPER SESSION HANDLING');
console.log('═'.repeat(80));
// Helper function to authenticate and call GitHub tool
async function callGithubTool(toolName, args) {
try {
// Authenticate first
const authRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: 'authenticate_user',
arguments: {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
}
},
id: 1
};
await axios.post('http://localhost:3000/sse', authRequest);
// Then call the tool immediately
const toolRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: toolName,
arguments: args
},
id: 2
};
const response = await axios.post('http://localhost:3000/sse', toolRequest);
return response.data.result.content[0].text;
} catch (error) {
return `❌ GitHub Error: ${error.response?.data?.error || error.message}`;
}
}
for (const q of questions) {
console.log(`\n📋 ${q.num}: ${q.query}`);
console.log('─'.repeat(80));
// Test GitHub version with proper auth
console.log('🐙 Testing GitHub version (with fresh auth)...');
const githubResponse = await callGithubTool(q.githubTool, q.githubArgs);
// Test OAuth version
console.log('🔐 Testing OAuth version...');
let oauthResponse = '';
try {
const oauthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: q.oauthTool,
arguments: {
sessionId: oauthSessionId,
...q.oauthArgs
}
},
id: parseInt(q.num.substring(1))
};
const response = await axios.post('http://localhost:3001/sse', oauthRequest);
oauthResponse = response.data.result.content[0].text;
} catch (error) {
oauthResponse = `❌ OAuth Error: ${error.response?.data?.error || error.message}`;
}
// Extract comparison data
const githubCost = githubResponse.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const oauthCost = oauthResponse.match(/(?:total|cost|savings).*?\$([0-9,]+\.?\d*)/i)?.[1] || 'N/A';
const githubHasError = githubResponse.includes('Error') || githubResponse.includes('❌');
const oauthHasError = oauthResponse.includes('Error') || oauthResponse.includes('❌') || oauthResponse.includes('Service temporarily');
results.push({
num: q.num,
query: q.query,
github: { response: githubResponse, cost: githubCost, hasError: githubHasError },
oauth: { response: oauthResponse, cost: oauthCost, hasError: oauthHasError }
});
// Show responses
console.log('\n🐙 GITHUB RESPONSE:');
console.log('─'.repeat(40));
console.log(githubResponse.substring(0, 300));
if (githubResponse.length > 300) console.log('... [truncated]');
console.log('\n🔐 OAUTH RESPONSE:');
console.log('─'.repeat(40));
console.log(oauthResponse.substring(0, 300));
if (oauthResponse.length > 300) console.log('... [truncated]');
// Comparison
const bothWork = !githubHasError && !oauthHasError;
const costsMatch = githubCost === oauthCost;
const perfectMatch = bothWork && costsMatch;
console.log(`\n💰 COST: GitHub=$${githubCost} | OAuth=$${oauthCost}`);
console.log(`📊 STATUS: GitHub=${githubHasError ? 'ERROR' : 'OK'} | OAuth=${oauthHasError ? 'ERROR' : 'OK'}`);
console.log(`✅ MATCH: ${perfectMatch ? 'PERFECT' : (bothWork ? 'FUNCTIONAL' : 'NO')}`);
}
// Final comparison table
console.log('\n\n' + '═'.repeat(100));
console.log('📊 FINAL COMPARISON: GITHUB (3a3d0f8) vs OAUTH VERSION');
console.log('═'.repeat(100));
console.log('\n| # | Question | GitHub Cost | OAuth Cost | GitHub Status | OAuth Status | Match |');
console.log('|---|----------|-------------|------------|---------------|--------------|-------|');
let perfectMatches = 0;
let functionalMatches = 0;
let githubWorking = 0;
let oauthWorking = 0;
results.forEach(r => {
const githubStatus = r.github.hasError ? '❌' : '✅';
const oauthStatus = r.oauth.hasError ? '❌' : '✅';
const bothWork = !r.github.hasError && !r.oauth.hasError;
const perfectMatch = bothWork && r.github.cost === r.oauth.cost;
if (perfectMatch) perfectMatches++;
if (bothWork) functionalMatches++;
if (!r.github.hasError) githubWorking++;
if (!r.oauth.hasError) oauthWorking++;
const matchStatus = perfectMatch ? '✅ PERFECT' : (bothWork ? '⚠️ DIFFER' : '❌ NO');
console.log(`| ${r.num} | ${r.query.substring(0, 30)}... | $${r.github.cost} | $${r.oauth.cost} | ${githubStatus} | ${oauthStatus} | ${matchStatus} |`);
});
// Statistical analysis
console.log('\n' + '═'.repeat(100));
console.log('📈 HONEST STATISTICAL ANALYSIS');
console.log('═'.repeat(100));
const total = results.length;
const perfectPercent = (perfectMatches / total * 100).toFixed(1);
const functionalPercent = (functionalMatches / total * 100).toFixed(1);
const githubPercent = (githubWorking / total * 100).toFixed(1);
const oauthPercent = (oauthWorking / total * 100).toFixed(1);
console.log(`\n📊 SUCCESS RATES:`);
console.log(` • GitHub (3a3d0f8) Working: ${githubWorking}/${total} (${githubPercent}%)`);
console.log(` • OAuth Version Working: ${oauthWorking}/${total} (${oauthPercent}%)`);
console.log(` • Both Working: ${functionalMatches}/${total} (${functionalPercent}%)`);
console.log(` • Perfect Matches: ${perfectMatches}/${total} (${perfectPercent}%)`);
console.log(`\n🔍 KEY DIFFERENCES:`);
console.log(` • GitHub has 4 tools vs OAuth's 27 tools`);
console.log(` • GitHub has session management issues`);
console.log(` • OAuth has modern authentication flow`);
console.log(` • OAuth has richer response formatting`);
// Question analysis
console.log(`\n🔍 DETAILED ANALYSIS:`);
results.forEach((r, i) => {
console.log(`\n${r.num}: ${r.query}`);
if (r.github.hasError && r.oauth.hasError) {
console.log(' • 🔴 Both failed - endpoint issue');
} else if (r.github.hasError) {
console.log(' • 🟡 GitHub failed, OAuth working');
} else if (r.oauth.hasError) {
console.log(' • 🟡 OAuth failed, GitHub working');
} else if (r.github.cost === r.oauth.cost) {
console.log(` • 🟢 Perfect match: $${r.github.cost}`);
} else {
console.log(` • 🟠 Different costs: GitHub=$${r.github.cost} vs OAuth=$${r.oauth.cost}`);
}
});
// Final verdict
console.log('\n' + '═'.repeat(100));
console.log('🎯 FINAL HONEST VERDICT');
console.log('═'.repeat(100));
if (perfectPercent >= 70) {
console.log('✅ HIGH COMPATIBILITY: Versions are highly compatible');
} else if (functionalPercent >= 60) {
console.log('⚠️ MODERATE COMPATIBILITY: Both work but different results');
} else if (oauthPercent > githubPercent) {
console.log('🔐 OAUTH SUPERIOR: OAuth version more stable');
} else if (githubPercent > oauthPercent) {
console.log('🐙 GITHUB SUPERIOR: GitHub version more stable');
} else {
console.log('❌ COMPATIBILITY ISSUES: Both need work');
}
console.log(`\n📝 PRODUCTION RECOMMENDATION:`);
if (perfectPercent >= 70) {
console.log('✅ Either version acceptable - OAuth has more features');
} else if (oauthPercent >= 80) {
console.log('✅ Deploy OAuth version - more stable and feature-rich');
} else if (githubPercent >= 80) {
console.log('⚠️ Consider GitHub version if simplicity preferred');
} else {
console.log('⚠️ Both versions need stability improvements');
}
} catch (error) {
console.error('❌ Comparison failed:', error.message);
}
}
fixedGithubComparison().catch(console.error);
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server