Umbrella MCP Server
COMPLETE-VERBOSE-GOODANSWERS-COMPARISON.cjs•17.6 kB
#!/usr/bin/env node
// COMPLETE VERBOSE GOODANSWERS.TXT COMPARISON
// Answer all 13 questions from goodanswers.txt with full responses from both versions
const axios = require('axios');
const fs = require('fs');
async function completeVerboseGoodAnswersComparison() {
console.log('🔬 COMPLETE VERBOSE GOODANSWERS.TXT COMPARISON');
console.log('═'.repeat(120));
console.log('Testing all 13 questions with FULL responses from both versions');
console.log('GitHub Latest: Bearer token + /mcp endpoint (JSON responses)');
console.log('OAuth Version: Session + /sse endpoint (formatted responses)');
console.log('═'.repeat(120));
const results = [];
try {
// Authenticate with GitHub version
console.log('🔐 Authenticating with GitHub version...');
const githubAuthResponse = await axios.post('http://localhost:3000/auth', {
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
}, {
headers: { 'Content-Type': 'application/json' }
});
const githubToken = githubAuthResponse.data.bearerToken;
console.log(`✅ GitHub authenticated with Bearer token`);
// Authenticate with OAuth version
console.log('🔐 Authenticating with OAuth version...');
const oauthSessionResponse = await axios.post('http://localhost:8080/api/session/create');
const oauthSessionId = oauthSessionResponse.data.sessionId;
await axios.post('http://localhost:8080/oauth/callback', {
sessionId: oauthSessionId,
username: 'david+saola@umbrellacost.com',
password: 'Dsamsung1!'
});
console.log(`✅ OAuth authenticated: ${oauthSessionId}`);
// All 13 questions from goodanswers.txt
const questions = [
{
id: 'Q1',
query: 'show me the list of MSP customers',
githubTool: 'get_costs',
oauthTool: 'api___msp_customers',
oauthArgs: {}
},
{
id: 'Q2',
query: 'what is my total cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
oauthArgs: { periodGranLevel: 'month' }
},
{
id: 'Q3',
query: 'what is my total AWS cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
oauthArgs: { cloud_context: 'aws', periodGranLevel: 'month' }
},
{
id: 'Q4',
query: 'what is my total GCP cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
oauthArgs: { cloud_context: 'gcp', periodGranLevel: 'month' }
},
{
id: 'Q5',
query: 'what is my total Azure cost?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
oauthArgs: { cloud_context: 'azure', periodGranLevel: 'month' }
},
{
id: 'Q6',
query: 'show me the total cost per month',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
oauthArgs: { periodGranLevel: 'month' }
},
{
id: 'Q7',
query: 'show me the total AWS amortized cost per month for the last 8 months',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
oauthArgs: {
cloud_context: 'aws',
isAmortized: true,
periodGranLevel: 'month',
startDate: '2025-01-01',
endDate: '2025-08-31'
}
},
{
id: 'Q8',
query: 'show me the total cost for ALL Azure accounts',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
oauthArgs: { cloud_context: 'azure', periodGranLevel: 'month' }
},
{
id: 'Q9',
query: 'show me all available accounts',
githubTool: 'get_costs',
oauthTool: 'api___user_management_accounts',
oauthArgs: {}
},
{
id: 'Q10',
query: 'what do you recommend for saving AWS costs?',
githubTool: 'get_costs',
oauthTool: 'api___recommendationsNew_heatmap_summary',
oauthArgs: {}
},
{
id: 'Q11',
query: 'what are the potential savings per category?',
githubTool: 'get_costs',
oauthTool: 'api___recommendationsNew_heatmap_summary',
oauthArgs: {}
},
{
id: 'Q12',
query: 'what is the cost impact of the anomalies in the last 10 days for AWS?',
githubTool: 'get_costs',
oauthTool: 'api___anomaly_detection',
oauthArgs: {
cloud_context: 'aws',
startDate: '2025-08-28',
endDate: '2025-09-07'
}
},
{
id: 'Q13',
query: 'what is the last 30 days (per day) amortized cost for Cloudwatch service?',
githubTool: 'get_costs',
oauthTool: 'api___invoices_caui',
oauthArgs: {
cloud_context: 'aws',
service: 'cloudwatch',
isAmortized: true,
periodGranLevel: 'day',
startDate: '2025-08-08',
endDate: '2025-09-07'
}
}
];
console.log('\n═'.repeat(120));
console.log('DETAILED QUESTION-BY-QUESTION COMPARISON');
console.log('═'.repeat(120));
for (const q of questions) {
console.log(`\n${'█'.repeat(120)}`);
console.log(`${q.id}: ${q.query}`);
console.log(`${'█'.repeat(120)}`);
// Test GitHub version
let githubResponse = '';
let githubData = null;
let githubError = false;
try {
const githubRequest = {
jsonrpc: '2.0',
id: Date.now(),
method: 'tools/call',
params: {
name: q.githubTool,
arguments: { query: q.query }
}
};
const response = await axios.post('http://localhost:3000/mcp', githubRequest, {
headers: {
'Authorization': `Bearer ${githubToken}`,
'Content-Type': 'application/json'
}
});
if (response.data?.result?.content?.[0]?.text) {
githubResponse = response.data.result.content[0].text;
try {
githubData = JSON.parse(githubResponse);
} catch (e) {
// Response might not be JSON
}
} else {
githubResponse = JSON.stringify(response.data, null, 2);
}
} catch (error) {
githubResponse = `❌ GitHub Error: ${error.response?.data?.error || error.message}`;
githubError = true;
}
// Test OAuth version
let oauthResponse = '';
let oauthError = false;
try {
const oauthRequest = {
jsonrpc: '2.0',
method: 'tools/call',
params: {
name: q.oauthTool,
arguments: {
sessionId: oauthSessionId,
userQuery: q.query,
...q.oauthArgs
}
},
id: parseInt(q.id.substring(1))
};
const response = await axios.post('http://localhost:3001/sse', oauthRequest);
oauthResponse = response.data.result.content[0].text;
} catch (error) {
oauthResponse = `❌ OAuth Error: ${error.response?.data?.error || error.message}`;
oauthError = true;
}
// Display full responses
console.log('\n🔵 GITHUB LATEST VERSION RESPONSE:');
console.log('─'.repeat(80));
console.log('📡 Endpoint: Bearer token + POST /mcp');
console.log('🛠️ Tool Used:', q.githubTool);
console.log('📊 Response Format: JSON Array/Object');
console.log('─'.repeat(80));
if (githubError) {
console.log('❌ ERROR OCCURRED:');
console.log(githubResponse);
} else if (githubData && Array.isArray(githubData)) {
console.log(`✅ SUCCESS: JSON Array with ${githubData.length} items`);
console.log('📋 Sample data structure:');
if (githubData.length > 0) {
const sample = githubData[0];
Object.keys(sample).forEach(key => {
const value = sample[key];
const type = typeof value;
console.log(` • ${key}: ${type} = ${type === 'number' ? value.toFixed(2) : String(value).substring(0, 50)}`);
});
}
// Calculate totals if cost data
if (githubData.some(item => item.total_cost !== undefined)) {
const totalCost = githubData.reduce((sum, item) => sum + (item.total_cost || 0), 0);
console.log(`💰 CALCULATED TOTAL: $${totalCost.toLocaleString()}`);
}
console.log('\n🔍 FULL RAW DATA:');
console.log(JSON.stringify(githubData, null, 2));
} else {
console.log('📄 Raw response:');
console.log(githubResponse);
}
console.log('\n🟢 OAUTH VERSION RESPONSE:');
console.log('─'.repeat(80));
console.log('📡 Endpoint: OAuth session + POST /sse');
console.log('🛠️ Tool Used:', q.oauthTool);
console.log('📊 Response Format: Formatted Text');
console.log('─'.repeat(80));
if (oauthError) {
console.log('❌ ERROR OCCURRED:');
console.log(oauthResponse);
} else {
console.log('✅ SUCCESS: Formatted response received');
// Extract key information
const costMatch = oauthResponse.match(/\*\*💰[^$]*\$([0-9,]+\.?\d*)\*\*/);
const monthlyMatches = oauthResponse.match(/• \w+ \d{4}: \$[0-9,]+\.\d{2}/g);
if (costMatch) {
console.log(`💰 EXTRACTED TOTAL: $${costMatch[1]}`);
}
if (monthlyMatches) {
console.log(`📅 MONTHLY ITEMS: ${monthlyMatches.length} months`);
monthlyMatches.slice(0, 3).forEach(item => console.log(` ${item}`));
if (monthlyMatches.length > 3) console.log(` ... and ${monthlyMatches.length - 3} more`);
}
console.log('\n🔍 FULL FORMATTED RESPONSE:');
console.log(oauthResponse);
}
// Comparison analysis
console.log('\n⚖️ DETAILED COMPARISON ANALYSIS:');
console.log('─'.repeat(80));
const githubWorking = !githubError && githubResponse && githubResponse !== '';
const oauthWorking = !oauthError && oauthResponse && oauthResponse !== '';
// Extract costs for comparison
let githubCost = 'N/A';
let oauthCost = 'N/A';
if (githubData && Array.isArray(githubData)) {
const totalCost = githubData.reduce((sum, item) => sum + (item.total_cost || 0), 0);
if (totalCost > 0) {
githubCost = totalCost.toFixed(2);
}
}
const oauthCostMatch = oauthResponse.match(/\*\*💰[^$]*\$([0-9,]+\.?\d*)\*\*/);
if (oauthCostMatch) {
oauthCost = oauthCostMatch[1].replace(/,/g, '');
}
console.log(`🎯 STATUS COMPARISON:`);
console.log(` GitHub Working: ${githubWorking ? '✅ YES' : '❌ NO'}`);
console.log(` OAuth Working: ${oauthWorking ? '✅ YES' : '❌ NO'}`);
console.log(` Both Working: ${githubWorking && oauthWorking ? '✅ YES' : '❌ NO'}`);
console.log(`\n💰 COST COMPARISON:`);
console.log(` GitHub Total: $${githubCost}`);
console.log(` OAuth Total: $${oauthCost}`);
if (githubCost !== 'N/A' && oauthCost !== 'N/A') {
const githubNum = parseFloat(githubCost);
const oauthNum = parseFloat(oauthCost);
const diff = Math.abs(githubNum - oauthNum);
const diffPercent = ((diff / Math.max(githubNum, oauthNum)) * 100).toFixed(2);
console.log(` Difference: $${diff.toFixed(2)} (${diffPercent}%)`);
console.log(` Match: ${diff < 0.01 ? '✅ EXACT' : diff < 100 ? '⚠️ CLOSE' : '❌ DIFFERENT'}`);
} else {
console.log(` Match: ❌ CANNOT COMPARE`);
}
console.log(`\n📊 RESPONSE FORMAT ANALYSIS:`);
console.log(` GitHub: ${githubData ? 'Structured JSON data' : 'Raw text/error'}`);
console.log(` OAuth: ${oauthWorking ? 'Human-readable formatted text' : 'Error message'}`);
console.log(` Compatibility: ${githubWorking && oauthWorking ? 'Both provide data but different formats' : 'Format mismatch or errors'}`);
// Store results
results.push({
id: q.id,
query: q.query,
github: {
working: githubWorking,
cost: githubCost,
response: githubResponse,
data: githubData,
error: githubError
},
oauth: {
working: oauthWorking,
cost: oauthCost,
response: oauthResponse,
error: oauthError
}
});
// Add delay between questions
await new Promise(resolve => setTimeout(resolve, 1000));
}
// Final comprehensive summary
console.log('\n' + '█'.repeat(120));
console.log('📊 COMPREHENSIVE FINAL SUMMARY');
console.log('█'.repeat(120));
const githubWorking = results.filter(r => r.github.working).length;
const oauthWorking = results.filter(r => r.oauth.working).length;
const bothWorking = results.filter(r => r.github.working && r.oauth.working).length;
const total = results.length;
console.log(`\n🎯 OVERALL SUCCESS RATES:`);
console.log(` GitHub Latest: ${githubWorking}/${total} (${(githubWorking/total*100).toFixed(1)}%)`);
console.log(` OAuth Version: ${oauthWorking}/${total} (${(oauthWorking/total*100).toFixed(1)}%)`);
console.log(` Both Working: ${bothWorking}/${total} (${(bothWorking/total*100).toFixed(1)}%)`);
console.log(`\n📋 QUESTION-BY-QUESTION SUMMARY:`);
console.log('│ ID │ Question │ GitHub │ OAuth │ Both │ Cost Match │');
console.log('├────┼──────────┼────────┼───────┼──────┼────────────┤');
results.forEach(r => {
const githubStatus = r.github.working ? '✅' : '❌';
const oauthStatus = r.oauth.working ? '✅' : '❌';
const bothStatus = r.github.working && r.oauth.working ? '✅' : '❌';
let costMatch = '❌';
if (r.github.cost !== 'N/A' && r.oauth.cost !== 'N/A') {
const diff = Math.abs(parseFloat(r.github.cost) - parseFloat(r.oauth.cost));
costMatch = diff < 0.01 ? '✅' : diff < 100 ? '⚠️' : '❌';
}
console.log(`│ ${r.id.padEnd(2)} │ ${r.query.substring(0, 8).padEnd(8)} │ ${githubStatus.padEnd(6)} │ ${oauthStatus.padEnd(5)} │ ${bothStatus.padEnd(4)} │ ${costMatch.padEnd(10)} │`);
});
console.log(`\n🔍 DETAILED FINDINGS:`);
console.log(`\n📈 GitHub Version Analysis:`);
console.log(` • Uses single versatile tool (get_costs) for all queries`);
console.log(` • Returns structured JSON data arrays`);
console.log(` • Bearer token authentication is simple and reliable`);
console.log(` • Success rate: ${(githubWorking/total*100).toFixed(1)}%`);
const githubFailures = results.filter(r => !r.github.working);
if (githubFailures.length > 0) {
console.log(` • Failed questions: ${githubFailures.map(r => r.id).join(', ')}`);
console.log(` • Common failure patterns:`);
githubFailures.forEach(r => {
console.log(` - ${r.id}: ${r.github.response.substring(0, 60)}...`);
});
}
console.log(`\n📈 OAuth Version Analysis:`);
console.log(` • Uses specialized tools for different query types`);
console.log(` • Returns human-readable formatted responses`);
console.log(` • OAuth session management is more complex but feature-rich`);
console.log(` • Success rate: ${(oauthWorking/total*100).toFixed(1)}%`);
const oauthFailures = results.filter(r => !r.oauth.working);
if (oauthFailures.length > 0) {
console.log(` • Failed questions: ${oauthFailures.map(r => r.id).join(', ')}`);
console.log(` • Common failure patterns:`);
oauthFailures.forEach(r => {
console.log(` - ${r.id}: ${r.oauth.response.substring(0, 60)}...`);
});
}
console.log(`\n⚖️ HONEST ASSESSMENT:`);
if (githubWorking >= oauthWorking && githubWorking >= total * 0.8) {
console.log(`✅ GITHUB VERSION SUPERIOR: Higher success rate and simpler architecture`);
} else if (oauthWorking >= githubWorking && oauthWorking >= total * 0.8) {
console.log(`✅ OAUTH VERSION SUPERIOR: Higher success rate and richer functionality`);
} else if (bothWorking >= total * 0.7) {
console.log(`✅ BOTH VERSIONS VIABLE: Different strengths, choose based on needs`);
} else {
console.log(`⚠️ BOTH NEED IMPROVEMENT: Neither version meets 80% success threshold`);
}
console.log(`\n📝 PRODUCTION RECOMMENDATION:`);
console.log(` • For simple integration: GitHub version (JSON responses, single tool)`);
console.log(` • For rich features: OAuth version (formatted responses, specialized tools)`);
console.log(` • For reliability: Choose version with higher success rate`);
console.log(` • For development: Both versions provide working solutions`);
} catch (error) {
console.error('❌ Comparison failed:', error.message);
console.error('Stack:', error.stack);
}
}
completeVerboseGoodAnswersComparison().catch(console.error);
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server