Umbrella MCP Server
- invoice-monitoring
- unitest
run-mcp-tests.sh•16.2 kB
#!/bin/bash
##############################################################################
# MCP Server Test Suite - Compares with MANUAL_ANSWERS.txt
# Tests both Saola and AllCloud accounts with exact matching (no tolerance)
##############################################################################
set -e # Exit on error
# Prompt for passwords
echo "Please enter passwords for test accounts:"
echo -n "Password for david+saola@umbrellacost.com: "
read -s SAOLA_PASSWORD
echo
echo -n "Password for david+allcloud@umbrellacost.com: "
read -s ALLCLOUD_PASSWORD
echo
echo
# Export for use in Node scripts
export SAOLA_PASSWORD
export ALLCLOUD_PASSWORD
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
BOLD='\033[1m'
# Test results tracking
TOTAL_TESTS=0
PASSED_TESTS=0
FAILED_TESTS=0
TEST_RESULTS=""
# Function to print colored output
print_color() {
local color=$1
shift
echo -e "${color}$@${NC}"
}
# Function to print test header
print_header() {
echo
print_color "$BLUE" "════════════════════════════════════════════════════════════════════════"
print_color "$BOLD$BLUE" "$1"
print_color "$BLUE" "════════════════════════════════════════════════════════════════════════"
}
# Function to run a test
run_test() {
local test_name=$1
local test_file=$2
TOTAL_TESTS=$((TOTAL_TESTS + 1))
print_color "$YELLOW" "\n🧪 Running: $test_name"
print_color "$YELLOW" "────────────────────────────────────────────────────"
if [ -f "$test_file" ]; then
if node "$test_file"; then
PASSED_TESTS=$((PASSED_TESTS + 1))
TEST_RESULTS="${TEST_RESULTS}\n ✅ $test_name"
return 0
else
FAILED_TESTS=$((FAILED_TESTS + 1))
TEST_RESULTS="${TEST_RESULTS}\n ❌ $test_name"
return 1
fi
else
print_color "$RED" " ❌ Test file not found: $test_file"
FAILED_TESTS=$((FAILED_TESTS + 1))
TEST_RESULTS="${TEST_RESULTS}\n ❌ $test_name (file not found)"
return 1
fi
}
# Main test execution
print_header "MCP SERVER REGRESSION TEST SUITE"
print_color "$BOLD" "\n📋 Testing against MANUAL_ANSWERS.txt baseline"
print_color "$BOLD" "⚠️ NO TOLERANCE - Exact matching required\n"
# Create test for Saola account
cat > test-saola-exact.cjs << 'EOFSAOLA'
#!/usr/bin/env node
const { spawn } = require('child_process');
const readline = require('readline');
const path = require('path');
const TEST_CASE = {
name: 'Saola March to July 2025',
credentials: {
username: 'david+saola@umbrellacost.com',
password: process.env.SAOLA_PASSWORD
},
requests: [
{
id: 1,
method: 'tools/call',
params: {
name: 'authenticate_user',
arguments: {
username: 'david+saola@umbrellacost.com',
password: process.env.SAOLA_PASSWORD
}
}
},
{
id: 2,
method: 'tools/call',
params: {
name: 'api___invoices_caui',
arguments: {
startDate: '2025-03-01',
endDate: '2025-07-31',
periodGranLevel: 'month',
groupBy: 'none',
costType: '["cost", "discount"]',
isUnblended: 'true'
}
}
}
],
expected: {
// Exact values from MANUAL_ANSWERS.txt - 4 decimal places
monthlyTotals: {
'2025-03': 104755.0743,
'2025-04': 111340.4244,
'2025-05': 149774.2380,
'2025-06': 165666.5701,
'2025-07': 183920.5781
},
accountId: '932213950603'
}
};
function sendRequest(proc, request) {
const jsonRequest = JSON.stringify({
jsonrpc: '2.0',
...request
});
proc.stdin.write(jsonRequest + '\n');
}
function parseResponse(line) {
try {
if (line.trim().startsWith('{') && line.includes('jsonrpc')) {
return JSON.parse(line);
}
} catch (e) {}
return null;
}
async function runTest() {
return new Promise((resolve) => {
console.log('Testing: Saola Account (March-July 2025)');
const result = {
passed: false,
errors: []
};
const proc = spawn('node', [path.join(__dirname, '../dist/index.js')], {
env: process.env
});
const rl = readline.createInterface({
input: proc.stdout,
crlfDelay: Infinity
});
rl.on('line', (line) => {
if (!result.passed && line.includes('"serverInfo"')) {
sendRequest(proc, TEST_CASE.requests[0]);
}
const response = parseResponse(line);
if (response && response.id) {
if (response.id === 1) {
if (response.result) {
sendRequest(proc, TEST_CASE.requests[1]);
} else if (response.error) {
console.log(` ❌ Authentication failed: ${response.error.message}`);
result.errors.push(`Authentication failed: ${response.error.message}`);
proc.kill();
resolve(result);
}
} else if (response.id === 2) {
if (response.result) {
let data = null;
if (response.result.content && response.result.content[0] && response.result.content[0].text) {
const content = response.result.content[0].text;
const jsonMatch = content.match(/```json\n([\s\S]*?)\n```/);
if (jsonMatch) {
data = JSON.parse(jsonMatch[1]);
}
}
if (data && Array.isArray(data)) {
let allPassed = true;
const actualByMonth = {};
data.forEach(row => {
actualByMonth[row.usage_date] = row.total_cost;
});
for (const [month, expectedCost] of Object.entries(TEST_CASE.expected.monthlyTotals)) {
const actualCost = actualByMonth[month];
// Round both to 4 decimal places
const roundedActual = actualCost ? Math.round(actualCost * 10000) / 10000 : undefined;
const roundedExpected = Math.round(expectedCost * 10000) / 10000;
if (actualCost === undefined) {
console.log(` ❌ ${month}: Missing data`);
result.errors.push(`${month}: Missing data`);
allPassed = false;
} else if (roundedActual !== roundedExpected) {
console.log(` ❌ ${month}: Expected ${expectedCost}, got ${actualCost}`);
result.errors.push(`${month}: Mismatch`);
allPassed = false;
} else {
console.log(` ✅ ${month}: ${roundedActual} (matches to 4 decimal places)`);
}
}
if (data[0] && data[0].account_id !== TEST_CASE.expected.accountId) {
console.log(` ❌ Account ID: Expected ${TEST_CASE.expected.accountId}, got ${data[0].account_id}`);
result.errors.push('Wrong account ID');
allPassed = false;
} else if (data[0]) {
console.log(` ✅ Account ID: ${data[0].account_id}`);
}
result.passed = allPassed;
} else {
result.errors.push('No data in response');
}
} else if (response.error) {
result.errors.push(`API call failed: ${response.error.message}`);
}
proc.kill();
resolve(result);
}
}
});
proc.stderr.on('data', () => {});
sendRequest(proc, {
jsonrpc: '2.0',
method: 'initialize',
params: {
protocolVersion: '2025-06-18',
capabilities: {},
clientInfo: { name: 'test-client', version: '1.0.0' }
},
id: 0
});
setTimeout(() => {
if (!result.passed && result.errors.length === 0) {
result.errors.push('Test timeout');
proc.kill();
resolve(result);
}
}, 30000);
});
}
runTest().then((result) => {
if (result.passed) {
console.log('\n✅ Saola test PASSED - Exact match!');
process.exit(0);
} else {
console.log('\n❌ Saola test FAILED');
process.exit(1);
}
});
EOFSAOLA
chmod +x test-saola-exact.cjs
# Create test for AllCloud account
cat > test-allcloud-exact.cjs << 'EOFALLCLOUD'
#!/usr/bin/env node
const { spawn } = require('child_process');
const readline = require('readline');
const path = require('path');
const TEST_CASE = {
name: 'AllCloud Bank Leumi Reseller-1 August 2025',
credentials: {
username: 'david+allcloud@umbrellacost.com',
password: process.env.ALLCLOUD_PASSWORD
},
requests: [
{
id: 1,
method: 'tools/call',
params: {
name: 'authenticate_user',
arguments: {
username: 'david+allcloud@umbrellacost.com',
password: process.env.ALLCLOUD_PASSWORD
}
}
},
{
id: 2,
method: 'tools/call',
params: {
name: 'api___invoices_caui',
arguments: {
userQuery: 'Bank Leumi Reseller-1 August no grouping',
startDate: '2025-08-01',
endDate: '2025-08-31',
periodGranLevel: 'month',
groupBy: 'none',
costType: '["cost", "discount"]',
isUnblended: 'true',
customer_account_key: '22676',
customer_division_id: '139'
}
}
}
],
expected: {
// Exact value from MANUAL_ANSWERS.txt - 4 decimal places
totalCost: 0.0027,
accountId: '696314371547'
}
};
function sendRequest(proc, request) {
const jsonRequest = JSON.stringify({
jsonrpc: '2.0',
...request
});
proc.stdin.write(jsonRequest + '\n');
}
function parseResponse(line) {
try {
if (line.trim().startsWith('{') && line.includes('jsonrpc')) {
return JSON.parse(line);
}
} catch (e) {}
return null;
}
async function runTest() {
return new Promise((resolve) => {
console.log('Testing: AllCloud Account (Bank Leumi Reseller-1, August 2025)');
const result = {
passed: false,
errors: []
};
const proc = spawn('node', [path.join(__dirname, '../dist/index.js')], {
env: process.env
});
const rl = readline.createInterface({
input: proc.stdout,
crlfDelay: Infinity
});
rl.on('line', (line) => {
if (!result.passed && line.includes('"serverInfo"')) {
sendRequest(proc, TEST_CASE.requests[0]);
}
const response = parseResponse(line);
if (response && response.id) {
if (response.id === 1) {
if (response.result) {
sendRequest(proc, TEST_CASE.requests[1]);
} else if (response.error) {
console.log(` ❌ Authentication failed: ${response.error.message}`);
result.errors.push(`Authentication failed: ${response.error.message}`);
proc.kill();
resolve(result);
}
} else if (response.id === 2) {
if (response.result) {
let data = null;
if (response.result.content && response.result.content[0] && response.result.content[0].text) {
const content = response.result.content[0].text;
const jsonMatch = content.match(/```json\n([\s\S]*?)\n```/);
if (jsonMatch) {
data = JSON.parse(jsonMatch[1]);
}
}
if (data && Array.isArray(data)) {
let allPassed = true;
// Check total cost (sum of all rows)
const actualTotal = data.reduce((sum, row) => sum + (row.total_cost || 0), 0);
// Round both to 4 decimal places
const roundedActual = Math.round(actualTotal * 10000) / 10000;
const roundedExpected = Math.round(TEST_CASE.expected.totalCost * 10000) / 10000;
if (roundedActual !== roundedExpected) {
console.log(` ❌ Total Cost: Expected ${TEST_CASE.expected.totalCost}, got ${actualTotal}`);
result.errors.push('Total cost mismatch');
allPassed = false;
} else {
console.log(` ✅ Total Cost: ${roundedActual} (matches to 4 decimal places)`);
}
// Check account ID
if (data[0] && data[0].account_id !== TEST_CASE.expected.accountId) {
console.log(` ❌ Account ID: Expected ${TEST_CASE.expected.accountId}, got ${data[0].account_id}`);
result.errors.push('Wrong account ID');
allPassed = false;
} else if (data[0]) {
console.log(` ✅ Account ID: ${data[0].account_id}`);
}
// Check customer key and division
console.log(' ℹ️ Customer Key: 22676, Division: 139');
result.passed = allPassed;
} else {
result.errors.push('No data in response');
}
} else if (response.error) {
result.errors.push(`API call failed: ${response.error.message}`);
}
proc.kill();
resolve(result);
}
}
});
proc.stderr.on('data', () => {});
sendRequest(proc, {
jsonrpc: '2.0',
method: 'initialize',
params: {
protocolVersion: '2025-06-18',
capabilities: {},
clientInfo: { name: 'test-client', version: '1.0.0' }
},
id: 0
});
setTimeout(() => {
if (!result.passed && result.errors.length === 0) {
result.errors.push('Test timeout');
proc.kill();
resolve(result);
}
}, 30000);
});
}
runTest().then((result) => {
if (result.passed) {
console.log('\n✅ AllCloud test PASSED - Exact match!');
process.exit(0);
} else {
console.log('\n❌ AllCloud test FAILED');
process.exit(1);
}
});
EOFALLCLOUD
chmod +x test-allcloud-exact.cjs
# Run tests
print_header "TEST 1: SAOLA ACCOUNT"
run_test "Saola Account (March-July 2025)" "./test-saola-exact.cjs" || true
print_header "TEST 2: ALLCLOUD ACCOUNT"
run_test "AllCloud Account (Bank Leumi Reseller-1)" "./test-allcloud-exact.cjs" || true
# Print summary
print_header "TEST SUMMARY"
echo -e "$TEST_RESULTS"
echo
print_color "$BOLD" "═══════════════════════════════════════════════════════════"
print_color "$BOLD" "Total Tests: $TOTAL_TESTS"
print_color "$GREEN" "Passed: $PASSED_TESTS"
print_color "$RED" "Failed: $FAILED_TESTS"
if [ $PASSED_TESTS -eq $TOTAL_TESTS ] && [ $TOTAL_TESTS -gt 0 ]; then
SUCCESS_RATE=100
else
if [ $TOTAL_TESTS -eq 0 ]; then
SUCCESS_RATE=0
else
SUCCESS_RATE=$((PASSED_TESTS * 100 / TOTAL_TESTS))
fi
fi
print_color "$BOLD" "Success Rate: ${SUCCESS_RATE}%"
print_color "$BOLD" "═══════════════════════════════════════════════════════════"
# Exit with appropriate code
if [ $FAILED_TESTS -gt 0 ]; then
echo
print_color "$RED$BOLD" "❌ REGRESSION DETECTED - Some tests failed!"
print_color "$RED" "The MCP server is not returning the exact values from MANUAL_ANSWERS.txt"
exit 1
else
echo
print_color "$GREEN$BOLD" "✅ ALL TESTS PASSED - MCP server matches MANUAL_ANSWERS.txt exactly!"
print_color "$GREEN" "No regression detected - the server is working correctly."
exit 0
fi
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server