Umbrella MCP Server
full-regression-test.cjs•21.2 kB
#!/usr/bin/env node
const axios = require('axios');
class FullRegressionTest {
constructor() {
// Test configurations
this.regularUser = {
apiKey: '57ade50e-c9a8-49f3-8ce7-28d44536a669',
authToken: 'eyJraWQiOiJoUFBoZTFRaWM4TklLU1dHcjQ4NEFHK3UwU2c5bCtmUHFWRWZUeCtcL0FcL1k9IiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiI1N2FkZTUwZS1jOWE4LTQ5ZjMtOGNlNy0yOGQ0NDUzNmE2NjkiLCJhdWQiOiI3aTgyY25wdDQ2OXJjZDkzZmlmMWdsaG5rbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJldmVudF9pZCI6ImIyZGFjZGNjLTEzN2YtNDJiYy1iYThlLTYyMzI0ZTJlNTk5OCIsInRva2VuX3VzZSI6ImlkIiwiYXV0aF90aW1lIjoxNzU2ODA0NTk1LCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAudXMtZWFzdC0xLmFtYXpvbmF3cy5jb21cL3VzLWVhc3QtMV9VdjZBck5kU0siLCJjb2duaXRvOnVzZXJuYW1lIjoiNTdhZGU1MGUtYzlhOC00OWYzLThjZTctMjhkNDQ1MzZhNjY5IiwiZXhwIjoxNzU2ODkwOTk1LCJpYXQiOjE3NTY4MDQ1OTUsImVtYWlsIjoiZGF2aWQrYWxsY2xvdWRAdW1icmVsbGFjb3N0LmNvbSJ9.laS8WHRdUF7nqnqZeIpdrkc-xAW_r8tHTJ9H-BvHGed1nKtpd1iaV9gbLg2oHG_-wMIvABWADqKdoAnBWhoEVoxJ_G0BznH4oaz11u65wZknZ-Qqn-XJFa2k7jH_i_TnrsVABQwTobU0xyv4Swnwi_svecVcbMYwDixciVjf6UC8VCt_8s7EAHd925H2uqsW2Lk-eYT5CYrlHo8Wm12cCk4S73aawm36gVO2FhWMxv_bbqETCiWpi_ikxR3Bu1ChWBYXylXsu5GuGgb1t8_Pi0FvKP060jZEFY3uxiz7h_i7QcwhPExGtelurAx3i-lzEZXdWGNaR89D7mOC2vov9Q'
};
this.bankHapoalimMSP = {
apiKey: '57ade50e-c9a8-49f3-8ce7-28d44536a669:16185:0', // Backend API (division 0)
frontendApiKey: '57ade50e-c9a8-49f3-8ce7-28d44536a669:16185:1', // Frontend API (division 1)
authToken: 'eyJraWQiOiJoUFBoZTFRaWM4TklLU1dHcjQ4NEFHK3UwU2c5bCtmUHFWRWZUeCtcL0FcL1k9IiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiI1N2FkZTUwZS1jOWE4LTQ5ZjMtOGNlNy0yOGQ0NDUzNmE2NjkiLCJhdWQiOiI3aTgyY25wdDQ2OXJjZDkzZmlmMWdsaG5rbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJldmVudF9pZCI6ImIyZGFjZGNjLTEzN2YtNDJiYy1iYThlLTYyMzI0ZTJlNTk5OCIsInRva2VuX3VzZSI6ImlkIiwiYXV0aF90aW1lIjoxNzU2ODA0NTk1LCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAudXMtZWFzdC0xLmFtYXpvbmF3cy5jb21cL3VzLWVhc3QtMV9VdjZBck5kU0siLCJjb2duaXRvOnVzZXJuYW1lIjoiNTdhZGU1MGUtYzlhOC00OWYzLThjZTctMjhkNDQ1MzZhNjY5IiwiZXhwIjoxNzU2ODkwOTk1LCJpYXQiOjE3NTY4MDQ1OTUsImVtYWlsIjoiZGF2aWQrYWxsY2xvdWRAdW1icmVsbGFjb3N0LmNvbSJ9.laS8WHRdUF7nqnqZeIpdrkc-xAW_r8tHTJ9H-BvHGed1nKtpd1iaV9gbLg2oHG_-wMIvABWADqKdoAnBWhoEVoxJ_G0BznH4oaz11u65wZknZ-Qqn-XJFa2k7jH_i_TnrsVABQwTobU0xyv4Swnwi_svecVcbMYwDixciVjf6UC8VCt_8s7EAHd925H2uqsW2Lk-eYT5CYrlHo8Wm12cCk4S73aawm36gVO2FhWMxv_bbqETCiWpi_ikxR3Bu1ChWBYXylXsu5GuGgb1t8_Pi0FvKP060jZEFY3uxiz7h_i7QcwhPExGtelurAx3i-lzEZXdWGNaR89D7mOC2vov9Q',
customerAccountKey: '16185'
};
this.baseURL = 'https://api-front.umbrellacost.io/api/v1';
this.backendURL = 'https://api.umbrellacost.io/api/v1';
this.results = {
passed: 0,
failed: 0,
tests: []
};
}
log(message) {
console.log(message);
}
logTest(testName, passed, message, expected = null, actual = null) {
const status = passed ? '✅ PASS' : '❌ FAIL';
this.log(`${status}: ${testName}`);
if (message) this.log(` ${message}`);
if (expected !== null && actual !== null) {
this.log(` Expected: ${expected}`);
this.log(` Actual: ${actual}`);
}
this.results.tests.push({
name: testName,
passed,
message,
expected,
actual
});
if (passed) {
this.results.passed++;
} else {
this.results.failed++;
}
}
async makeRequest(url, config, description = '') {
try {
const response = await axios({
url,
timeout: 30000,
...config
});
return { success: true, data: response.data, status: response.status };
} catch (error) {
return {
success: false,
error: error.message,
status: error.response?.status,
data: error.response?.data
};
}
}
// Test 1: Authentication Tests
async testAuthentication() {
this.log('\n🔐 TESTING AUTHENTICATION');
this.log('=' .repeat(50));
// Test regular user authentication
const regularAuthResult = await this.makeRequest(`${this.baseURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.regularUser.apiKey,
'authorization': this.regularUser.authToken,
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
this.logTest(
'Regular User Authentication',
regularAuthResult.success && regularAuthResult.status === 200,
regularAuthResult.success ? 'Regular user can authenticate' : `Auth failed: ${regularAuthResult.error}`
);
// Test MSP user authentication (backend API)
const mspBackendResult = await this.makeRequest(`${this.backendURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.apiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
this.logTest(
'MSP User Authentication (Backend API)',
mspBackendResult.success && mspBackendResult.status === 200,
mspBackendResult.success ? 'MSP user can authenticate to backend API' : `Auth failed: ${mspBackendResult.error}`
);
// Test MSP user authentication (frontend API)
const mspFrontendResult = await this.makeRequest(`${this.baseURL}/recommendationsNew/heatmap/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.frontendApiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
"filters": {
"status_filter": "potential_savings",
"is_open": true,
"user_status": { "done": false, "excluded": false },
"open_recs_creation_date": { "from": "2024-09-02", "to": "2025-09-02" },
"cost_mode": "UNBLENDED"
}
}
});
this.logTest(
'MSP User Authentication (Frontend API)',
mspFrontendResult.success && mspFrontendResult.status === 200,
mspFrontendResult.success ? 'MSP user can authenticate to frontend API' : `Auth failed: ${mspFrontendResult.error}`
);
}
// Test 2: Data Isolation
async testDataIsolation() {
this.log('\n🔒 TESTING DATA ISOLATION');
this.log('=' .repeat(50));
// Get regular user cost data
const regularCostResult = await this.makeRequest(`${this.baseURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.regularUser.apiKey,
'authorization': this.regularUser.authToken,
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
// Get MSP user cost data
const mspCostResult = await this.makeRequest(`${this.backendURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.apiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
const regularCost = regularCostResult.success ? regularCostResult.data.totalCost : 0;
const mspCost = mspCostResult.success ? mspCostResult.data.totalCost : 0;
this.logTest(
'Cost Data Isolation',
regularCostResult.success && mspCostResult.success && regularCost !== mspCost,
`Regular user and MSP user see different data sets`,
`Different costs`,
`Regular: $${regularCost}, MSP: $${mspCost}`
);
}
// Test 3: Critical Value Accuracy
async testCriticalAccuracy() {
this.log('\n🎯 TESTING CRITICAL VALUE ACCURACY');
this.log('=' .repeat(50));
// Test Bank Hapoalim January 2025 cost accuracy
const costResult = await this.makeRequest(`${this.backendURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.apiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
const expectedCost = 2839.38;
const actualCost = costResult.success ? parseFloat(costResult.data.totalCost) : 0;
const costAccurate = Math.abs(actualCost - expectedCost) < 0.01;
this.logTest(
'Bank Hapoalim January 2025 Cost Accuracy',
costResult.success && costAccurate,
costAccurate ? '100% accurate cost data' : 'Cost mismatch detected',
`$${expectedCost}`,
`$${actualCost}`
);
// Test Bank Hapoalim recommendations accuracy
const recsResult = await this.makeRequest(`${this.baseURL}/recommendationsNew/heatmap/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.frontendApiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
"filters": {
"status_filter": "potential_savings",
"is_open": true,
"user_status": { "done": false, "excluded": false },
"open_recs_creation_date": { "from": "2024-09-02", "to": "2025-09-02" },
"cost_mode": "UNBLENDED"
}
}
});
const expectedPotential = 5325.36;
const expectedActual = 0;
const actualPotential = recsResult.success ? parseFloat(recsResult.data.potentialAnnualSavings) : 0;
const actualActualSavings = recsResult.success ? parseFloat(recsResult.data.actualAnnualSavings) : 0;
const recsAccurate = Math.abs(actualPotential - expectedPotential) < 0.01 &&
Math.abs(actualActualSavings - expectedActual) < 0.01;
this.logTest(
'Bank Hapoalim Recommendations Accuracy',
recsResult.success && recsAccurate,
recsAccurate ? '100% accurate recommendations data' : 'Recommendations mismatch detected',
`$${expectedPotential} potential, $${expectedActual} actual`,
`$${actualPotential} potential, $${actualActualSavings} actual`
);
}
// Test 4: MSP-Specific Features
async testMSPFeatures() {
this.log('\n🏢 TESTING MSP-SPECIFIC FEATURES');
this.log('=' .repeat(50));
// Test isPpApplied header functionality
const withoutPpResult = await this.makeRequest(`${this.backendURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.apiKey,
'authorization': this.bankHapoalimMSP.authToken,
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
const withPpResult = await this.makeRequest(`${this.backendURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.apiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
const costWithoutPp = withoutPpResult.success ? parseFloat(withoutPpResult.data.totalCost) : 0;
const costWithPp = withPpResult.success ? parseFloat(withPpResult.data.totalCost) : 0;
this.logTest(
'isPpApplied Header Functionality',
withoutPpResult.success && withPpResult.success && costWithoutPp !== costWithPp,
'isPpApplied header changes the results',
'Different costs',
`Without PP: $${costWithoutPp}, With PP: $${costWithPp}`
);
// Test division ID handling (backend vs frontend)
const backendApiResult = await this.makeRequest(`${this.backendURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.apiKey, // Division 0
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
const frontendApiResult = await this.makeRequest(`${this.baseURL}/recommendationsNew/heatmap/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.frontendApiKey, // Division 1
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
"filters": {
"status_filter": "potential_savings",
"is_open": true,
"user_status": { "done": false, "excluded": false },
"open_recs_creation_date": { "from": "2024-09-02", "to": "2025-09-02" },
"cost_mode": "UNBLENDED"
}
}
});
this.logTest(
'Division ID Handling',
backendApiResult.success && frontendApiResult.success,
'Both backend (division 0) and frontend (division 1) APIs work',
'Both APIs accessible',
`Backend: ${backendApiResult.success ? 'OK' : 'FAIL'}, Frontend: ${frontendApiResult.success ? 'OK' : 'FAIL'}`
);
}
// Test 5: Core API Endpoints
async testCoreEndpoints() {
this.log('\n🌐 TESTING CORE API ENDPOINTS');
this.log('=' .repeat(50));
const endpoints = [
{
name: 'Cost Summary',
url: `${this.backendURL}/cost/summary`,
method: 'POST',
apiKey: this.bankHapoalimMSP.apiKey,
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
},
{
name: 'Recommendations Heatmap',
url: `${this.baseURL}/recommendationsNew/heatmap/summary`,
method: 'POST',
apiKey: this.bankHapoalimMSP.frontendApiKey,
data: {
"filters": {
"status_filter": "potential_savings",
"is_open": true,
"user_status": { "done": false, "excluded": false },
"open_recs_creation_date": { "from": "2024-09-02", "to": "2025-09-02" },
"cost_mode": "UNBLENDED"
}
}
},
{
name: 'Budget Summary',
url: `${this.backendURL}/budget/summary`,
method: 'GET',
apiKey: this.bankHapoalimMSP.apiKey,
data: null
}
];
for (const endpoint of endpoints) {
const result = await this.makeRequest(endpoint.url, {
method: endpoint.method,
headers: {
'apikey': endpoint.apiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: endpoint.data
});
this.logTest(
endpoint.name,
result.success && result.status === 200,
result.success ? 'Endpoint accessible' : `Failed: ${result.error}`,
'200 OK',
`${result.status || 'ERROR'}`
);
}
}
// Test 6: Error Handling
async testErrorHandling() {
this.log('\n🚨 TESTING ERROR HANDLING');
this.log('=' .repeat(50));
// Test invalid API key
const invalidKeyResult = await this.makeRequest(`${this.backendURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': 'invalid-key',
'authorization': this.bankHapoalimMSP.authToken,
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
this.logTest(
'Invalid API Key Handling',
!invalidKeyResult.success && (invalidKeyResult.status === 401 || invalidKeyResult.status === 403),
'Invalid API key properly rejected',
'401 or 403',
`${invalidKeyResult.status}`
);
// Test invalid auth token
const invalidTokenResult = await this.makeRequest(`${this.backendURL}/cost/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.apiKey,
'authorization': 'invalid-token',
'content-type': 'application/json'
},
data: {
period: 'monthly',
start: '2025-01-01',
end: '2025-01-31'
}
});
this.logTest(
'Invalid Auth Token Handling',
!invalidTokenResult.success && (invalidTokenResult.status === 401 || invalidTokenResult.status === 403),
'Invalid auth token properly rejected',
'401 or 403',
`${invalidTokenResult.status}`
);
}
// Test 7: Filter Functionality
async testFilterFunctionality() {
this.log('\n🔍 TESTING FILTER FUNCTIONALITY');
this.log('=' .repeat(50));
// Test recommendations with different filters
const openOnlyResult = await this.makeRequest(`${this.baseURL}/recommendationsNew/heatmap/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.frontendApiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {
"filters": {
"status_filter": "potential_savings",
"is_open": true,
"user_status": { "done": false, "excluded": false },
"open_recs_creation_date": { "from": "2024-09-02", "to": "2025-09-02" },
"cost_mode": "UNBLENDED"
}
}
});
const noFilterResult = await this.makeRequest(`${this.baseURL}/recommendationsNew/heatmap/summary`, {
method: 'POST',
headers: {
'apikey': this.bankHapoalimMSP.frontendApiKey,
'authorization': this.bankHapoalimMSP.authToken,
'commonparams': JSON.stringify({ isPpApplied: true }),
'content-type': 'application/json'
},
data: {}
});
const openPotential = openOnlyResult.success ? parseFloat(openOnlyResult.data.potentialAnnualSavings) : 0;
const noFilterPotential = noFilterResult.success ? parseFloat(noFilterResult.data.potentialAnnualSavings) : 0;
this.logTest(
'Open Recommendations Filter',
openOnlyResult.success && noFilterResult.success && openPotential !== noFilterPotential,
'Filter affects results as expected',
'Different values',
`Open only: $${openPotential}, No filter: $${noFilterPotential}`
);
// Verify exact UI match for open filter
this.logTest(
'UI Filter Match',
openOnlyResult.success && Math.abs(openPotential - 5325.36) < 0.01,
'Open filter matches UI exactly',
'$5,325.36',
`$${openPotential}`
);
}
async runAllTests() {
this.log('🧪 FULL REGRESSION TEST SUITE');
this.log('=' .repeat(60));
this.log(`🕒 Started: ${new Date().toISOString()}`);
this.log(`🌍 Testing against: ${this.baseURL} & ${this.backendURL}`);
try {
await this.testAuthentication();
await this.testDataIsolation();
await this.testCriticalAccuracy();
await this.testMSPFeatures();
await this.testCoreEndpoints();
await this.testErrorHandling();
await this.testFilterFunctionality();
} catch (error) {
this.log(`\n❌ CRITICAL ERROR: ${error.message}`);
this.results.failed++;
}
// Final Results
this.log('\n📊 FINAL REGRESSION TEST RESULTS');
this.log('=' .repeat(60));
this.log(`✅ Tests Passed: ${this.results.passed}`);
this.log(`❌ Tests Failed: ${this.results.failed}`);
this.log(`📈 Success Rate: ${((this.results.passed / (this.results.passed + this.results.failed)) * 100).toFixed(1)}%`);
if (this.results.failed === 0) {
this.log('\n🎉 ALL TESTS PASSED! MSP functionality is working correctly.');
this.log('✅ No regressions detected.');
this.log('✅ 100% accuracy maintained for critical values.');
this.log('✅ Data isolation working properly.');
this.log('✅ Authentication systems functioning.');
} else {
this.log('\n⚠️ REGRESSION DETECTED!');
this.log(`❌ ${this.results.failed} test(s) failed.`);
this.log('\nFailed Tests:');
this.results.tests.filter(t => !t.passed).forEach(test => {
this.log(` - ${test.name}: ${test.message}`);
if (test.expected && test.actual) {
this.log(` Expected: ${test.expected}, Got: ${test.actual}`);
}
});
}
this.log(`\n🕒 Completed: ${new Date().toISOString()}`);
// Return summary for programmatic use
return {
passed: this.results.passed,
failed: this.results.failed,
successRate: (this.results.passed / (this.results.passed + this.results.failed)) * 100,
tests: this.results.tests
};
}
}
// Run the regression test
(async () => {
const tester = new FullRegressionTest();
await tester.runAllTests();
})();
Latest Blog Posts
- Federated Learning with MCP: Building Privacy-Preserving Agents Across Distributed EdgesBy Om-Shree-0709 on .SecuremcpLearning
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/daviddraiumbrella/invoice-monitoring'
If you have feedback or need assistance with the MCP directory API, please join our Discord server