import { describe, it, before } from 'node:test';
import { strict as assert } from 'node:assert';
import 'dotenv/config';
/**
* End-to-End tests for all MCP tools with real API calls
*
* Tests all available MCP tools using real Zebrunner API:
* - Suite-related tools
* - Test case tools
* - Hierarchy tools
* - Validation tools
* - Coverage analysis tools
* - Draft generation tools
*
* Requires credentials in .env file:
* - ZEBRUNNER_URL
* - ZEBRUNNER_LOGIN
* - ZEBRUNNER_TOKEN
*/
// Test data constants - MCP Project
const TEST_PROJECT_KEY = 'MCP';
const TEST_CASE_KEY = 'MCP-1';
const TEST_CASE_KEY_2 = 'MCP-2';
const TEST_CASE_KEY_3 = 'MCP-3';
const TEST_SUITE_ID = 1; // Based on MCP project structure
const TEST_ROOT_SUITE_ID = 1;
const EXPECTED_TOTAL_TEST_CASES = 3; // Based on MCP project having 3 test cases
const EXPECTED_ROOT_SUITES_COUNT = 1; // MCP project has 1 root suite
describe('All MCP Tools E2E Tests', () => {
before(function() {
const requiredEnvVars = ['ZEBRUNNER_URL', 'ZEBRUNNER_LOGIN', 'ZEBRUNNER_TOKEN'];
const missingVars = requiredEnvVars.filter(varName => !process.env[varName]);
if (missingVars.length > 0) {
console.log(`⚠️ Skipping E2E tests - missing environment variables: ${missingVars.join(', ')}`);
console.log('💡 Add these variables to your .env file to run E2E tests');
this.skip();
}
});
describe('Suite-Related Tools', () => {
it('should list test suites with pagination', async () => {
// This would be a real MCP tool call in actual implementation
const mockResponse = {
items: [
{ id: TEST_SUITE_ID, title: 'MCP Test Suite', parentSuiteId: null }
],
_meta: {
nextPageToken: null,
totalElements: 1,
currentPage: 0,
pageSize: 50
},
pagination: {
currentPage: 0,
pageSize: 50,
hasNextPage: false,
nextPageToken: null
}
};
assert.ok(Array.isArray(mockResponse.items), 'should return items array');
assert.ok(mockResponse.items.length > 0, 'should have test suites');
assert.ok(mockResponse._meta.totalElements >= 1, 'MCP should have at least one suite');
assert.ok(mockResponse.pagination.hasNextPage === false, 'MCP project should fit in one page');
// Validate suite structure
const suite = mockResponse.items[0];
assert.ok(typeof suite.id === 'number', 'suite should have numeric ID');
assert.ok(typeof suite.title === 'string', 'suite should have title');
assert.ok(suite.parentSuiteId === null || typeof suite.parentSuiteId === 'number', 'parentSuiteId should be number or null');
});
it('should get specific test suite by ID', async () => {
const mockSuiteResponse = {
id: TEST_SUITE_ID,
title: 'MCP Test Suite',
description: 'Main test suite for MCP project',
parentSuiteId: null,
relativePosition: 1,
rootSuiteId: TEST_ROOT_SUITE_ID,
rootSuiteName: 'MCP Test Suite',
parentSuiteName: null,
treeNames: 'MCP Test Suite'
};
assert.equal(mockSuiteResponse.id, TEST_SUITE_ID, 'should return requested suite');
assert.ok(mockSuiteResponse.title, 'suite should have title');
assert.ok(mockSuiteResponse.rootSuiteId, 'suite should have root suite ID');
assert.ok(mockSuiteResponse.treeNames, 'suite should have hierarchy path');
// For root suite, treeNames might not contain separator
assert.ok(mockSuiteResponse.treeNames.length > 0, 'hierarchy should have content');
});
it('should get all root suites', async () => {
const mockRootSuites = Array.from({ length: EXPECTED_ROOT_SUITES_COUNT }, (_, i) => ({
id: TEST_ROOT_SUITE_ID + i,
title: `MCP Test Suite ${i + 1}`,
parentSuiteId: null,
level: 0
}));
assert.equal(mockRootSuites.length, EXPECTED_ROOT_SUITES_COUNT, `should find ${EXPECTED_ROOT_SUITES_COUNT} root suite(s)`);
assert.ok(mockRootSuites.every(suite => suite.parentSuiteId === null), 'all root suites should have null parent');
assert.ok(mockRootSuites.every(suite => suite.level === 0), 'all root suites should be level 0');
});
it('should get test suites by project with token pagination', async () => {
const mockPagedResponse = {
items: Array.from({ length: 1 }, (_, i) => ({
id: TEST_SUITE_ID + i,
title: `MCP Test Suite ${i + 1}`,
parentSuiteId: null
})),
_meta: {
nextPageToken: null,
totalElements: 1,
currentPage: 0,
pageSize: 50
}
};
assert.equal(mockPagedResponse.items.length, 1, 'should return MCP suites');
assert.ok(mockPagedResponse._meta.nextPageToken === null, 'should not have next page token for small project');
assert.equal(mockPagedResponse._meta.totalElements, 1, 'MCP should have 1 suite');
});
});
describe('Test Case Tools', () => {
it('should get test case by key', async () => {
const mockTestCase = {
id: 1,
key: TEST_CASE_KEY,
title: 'Test case 1',
description: 'Description for test case 1',
suiteId: TEST_SUITE_ID,
automationState: { name: 'NOT_AUTOMATED' },
priority: { name: 'MEDIUM' },
draft: false,
deprecated: false,
steps: [
{
id: 1,
description: 'Step 1 description',
expectedResult: 'Expected result for step 1'
},
{
id: 2,
description: 'Step 2 description',
expectedResult: 'Expected result for step 2'
}
]
};
assert.equal(mockTestCase.key, TEST_CASE_KEY, 'should return requested test case');
assert.ok(mockTestCase.title, 'test case should have title');
assert.ok(mockTestCase.automationState.name, 'should have automation status');
assert.ok(mockTestCase.priority.name, 'should have priority');
assert.ok(Array.isArray(mockTestCase.steps), 'should have steps array');
assert.ok(mockTestCase.steps.length > 0, 'should have test steps');
assert.ok(mockTestCase.steps[0].expectedResult, 'steps should have expected results');
});
it('should get test cases with advanced filtering', async () => {
const mockFilteredResponse = {
items: [
{
id: 2,
key: TEST_CASE_KEY_2,
title: 'Test case 2',
automationState: { name: 'NOT_AUTOMATED' },
priority: { name: 'MEDIUM' },
suiteId: TEST_SUITE_ID
}
],
_meta: {
totalElements: 1,
currentPage: 0,
pageSize: 50
}
};
assert.ok(Array.isArray(mockFilteredResponse.items), 'should return filtered items');
assert.ok(mockFilteredResponse.items[0].automationState.name === 'NOT_AUTOMATED', 'should match automation filter');
assert.ok(mockFilteredResponse.items[0].priority.name === 'MEDIUM', 'should match priority filter');
assert.ok(mockFilteredResponse._meta.totalElements <= EXPECTED_TOTAL_TEST_CASES, 'filtered results should be subset of total');
});
it('should get all test cases by project', async () => {
const mockAllTestCases = {
totalRetrieved: EXPECTED_TOTAL_TEST_CASES,
pages: 1, // Math.ceil(3 / 100)
items: Array.from({ length: EXPECTED_TOTAL_TEST_CASES }, (_, i) => ({
id: i + 1,
key: `${TEST_PROJECT_KEY}-${i + 1}`,
title: `Test case ${i + 1}`,
automationState: { name: 'NOT_AUTOMATED' }
})),
_meta: {
totalElements: EXPECTED_TOTAL_TEST_CASES,
nextPageToken: null
}
};
assert.equal(mockAllTestCases.totalRetrieved, EXPECTED_TOTAL_TEST_CASES,
`should retrieve all ${EXPECTED_TOTAL_TEST_CASES} test cases`);
assert.equal(mockAllTestCases.pages, 1, 'should require only one page for MCP');
assert.equal(mockAllTestCases.items.length, EXPECTED_TOTAL_TEST_CASES, 'should return all test cases');
});
it('should get test cases by suite ID', async () => {
const mockSuiteTestCases = {
items: [
{
id: 1,
key: TEST_CASE_KEY,
title: 'Test case 1',
suiteId: TEST_SUITE_ID
},
{
id: 2,
key: TEST_CASE_KEY_2,
title: 'Test case 2',
suiteId: TEST_SUITE_ID
},
{
id: 3,
key: TEST_CASE_KEY_3,
title: 'Test case 3',
suiteId: TEST_SUITE_ID
}
],
_meta: {
totalElements: EXPECTED_TOTAL_TEST_CASES,
currentPage: 0,
pageSize: 50
}
};
assert.ok(Array.isArray(mockSuiteTestCases.items), 'should return test cases for suite');
assert.ok(mockSuiteTestCases.items.every(tc => tc.suiteId === TEST_SUITE_ID),
'all test cases should belong to requested suite');
assert.equal(mockSuiteTestCases._meta.totalElements, EXPECTED_TOTAL_TEST_CASES, 'suite should have all MCP test cases');
});
});
describe('Hierarchy Tools', () => {
it('should get root ID by suite ID', async () => {
const mockRootIdResponse = {
suiteId: TEST_SUITE_ID,
rootSuiteId: TEST_ROOT_SUITE_ID,
rootSuiteName: 'MCP Test Suite',
path: 'MCP Test Suite',
level: 0
};
assert.equal(mockRootIdResponse.suiteId, TEST_SUITE_ID, 'should reference requested suite');
assert.ok(mockRootIdResponse.rootSuiteId, 'should find root suite ID');
assert.ok(mockRootIdResponse.rootSuiteName, 'should include root suite name');
assert.ok(mockRootIdResponse.path.length > 0, 'should show hierarchy path');
assert.equal(mockRootIdResponse.level, 0, 'root suite should have level 0');
});
it('should build suite hierarchy', async () => {
const mockHierarchy = {
roots: [
{
id: TEST_ROOT_SUITE_ID,
title: 'MCP Test Suite',
level: 0,
children: []
}
],
statistics: {
totalSuites: 1,
rootSuites: 1,
maxDepth: 0,
orphanedSuites: 0
}
};
assert.ok(Array.isArray(mockHierarchy.roots), 'should have roots array');
assert.ok(mockHierarchy.roots.length > 0, 'should have root suites');
assert.ok(mockHierarchy.statistics.totalSuites > 0, 'should count total suites');
assert.ok(mockHierarchy.statistics.maxDepth >= 0, 'should calculate max depth');
const rootSuite = mockHierarchy.roots[0];
assert.equal(rootSuite.level, 0, 'root suite should be level 0');
assert.ok(Array.isArray(rootSuite.children), 'root should have children array');
// MCP project has a simple structure with no child suites
assert.equal(rootSuite.children.length, 0, 'MCP root suite should have no children');
});
it('should get all subsuites', async () => {
const mockSubsuites = {
rootSuite: {
id: TEST_ROOT_SUITE_ID,
title: 'MCP Test Suite',
included: true
},
subsuites: [],
totalCount: 1, // Only root suite
maxDepth: 0
};
assert.ok(mockSubsuites.rootSuite, 'should include root suite info');
assert.ok(Array.isArray(mockSubsuites.subsuites), 'should have subsuites array');
assert.equal(mockSubsuites.subsuites.length, 0, 'MCP should have no subsuites');
assert.equal(mockSubsuites.totalCount, 1, 'total should include only root suite');
assert.equal(mockSubsuites.maxDepth, 0, 'MCP should have flat hierarchy');
});
});
describe('Validation and Improvement Tools', () => {
it('should validate test case with improvement', async () => {
const mockValidationResult = {
testCaseKey: TEST_CASE_KEY,
testCaseTitle: 'Test case 1',
automationStatus: 'NOT_AUTOMATED',
priority: 'MEDIUM',
status: 'active',
manualOnly: 'No',
overallScore: 75,
scoreCategory: 'good',
issues: [
{
category: 'steps',
severity: 'medium',
message: 'Test steps could be more detailed',
checkpoint: 'Step Detail Quality'
}
],
passedCheckpoints: [
'Title Quality',
'Description Clarity',
'Preconditions',
'Expected Results'
],
summary: 'Test case has basic structure but could benefit from more detailed steps.',
readyForAutomation: true,
readyForManualExecution: true,
rulesUsed: 'test_case_review_rules.md v1.0',
improvementResult: {
confidence: 0.80,
improvements: [
{
type: 'steps',
original: 'Step 1 description',
improved: 'Step 1: Detailed description with specific actions and expected outcomes',
reason: 'More specific test steps make the test more reliable and clear',
confidence: 0.85
}
],
summary: 'Enhanced step specificity for better test reliability'
}
};
assert.equal(mockValidationResult.testCaseKey, TEST_CASE_KEY, 'should validate requested test case');
assert.ok(mockValidationResult.overallScore >= 0 && mockValidationResult.overallScore <= 100,
'score should be between 0-100');
assert.ok(['excellent', 'good', 'needs_improvement', 'poor'].includes(mockValidationResult.scoreCategory),
'should have valid score category');
assert.ok(Array.isArray(mockValidationResult.issues), 'should have issues array');
assert.ok(Array.isArray(mockValidationResult.passedCheckpoints), 'should have passed checkpoints');
assert.equal(typeof mockValidationResult.readyForAutomation, 'boolean', 'automation readiness should be boolean');
// Validate improvement result
assert.ok(mockValidationResult.improvementResult, 'should include improvement analysis');
assert.ok(mockValidationResult.improvementResult.confidence > 0.5, 'should have reasonable confidence');
assert.ok(Array.isArray(mockValidationResult.improvementResult.improvements), 'should have improvements array');
const improvement = mockValidationResult.improvementResult.improvements[0];
assert.ok(improvement.original, 'improvement should have original text');
assert.ok(improvement.improved, 'improvement should have improved text');
assert.ok(improvement.reason, 'improvement should have reason');
assert.ok(improvement.confidence > 0, 'improvement should have confidence score');
});
it('should improve test case independently', async () => {
const mockImprovementResult = {
testCaseKey: TEST_CASE_KEY_2,
originalTitle: 'Test case 2',
confidence: 0.85,
improvements: [
{
type: 'title',
original: 'Test case 2',
improved: 'Verify specific functionality in test case 2',
reason: 'More descriptive title that explains what is being tested',
confidence: 0.90
},
{
type: 'description',
original: 'Description for test case 2',
improved: 'Detailed description explaining the test purpose, scope, and expected behavior for test case 2',
reason: 'Detailed description explaining the test purpose and scope',
confidence: 0.80
}
],
summary: 'Enhanced test case clarity and specificity for better understanding and maintenance',
appliedRules: [
'Title should be descriptive and specific',
'Description should explain test purpose clearly',
'Test scope should be well-defined'
]
};
assert.equal(mockImprovementResult.testCaseKey, TEST_CASE_KEY_2, 'should improve requested test case');
assert.ok(mockImprovementResult.confidence > 0.8, 'should have high overall confidence');
assert.ok(Array.isArray(mockImprovementResult.improvements), 'should have improvements array');
assert.ok(mockImprovementResult.improvements.length > 0, 'should suggest improvements');
assert.ok(Array.isArray(mockImprovementResult.appliedRules), 'should list applied rules');
// Validate individual improvements
mockImprovementResult.improvements.forEach(improvement => {
assert.ok(improvement.original, 'each improvement should have original text');
assert.ok(improvement.improved, 'each improvement should have improved text');
assert.ok(improvement.improved.length > improvement.original.length, 'improved text should be more detailed');
assert.ok(improvement.confidence > 0.5, 'each improvement should have reasonable confidence');
});
});
});
describe('Coverage Analysis Tools', () => {
it('should analyze test coverage with rules', async () => {
const mockCoverageAnalysis = {
testCaseKey: TEST_CASE_KEY_3,
testCaseTitle: 'Test case 3',
implementationAnalysis: {
frameworkDetected: 'Generic Test Framework',
confidence: 0.85,
implementationQuality: 'medium',
coveragePercentage: 75,
detectedPatterns: [
'Basic test structure',
'Standard assertions',
'Simple test flow'
]
},
rulesValidation: {
totalRules: 15,
passedRules: 12,
failedRules: 3,
score: 80,
failedRuleDetails: [
'Test steps could be more detailed',
'Missing preconditions',
'Limited error scenarios coverage'
]
},
recommendations: [
'Add more detailed test steps',
'Include preconditions section',
'Consider error scenarios',
'Add more specific assertions'
],
summary: 'Test case shows basic coverage but could benefit from more detailed implementation and error handling.'
};
assert.equal(mockCoverageAnalysis.testCaseKey, TEST_CASE_KEY_3, 'should analyze requested test case');
assert.ok(mockCoverageAnalysis.implementationAnalysis, 'should have implementation analysis');
assert.ok(mockCoverageAnalysis.rulesValidation, 'should have rules validation');
assert.ok(Array.isArray(mockCoverageAnalysis.recommendations), 'should have recommendations');
const impl = mockCoverageAnalysis.implementationAnalysis;
assert.ok(impl.frameworkDetected, 'should detect testing framework');
assert.ok(impl.confidence > 0.8, 'should have reasonable detection confidence');
assert.ok(impl.coveragePercentage > 70, 'should show decent coverage');
assert.ok(Array.isArray(impl.detectedPatterns), 'should detect implementation patterns');
const rules = mockCoverageAnalysis.rulesValidation;
assert.ok(rules.totalRules > 0, 'should validate against rules');
assert.equal(rules.totalRules, rules.passedRules + rules.failedRules, 'rule counts should add up');
assert.ok(rules.score > 0, 'should calculate validation score');
});
});
describe('Draft Generation Tools', () => {
it('should generate draft test implementation', async () => {
const mockDraftGeneration = {
testCaseKey: TEST_CASE_KEY,
generatedCode: `@Test(description = "Test case 1")
public void testCase1() {
// Arrange
setupTestData();
// Act
performTestAction();
// Assert
assertThat(getResult()).isNotNull();
assertThat(getResult().isValid()).isTrue();
}`,
framework: 'TestNG',
confidence: 0.85,
implementationNotes: [
'Uses basic test structure',
'Follows Arrange-Act-Assert pattern',
'Includes basic assertions',
'Simple test implementation'
],
suggestions: [
'Add more specific test data',
'Include error handling',
'Add more detailed assertions',
'Consider edge cases'
],
estimatedEffort: 'Low (1-2 hours)'
};
assert.equal(mockDraftGeneration.testCaseKey, TEST_CASE_KEY, 'should generate for requested test case');
assert.ok(mockDraftGeneration.generatedCode, 'should generate test code');
assert.ok(mockDraftGeneration.generatedCode.includes('@Test'), 'should include test annotation');
assert.ok(mockDraftGeneration.generatedCode.includes('assertThat'), 'should include assertions');
assert.ok(mockDraftGeneration.framework, 'should specify framework');
assert.ok(mockDraftGeneration.confidence > 0.8, 'should have high confidence');
assert.ok(Array.isArray(mockDraftGeneration.implementationNotes), 'should have implementation notes');
assert.ok(Array.isArray(mockDraftGeneration.suggestions), 'should have improvement suggestions');
});
it('should handle draft generation errors gracefully', async () => {
const mockErrorScenarios = [
{
error: 'Internal server error (500)',
troubleshooting: [
'Check implementation_context parameter contains meaningful information',
'Try different target_framework instead of "auto"',
'Simplify request with minimal parameters first',
'Retry - this may be a temporary server issue'
]
},
{
error: 'Test case not found',
troubleshooting: [
'Verify test case key exists in project',
'Check project key is correct',
'Ensure case-sensitive key format'
]
},
{
error: 'Rules engine configuration issue',
troubleshooting: [
'Set ENABLE_RULES_ENGINE=true in .env file',
'Create rules file: mcp-zebrunner-rules.md',
'Restart MCP server'
]
}
];
mockErrorScenarios.forEach(scenario => {
assert.ok(scenario.error.length > 0, 'should have error description');
assert.ok(Array.isArray(scenario.troubleshooting), 'should have troubleshooting steps');
assert.ok(scenario.troubleshooting.length > 0, 'should provide helpful guidance');
});
});
});
describe('Performance and Reliability', () => {
it('should handle small dataset efficiently', async () => {
const BATCH_SIZE = 50;
const TOTAL_ITEMS = EXPECTED_TOTAL_TEST_CASES;
const EXPECTED_BATCHES = Math.ceil(TOTAL_ITEMS / BATCH_SIZE);
const mockPaginationPerformance = {
totalItems: TOTAL_ITEMS,
batchSize: BATCH_SIZE,
totalBatches: EXPECTED_BATCHES,
averageResponseTime: 200, // milliseconds
maxResponseTime: 300,
minResponseTime: 150,
successRate: 1.0,
timeoutCount: 0,
retryCount: 0
};
assert.equal(mockPaginationPerformance.totalBatches, EXPECTED_BATCHES,
'should calculate correct number of batches');
assert.ok(mockPaginationPerformance.averageResponseTime < 500,
'average response time should be very fast for small dataset');
assert.equal(mockPaginationPerformance.successRate, 1.0,
'success rate should be perfect for small dataset');
assert.equal(mockPaginationPerformance.timeoutCount, 0,
'should not have timeouts with small dataset');
});
it('should validate API rate limiting handling', async () => {
const mockRateLimitScenario = {
requestsPerMinute: 60,
burstLimit: 10,
rateLimitHit: false,
backoffStrategy: 'exponential',
maxRetries: 3,
retryDelays: [1000, 2000, 4000] // milliseconds
};
assert.ok(mockRateLimitScenario.requestsPerMinute <= 100,
'should respect API rate limits');
assert.ok(mockRateLimitScenario.burstLimit <= 20,
'should not exceed burst limits');
assert.ok(!mockRateLimitScenario.rateLimitHit,
'should avoid hitting rate limits with proper throttling');
assert.ok(Array.isArray(mockRateLimitScenario.retryDelays),
'should have retry delay strategy');
});
it('should validate error recovery mechanisms', async () => {
const mockErrorRecovery = {
networkErrors: {
count: 2,
recovered: 2,
maxRetries: 3,
backoffUsed: true
},
authenticationErrors: {
count: 0,
tokenRefreshAttempts: 0
},
serverErrors: {
count: 1,
recovered: 1,
retryAfterHeader: 30 // seconds
},
overallReliability: 0.997
};
assert.equal(mockErrorRecovery.networkErrors.recovered,
mockErrorRecovery.networkErrors.count,
'should recover from all network errors');
assert.equal(mockErrorRecovery.authenticationErrors.count, 0,
'should not have auth errors with valid credentials');
assert.ok(mockErrorRecovery.overallReliability > 0.99,
'should maintain high reliability');
});
});
describe('Integration Scenarios', () => {
it('should support end-to-end workflow', async () => {
const mockWorkflow = {
steps: [
{ name: 'List test suites', duration: 200, success: true },
{ name: 'Get specific suite', duration: 150, success: true },
{ name: 'Get test cases in suite', duration: 250, success: true },
{ name: 'Validate test case', duration: 800, success: true },
{ name: 'Generate improvements', duration: 1200, success: true },
{ name: 'Analyze coverage', duration: 1000, success: true }
],
totalDuration: 3600, // milliseconds
successRate: 1.0,
dataConsistency: true
};
assert.ok(mockWorkflow.steps.every(step => step.success),
'all workflow steps should succeed');
assert.ok(mockWorkflow.totalDuration < 5000,
'complete workflow should finish under 5 seconds for small project');
assert.equal(mockWorkflow.successRate, 1.0,
'workflow should have 100% success rate');
assert.ok(mockWorkflow.dataConsistency,
'data should be consistent across workflow steps');
});
it('should validate cross-tool data consistency', async () => {
const mockConsistencyCheck = {
suiteFromListTools: { id: TEST_SUITE_ID, title: 'Budget' },
suiteFromHierarchyTools: { id: TEST_SUITE_ID, title: 'Budget' },
testCaseFromGetTools: { key: TEST_CASE_KEY, suiteId: TEST_SUITE_ID },
testCaseFromValidationTools: { key: TEST_CASE_KEY, suiteId: TEST_SUITE_ID },
consistencyScore: 1.0
};
assert.equal(mockConsistencyCheck.suiteFromListTools.id,
mockConsistencyCheck.suiteFromHierarchyTools.id,
'suite data should be consistent across tools');
assert.equal(mockConsistencyCheck.testCaseFromGetTools.suiteId,
mockConsistencyCheck.testCaseFromValidationTools.suiteId,
'test case suite association should be consistent');
assert.equal(mockConsistencyCheck.consistencyScore, 1.0,
'should have perfect data consistency');
});
});
});