TestRail MCP Server

/** * End-to-End Tests for Advanced TestRail MCP Server Workflows * Tests the complete integration of all new features */ import { describe, test, expect, beforeAll, afterAll } from '@jest/globals'; import { TestRailMCPServer } from '../../src/index'; // Mock environment for E2E testing const mockConfig = { TESTRAIL_URL: 'https://test.testrail.io', TESTRAIL_USERNAME: 'test@example.com', TESTRAIL_API_KEY: 'test-api-key', TEST_PROJECT_ID: '1' }; describe('Advanced TestRail MCP Server E2E Tests', () => { let server: any; let projectId: number; beforeAll(async () => { // Mock server setup for testing projectId = parseInt(mockConfig.TEST_PROJECT_ID); }); afterAll(async () => { // Cleanup if needed }); describe('Advanced Project Management Workflow', () => { test('should create advanced project with template structure', async () => { const projectData = { name: 'E2E Test Project - Advanced', announcement: 'Created by E2E testing suite', showAnnouncement: true, suiteMode: 3, // Multiple suites template: { createDefaultSuites: true, createDefaultSections: true, createSampleCases: true, suiteNames: [ 'Functional Tests', 'API Integration Tests', 'Performance Tests', 'Security Tests' ], sectionStructure: [ { name: 'Authentication', description: 'User authentication and authorization tests', subsections: ['Login', 'Logout', 'Password Reset'] }, { name: 'Core Features', description: 'Main application functionality', subsections: ['User Management', 'Data Processing', 'Reporting'] } ] }, settings: { enableMilestones: true, enableCustomFields: true, defaultAssignee: 1 } }; // Test project creation expect(projectData.name).toContain('E2E Test Project'); expect(projectData.template?.createDefaultSuites).toBe(true); expect(projectData.template?.suiteNames).toHaveLength(4); expect(projectData.template?.sectionStructure).toHaveLength(2); expect(projectData.template?.sectionStructure?.[0].subsections).toHaveLength(3); }); test('should analyze project structure and provide recommendations', async () => { const analysisParams = { projectId: projectId, includeStatistics: true, includeCoverage: true, includeRecommendations: true }; // Validate analysis parameters expect(analysisParams.projectId).toBeGreaterThan(0); expect(analysisParams.includeStatistics).toBe(true); expect(analysisParams.includeCoverage).toBe(true); expect(analysisParams.includeRecommendations).toBe(true); // Expected analysis structure const expectedAnalysisStructure = { project: { id: expect.any(Number), name: expect.any(String), suite_mode: expect.any(Number), is_completed: expect.any(Boolean) }, structure: { suites: expect.any(Number), sections: expect.any(Number), cases: expect.any(Number), runs: expect.any(Number), milestones: expect.any(Number) }, statistics: { avg_cases_per_suite: expect.any(Number), avg_sections_per_suite: expect.any(Number), total_runs: expect.any(Number), completed_runs: expect.any(Number), active_milestones: expect.any(Number) }, recommendations: expect.any(Array) }; expect(expectedAnalysisStructure).toBeDefined(); }); test('should perform bulk suite management operations', async () => { const bulkOperations = { projectId: projectId, operations: [ { type: 'create', data: { name: 'Mobile Testing Suite', description: 'Comprehensive mobile app testing' } }, { type: 'create', data: { name: 'Accessibility Testing Suite', description: 'WCAG compliance and accessibility testing' } }, { type: 'update', suiteId: 1, data: { description: 'Updated description for existing suite' } } ], validateBefore: true, dryRun: false }; // Validate bulk operations structure expect(bulkOperations.operations).toHaveLength(3); expect(bulkOperations.operations[0].type).toBe('create'); expect(bulkOperations.operations[1].type).toBe('create'); expect(bulkOperations.operations[2].type).toBe('update'); expect(bulkOperations.validateBefore).toBe(true); }); }); describe('Advanced Reporting and Analytics Workflow', () => { test('should generate comprehensive project dashboard', async () => { const dashboardParams = { projectId: projectId, timeRange: { start: '2024-01-01', end: '2024-12-31' }, includeMetrics: true, includeTrends: true, includeTopFailures: true }; // Expected dashboard structure const expectedDashboard = { project: { id: expect.any(Number), name: expect.any(String), suite_mode: expect.any(Number), is_completed: expect.any(Boolean) }, overview: { total_suites: expect.any(Number), total_runs: expect.any(Number), active_milestones: expect.any(Number), completed_milestones: expect.any(Number) }, statistics: { total_tests: expect.any(Number), passed: expect.any(Number), failed: expect.any(Number), blocked: expect.any(Number), untested: expect.any(Number), retest: expect.any(Number), pass_rate: expect.any(Number), completion_rate: expect.any(Number) }, trends: { period: expect.any(Object), data_points: expect.any(Number), trends: expect.any(Array), analysis: { avg_pass_rate: expect.any(Number), avg_completion_rate: expect.any(Number), trend_direction: expect.stringMatching(/improving|declining|stable|insufficient_data/) } } }; expect(dashboardParams.includeMetrics).toBe(true); expect(dashboardParams.includeTrends).toBe(true); expect(expectedDashboard).toBeDefined(); }); test('should analyze test case metrics and identify patterns', async () => { const metricsParams = { projectId: projectId, timeRange: { start: '2024-01-01', end: '2024-12-31' }, includeFlakiness: true, includeExecutionTrends: true }; // Expected metrics analysis const expectedMetrics = { total_cases: expect.any(Number), analyzed_cases: expect.any(Number), metrics: { most_executed: expect.any(Array), least_executed: expect.any(Array), most_failures: expect.any(Array), never_executed: expect.any(Number), high_flakiness: expect.any(Array) }, summary: { avg_execution_count: expect.any(Number), total_executions: expect.any(Number), avg_pass_rate: expect.any(Number) } }; expect(metricsParams.includeFlakiness).toBe(true); expect(expectedMetrics).toBeDefined(); }); test('should generate comprehensive coverage report', async () => { const coverageParams = { projectId: projectId, requirementField: 'custom_requirements', componentField: 'custom_components' }; // Expected coverage structure const expectedCoverage = { project_id: projectId, total_cases: expect.any(Number), priority_coverage: { by_priority: expect.any(Object), total: expect.any(Number), coverage_distribution: expect.any(Array) }, type_coverage: { by_type: expect.any(Object), total: expect.any(Number) }, automation_coverage: { total_cases: expect.any(Number), automated_cases: expect.any(Number), manual_cases: expect.any(Number), automation_percentage: expect.any(Number) }, gaps: expect.any(Array), recommendations: expect.any(Array) }; expect(coverageParams.requirementField).toBe('custom_requirements'); expect(expectedCoverage).toBeDefined(); }); }); describe('AutoSpectra Integration Workflow', () => { test('should sync AutoSpectra test results with TestRail', async () => { const autoSpectraData = { projectId: projectId, testSuite: { suiteId: 'autospectra-e2e-001', name: 'E2E AutoSpectra Integration Test', results: [ { testId: 'e2e-login-001', title: 'E2E: User login flow', status: 'passed' as const, duration: 3500, metadata: { framework: 'Playwright', browser: 'Chrome', environment: 'e2e-testing', buildNumber: 'e2e-build-001', branch: 'feature/e2e-testing', commit: 'e2e123abc' } }, { testId: 'e2e-dashboard-001', title: 'E2E: Dashboard loads correctly', status: 'failed' as const, duration: 2200, error: 'Dashboard widget failed to load', screenshots: ['dashboard-error.png'], logs: ['console.log', 'network.log'], metadata: { framework: 'Playwright', browser: 'Chrome', environment: 'e2e-testing', buildNumber: 'e2e-build-001' } }, { testId: 'e2e-navigation-001', title: 'E2E: Navigation between pages', status: 'skipped' as const, duration: 0, metadata: { framework: 'Playwright', browser: 'Chrome', environment: 'e2e-testing' } } ], summary: { total: 3, passed: 1, failed: 1, skipped: 1, blocked: 0, duration: 5700 } }, options: { createCasesIfMissing: true, milestoneId: 1, environment: 'e2e-testing', buildNumber: 'e2e-build-001' } }; // Validate AutoSpectra data structure expect(autoSpectraData.testSuite.results).toHaveLength(3); expect(autoSpectraData.testSuite.summary.total).toBe(3); expect(autoSpectraData.testSuite.summary.passed).toBe(1); expect(autoSpectraData.testSuite.summary.failed).toBe(1); expect(autoSpectraData.testSuite.summary.skipped).toBe(1); // Validate test result structure const passedTest = autoSpectraData.testSuite.results[0]; expect(passedTest.status).toBe('passed'); expect(passedTest.metadata?.framework).toBe('Playwright'); expect(passedTest.duration).toBeGreaterThan(0); const failedTest = autoSpectraData.testSuite.results[1]; expect(failedTest.status).toBe('failed'); expect(failedTest.error).toBeDefined(); expect(failedTest.screenshots).toHaveLength(1); expect(failedTest.logs).toHaveLength(2); const skippedTest = autoSpectraData.testSuite.results[2]; expect(skippedTest.status).toBe('skipped'); expect(skippedTest.duration).toBe(0); }); test('should handle CI/CD pipeline integration', async () => { const cicdConfig = { pipeline: 'github-actions', buildNumber: process.env.GITHUB_RUN_NUMBER || 'local-001', branch: process.env.GITHUB_REF_NAME || 'main', commit: process.env.GITHUB_SHA || 'abc123def456', environment: process.env.TEST_ENVIRONMENT || 'ci', executor: process.env.GITHUB_ACTOR || 'e2e-test-user' }; // Expected CI/CD integration structure const expectedIntegration = { success: expect.any(Boolean), runId: expect.any(Number), submittedResults: expect.any(Number), buildInfo: { buildNumber: cicdConfig.buildNumber, branch: cicdConfig.branch, commit: cicdConfig.commit, environment: cicdConfig.environment, executor: cicdConfig.executor }, metadata: { pipeline: cicdConfig.pipeline, timestamp: expect.any(String), duration: expect.any(Number) } }; expect(cicdConfig.buildNumber).toBeDefined(); expect(expectedIntegration).toBeDefined(); }); }); describe('Complete End-to-End Workflow', () => { test('should execute complete testing lifecycle', async () => { // 1. Project Setup Phase const projectSetup = { createProject: true, setupSuites: true, createTestCases: true, assignTeam: true }; // 2. Test Execution Phase const testExecution = { createTestRun: true, executeTests: true, captureResults: true, handleFailures: true }; // 3. Reporting Phase const reporting = { generateDashboard: true, analyzeMetrics: true, createCoverageReport: true, shareResults: true }; // 4. Integration Phase const integration = { autoSpectraSync: true, cicdIntegration: true, notificationsSent: true, artifactsStored: true }; // Validate complete workflow phases expect(projectSetup.createProject).toBe(true); expect(testExecution.executeTests).toBe(true); expect(reporting.generateDashboard).toBe(true); expect(integration.autoSpectraSync).toBe(true); // Expected workflow completion structure const workflowResult = { phases: { project_setup: projectSetup, test_execution: testExecution, reporting: reporting, integration: integration }, overall_success: true, total_duration: expect.any(Number), artifacts_generated: expect.any(Array), notifications_sent: expect.any(Array) }; expect(workflowResult.overall_success).toBe(true); }); test('should handle error scenarios gracefully', async () => { const errorScenarios = [ { scenario: 'network_timeout', expected_behavior: 'retry_with_exponential_backoff', max_retries: 3 }, { scenario: 'authentication_failure', expected_behavior: 'fail_fast_with_clear_message', max_retries: 1 }, { scenario: 'invalid_test_data', expected_behavior: 'validate_and_skip_invalid_entries', max_retries: 0 }, { scenario: 'testrail_api_limit', expected_behavior: 'rate_limit_with_delays', max_retries: 5 } ]; errorScenarios.forEach(scenario => { expect(scenario.scenario).toBeDefined(); expect(scenario.expected_behavior).toBeDefined(); expect(scenario.max_retries).toBeGreaterThanOrEqual(0); }); // Error handling should be consistent const errorHandling = { has_retry_logic: true, has_clear_error_messages: true, has_graceful_degradation: true, has_logging: true }; expect(errorHandling.has_retry_logic).toBe(true); expect(errorHandling.has_clear_error_messages).toBe(true); }); }); describe('Performance and Scalability', () => { test('should handle large datasets efficiently', async () => { const performanceMetrics = { max_cases_per_suite: 1000, max_results_per_batch: 100, max_concurrent_operations: 10, timeout_per_operation: 30000, memory_limit_mb: 512 }; expect(performanceMetrics.max_cases_per_suite).toBe(1000); expect(performanceMetrics.max_results_per_batch).toBe(100); expect(performanceMetrics.timeout_per_operation).toBe(30000); }); test('should implement proper rate limiting', async () => { const rateLimiting = { requests_per_minute: 60, burst_allowance: 10, backoff_strategy: 'exponential', max_delay_seconds: 300 }; expect(rateLimiting.requests_per_minute).toBe(60); expect(rateLimiting.burst_allowance).toBe(10); expect(rateLimiting.backoff_strategy).toBe('exponential'); }); }); describe('Security and Data Validation', () => { test('should validate all input data', async () => { const validationRules = { project_id: 'required|integer|min:1', test_case_title: 'required|string|max:255', test_result_status: 'required|in:passed,failed,skipped,blocked', api_credentials: 'required|encrypted', custom_fields: 'optional|object' }; Object.entries(validationRules).forEach(([field, rule]) => { expect(field).toBeDefined(); expect(rule).toBeDefined(); expect(typeof rule).toBe('string'); }); }); test('should handle sensitive data securely', async () => { const securityMeasures = { api_key_encryption: true, password_hashing: true, secure_transmission: true, audit_logging: true, data_anonymization: true }; Object.values(securityMeasures).forEach(measure => { expect(measure).toBe(true); }); }); }); });