validate_test_case

Instructions

Implementation Reference

• src/handlers/tools.ts:371-445 (handler)

  Main MCP tool handler: fetches Zebrunner test case by key, initializes TestCaseValidator (with optional custom rules files), runs validation, optionally generates improvements via TestCaseImprover, formats output (JSON/markdown/string), returns MCP content response.

  async validateTestCase(input: z.infer<typeof ValidateTestCaseInputSchema>) {
    const { projectKey, caseKey, rulesFilePath, checkpointsFilePath, format, improveIfPossible } = input;
    
    try {
      // Get the test case data first
      const testCase = await this.client.getTestCaseByKey(projectKey, caseKey);
      
      // Initialize validator with dynamic rules
      let validator: TestCaseValidator;
      if (rulesFilePath && checkpointsFilePath) {
        // Use custom rules from files - validate paths first
        try {
          const resolvedRulesPath = validateFilePath(rulesFilePath, process.cwd());
          const resolvedCheckpointsPath = validateFilePath(checkpointsFilePath, process.cwd());
          validator = await TestCaseValidator.fromMarkdownFiles(resolvedRulesPath, resolvedCheckpointsPath);
        } catch (error) {
          throw new Error(`Invalid file path provided: ${error instanceof Error ? error.message : error}`);
        }
      } else {
        // Use default rules, but try to load from standard files if they exist
        const defaultRulesPath = path.resolve(process.cwd(), 'test_case_review_rules.md');
        const defaultCheckpointsPath = path.resolve(process.cwd(), 'test_case_analysis_checkpoints.md');
        
        try {
          validator = await TestCaseValidator.fromMarkdownFiles(defaultRulesPath, defaultCheckpointsPath);
        } catch (error) {
          // Fall back to default rules if files don't exist
          validator = new TestCaseValidator();
        }
      }
      
      // Validate the test case
      const validationResult = await validator.validateTestCase(testCase);
      
      // Attempt improvement if requested
      let improvementResult = null;
      if (improveIfPossible) {
        const improver = new TestCaseImprover();
        improvementResult = await improver.improveTestCase(testCase, validationResult);
      }
      
      // Format the result based on requested format
      let formattedResult: string;
      
      if (format === 'markdown') {
        formattedResult = this.formatValidationResultAsMarkdown(validationResult, improvementResult);
      } else if (format === 'string') {
        formattedResult = this.formatValidationResultAsString(validationResult, improvementResult);
      } else {
        const result = improvementResult 
          ? { validation: validationResult, improvement: improvementResult }
          : validationResult;
        formattedResult = JSON.stringify(result, null, 2);
      }
      
      return {
        content: [
          {
            type: "text" as const,
            text: formattedResult
          }
        ]
      };
    } catch (error: any) {
      const errorMsg = sanitizeErrorMessage(error, 'Error validating test case', 'validateTestCase');
      return {
        content: [
          {
            type: "text" as const,
            text: errorMsg
          }
        ]
      };
    }
  }

• src/types/api.ts:142-149 (schema)

  Zod input schema for validate_test_case tool defining required projectKey/caseKey and optional custom rules paths, output format, improvement flag.

  export const ValidateTestCaseInputSchema = z.object({
    projectKey: z.string().min(1),
    caseKey: z.string().min(1),
    rulesFilePath: z.string().optional(),
    checkpointsFilePath: z.string().optional(),
    format: z.enum(['dto', 'json', 'string', 'markdown']).default('json'),
    improveIfPossible: z.boolean().default(true)
  });

• src/utils/test-case-validator.ts:89-159 (helper)

  Core helper implementing test case validation logic: iterates over dynamic rule set, executes specific validators (title, steps, preconditions, etc.), calculates score, categorizes readiness for automation/manual, generates issues/summary.

  async validateTestCase(testCase: ZebrunnerTestCase): Promise<ValidationResult> {
    const issues: ValidationIssue[] = [];
    const passedCheckpoints: string[] = [];
  
    // Run validation based on enabled rules
    for (const rule of this.ruleSet.rules.filter(r => r.enabled)) {
      const validationFunction = this.validationFunctions.get(rule.checkFunction);
      if (validationFunction) {
        try {
          const result = validationFunction(testCase, rule);
          if (result.passed) {
            passedCheckpoints.push(rule.name);
          } else {
            issues.push({
              category: rule.category,
              severity: rule.severity,
              checkpoint: rule.name,
              description: result.message || rule.description,
              suggestion: result.suggestion || rule.suggestion,
              ruleId: rule.id
            });
          }
        } catch (error) {
          console.warn(`Error executing validation rule ${rule.id}: ${error}`);
        }
      }
    }
  
    // Calculate score
    const totalCheckpoints = issues.length + passedCheckpoints.length;
    const score = totalCheckpoints > 0 ? Math.round((passedCheckpoints.length / totalCheckpoints) * 100) : 0;
    
    const scoreCategory = this.getScoreCategory(score);
    const readyForAutomation = score >= this.ruleSet.scoreThresholds.good && !this.hasAutomationBlockers(issues);
    const readyForManualExecution = score >= this.ruleSet.scoreThresholds.needs_improvement;
  
    // Extract automation status, priority, and status
    const automationStatus = testCase.automationState?.name || 'Unknown';
    const priority = testCase.priority?.name || undefined;
    
    // Construct status from available boolean fields
    let status: string | undefined;
    if (testCase.draft) {
      status = 'Draft';
    } else if (testCase.deprecated) {
      status = 'Deprecated';
    } else {
      // If neither draft nor deprecated, it's likely active
      status = 'Active';
    }
    
    // Extract Manual Only from custom fields
    const manualOnly = testCase.customField?.manualOnly || undefined;
  
    return {
      testCaseKey: testCase.key || 'Unknown',
      testCaseTitle: testCase.title || 'Untitled',
      automationStatus,
      priority,
      status,
      manualOnly,
      overallScore: score,
      scoreCategory,
      issues,
      passedCheckpoints,
      summary: this.generateSummary(score, issues, readyForAutomation, readyForManualExecution, automationStatus),
      readyForAutomation,
      readyForManualExecution,
      rulesUsed: `${this.ruleSet.name} v${this.ruleSet.version}`
    };
  }