explain_failure
Analyze test failures to classify root causes and provide actionable recommendations for fixing issues in MCP server testing.
Instructions
Analyze test run results and explain failures with heuristic classification and actionable recommendations. Pass the structured result from run_spec.
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| runResult | Yes | The RunReport object from a run_spec call |
Implementation Reference
- src/tools/explainFailure.ts:54-112 (handler)The main handler function for the explain_failure tool, which analyzes test run results and categorizes failures.
export function explainFailure( input: ExplainFailureInput, ): ExplainFailureOutput { const { runResult } = input; const failedTests = runResult.tests.filter( (t) => t.status === "failed" || t.status === "error", ); if (failedTests.length === 0) { return { text: "All tests passed \u2014 no failures to explain.", structured: { summary: "All tests passed", causes: [], recommendations: [], }, }; } const counts = new Map<CauseType, number>(); for (const test of failedTests) { const cause = classifyCause(test); counts.set(cause, (counts.get(cause) ?? 0) + 1); } const causes: FailureCause[] = []; const recommendations: string[] = []; for (const [type, count] of counts) { causes.push({ type, count, description: CAUSE_DESCRIPTIONS[type], }); recommendations.push(RECOMMENDATIONS[type]); } const summary = causes .map((c) => `${c.count} ${c.type} failure${c.count > 1 ? "s" : ""}`) .join(", "); const textParts = [`${failedTests.length} failed test(s): ${summary}`, ""]; textParts.push("Causes:"); for (const c of causes) { textParts.push(` [${c.type}] ${c.count}x \u2014 ${c.description}`); } textParts.push(""); textParts.push("Recommendations:"); for (const r of recommendations) { textParts.push(` \u2022 ${r}`); } return { text: textParts.join("\n"), structured: { summary, causes, recommendations }, }; } - src/tools/explainFailure.ts:5-16 (schema)Input and output type definitions for the explain_failure tool.
export interface ExplainFailureInput { runResult: RunReport; } export interface ExplainFailureOutput { text: string; structured: { summary: string; causes: FailureCause[]; recommendations: string[]; }; } - src/tools/explainFailure.ts:114-161 (helper)Helper function to classify the type of test failure.
function classifyCause(test: TestResult): CauseType { const errorText = gatherErrorText(test).toLowerCase(); // auth if (AUTH_PATTERNS.some((p) => errorText.includes(p))) { return "auth"; } // placeholder const idAndError = `${test.testId} ${errorText}`.toLowerCase(); if ( idAndError.includes("todo") || idAndError.includes("placeholder") || idAndError.includes("todo_") ) { return "placeholder"; } // discovery if ( test.status === "error" && (errorText.includes("method not found") || errorText.includes("not supported") || errorText.includes("capability")) ) { return "discovery"; } // protocol if ( errorText.includes("timeout") || errorText.includes("connection") || errorText.includes("econnrefused") || errorText.includes("parse error") ) { return "protocol"; } // assertion if ( test.status === "failed" && test.assertionResults.some((a) => !a.passed) ) { return "assertion"; } return "unknown"; }