verify

Instructions

Implementation Reference

• src/server.ts:533-575 (handler)

  The 'verify' MCP tool handler — registered as an MCP server tool. It loads a cassette, runs the server live, records responses, and compares them against the cassette using compareResponses().

  server.tool(
    "verify",
    "Use this after updating a server to confirm nothing broke. Connects to the live server, sends the same requests from a recorded cassette, and compares responses. Reports exactly what changed — added tools, removed parameters, different response shapes.",
    {
      cassette: z.string().describe("Path to a cassette JSON file."),
      command: z.string().describe("The command to launch the MCP server."),
      args: z.array(z.string()).optional().describe("Additional arguments for the command."),
    },
    async ({ cassette: cassettePath, command, args }) => {
      const startMs = Date.now();
      try {
        validateCommand(command);
        const cassettesDir = defaultCassettesDirectory();
        validatePath(cassettePath, cassettesDir);
        const cassette = await loadCassette(cassettePath);
        const target = {
          targetId: command,
          adapter: "local-process" as const,
          command,
          args: args ?? [],
          timeoutMs: 15_000,
        };
  
        const { cassetteEntries } = await runTargetRecording(target, { invokeTools: true });
        if (!cassetteEntries) {
          return { content: [{ type: "text" as const, text: "Failed to record live session for comparison." }], isError: true };
        }
  
        const result = compareResponses(cassette, cassetteEntries);
        const lines: string[] = [`Verify: ${result.passed} passed, ${result.failed} changed, ${result.missing} missing\n`];
        for (const entry of result.entries) {
          const icon = entry.status === "pass" ? "✓" : entry.status === "fail" ? "✗" : "?";
          lines.push(`  ${icon} ${entry.method}${entry.diff ? ` — ${entry.diff}` : ""}`);
        }
        logRequest("verify", startMs);
        return { content: [{ type: "text" as const, text: lines.join("\n") }] };
      } catch (error) {
        const msg = errorMessage(error);
        logRequest("verify", startMs, true);
        return { content: [{ type: "text" as const, text: `Error verifying: ${msg}` }], isError: true };
      }
    },
  );

• src/verify.ts:5-20 (schema)

  Type definitions for verify results — VerifyEntryResult (per-entry pass/fail/missing) and VerifyResult (aggregated stats).

  export interface VerifyEntryResult {
    method: string;
    status: "pass" | "fail" | "missing";
    expected?: unknown;
    actual?: unknown;
    diff?: string;
  }
  
  export interface VerifyResult {
    targetId: string;
    totalEntries: number;
    passed: number;
    failed: number;
    missing: number;
    entries: VerifyEntryResult[];
  }

• src/verify.ts:28-97 (helper)

  compareResponses() — the core comparison logic that compares cassette response entries against live response entries and produces a VerifyResult.

  export function compareResponses(
    cassette: Cassette,
    liveEntries: CassetteEntry[],
  ): VerifyResult {
    const cassetteResponses = cassette.entries.filter((e) => e.direction === "response");
    const liveResponses = liveEntries.filter((e) => e.direction === "response");
  
    const entries: VerifyEntryResult[] = [];
  
    for (let i = 0; i < cassetteResponses.length; i++) {
      const expected = cassetteResponses[i]!;
      const actual = liveResponses[i];
  
      if (!actual) {
        entries.push({
          method: expected.method,
          status: "missing",
          expected: expected.result ?? expected.error,
        });
        continue;
      }
  
      if (expected.method !== actual.method) {
        entries.push({
          method: expected.method,
          status: "fail",
          expected: expected.result ?? expected.error,
          actual: actual.result ?? actual.error,
          diff: `Method mismatch: expected "${expected.method}", got "${actual.method}"`,
        });
        continue;
      }
  
      // Compare results structurally
      const expectedPayload = expected.error ? { error: expected.error } : { result: expected.result };
      const actualPayload = actual.error ? { error: actual.error } : { result: actual.result };
  
      if (deepEqual(expectedPayload, actualPayload)) {
        entries.push({ method: expected.method, status: "pass" });
      } else {
        entries.push({
          method: expected.method,
          status: "fail",
          expected: expectedPayload,
          actual: actualPayload,
          diff: summarizeDiff(expectedPayload, actualPayload),
        });
      }
    }
  
    // Check for extra live responses not in cassette
    for (let i = cassetteResponses.length; i < liveResponses.length; i++) {
      const actual = liveResponses[i]!;
      entries.push({
        method: actual.method,
        status: "fail",
        actual: actual.result ?? actual.error,
        diff: "Extra response not in cassette",
      });
    }
  
    return {
      targetId: cassette.targetId,
      totalEntries: entries.length,
      passed: entries.filter((e) => e.status === "pass").length,
      failed: entries.filter((e) => e.status === "fail").length,
      missing: entries.filter((e) => e.status === "missing").length,
      entries,
    };
  }

• src/verify.ts:101-131 (helper)

  Helper functions deepEqual() and summarizeDiff() used by compareResponses for structural comparison and diff formatting.

  function deepEqual(a: unknown, b: unknown): boolean {
    if (a === b) return true;
    if (a === null || b === null) return false;
    if (typeof a !== typeof b) return false;
  
    if (Array.isArray(a)) {
      if (!Array.isArray(b) || a.length !== b.length) return false;
      return a.every((val, i) => deepEqual(val, b[i]));
    }
  
    if (typeof a === "object") {
      const aObj = a as Record<string, unknown>;
      const bObj = b as Record<string, unknown>;
      const aKeys = Object.keys(aObj).sort();
      const bKeys = Object.keys(bObj).sort();
      if (aKeys.length !== bKeys.length) return false;
      return aKeys.every((key, i) => key === bKeys[i] && deepEqual(aObj[key], bObj[key]));
    }
  
    return false;
  }
  
  function summarizeDiff(expected: unknown, actual: unknown): string {
    const expStr = JSON.stringify(expected, null, 2);
    const actStr = JSON.stringify(actual, null, 2);
  
    if (expStr.length > 200 || actStr.length > 200) {
      return "Response content differs (too large to display inline)";
    }
    return `Expected: ${expStr}\nActual: ${actStr}`;
  }

• src/server.ts:117-154 (registration)

  The MCP server instance where the 'verify' tool is registered via server.tool() along with all other tools.

  server.tool(
    "scan",
    "Use this to check if all your MCP servers are healthy. Auto-discovers servers from Claude config files, connects to each one, and verifies tools/prompts/resources respond correctly. Use with deep=true to also invoke tools and confirm they actually execute. Returns pass/fail status for every server.",
    {
      config: z.string().optional().describe("Path to a specific MCP config file. If omitted, scans default locations."),
      deep: z.boolean().optional().describe("Also invoke safe tools to verify they execute."),
      security: z.boolean().optional().describe("Run security analysis on tool schemas."),
    },
    async ({ config, deep, security }) => {
      const startMs = Date.now();
      const targets = await scanForTargets(config);
      if (targets.length === 0) {
        logRequest("scan", startMs);
        return { content: [{ type: "text" as const, text: "No MCP server configs found." }] };
      }
  
      const opts: RunOptions = {};
      if (deep) opts.invokeTools = true;
      if (security) opts.securityCheck = true;
  
      const lines: string[] = [`Discovered ${targets.length} server(s):\n`];
      for (const t of targets) {
        if (t.config.targetId === "mcp-observatory") continue;
  
        lines.push(`--- ${t.config.targetId} (from ${t.source}) ---`);
        try {
          const artifact = await runTarget(t.config, opts);
          lines.push(formatRun(artifact));
        } catch (error) {
          const msg = errorMessage(error);
          lines.push(`  Error: ${msg}`);
        }
        lines.push("");
      }
      logRequest("scan", startMs);
      return { content: [{ type: "text" as const, text: lines.join("\n") }] };
    },
  );