Run Test Suite

Instructions

Implementation Reference

• handlers/runTestSuiteHandler.ts:19-154 (handler)

  The main handler function for 'run_test_suite'. Resolves suite UUID (via project/suite name lookup), handles tunneling for localhost URLs (port probe, tunnel provision/ensure, health check), and calls the Debugg AI API to trigger the test suite run asynchronously. Returns result with a note to poll with get_test_suite_results.

  export async function runTestSuiteHandler(
    input: RunTestSuiteInput,
    _context: ToolContext,
  ): Promise<ToolResponse> {
    const start = Date.now();
    logger.toolStart('run_test_suite', input);
  
    const client = new DebuggAIServerClient(config.api.key);
    await client.init();
  
    let acquiredKeyId: string | null = null;
    let tunnelId: string | undefined;
  
    try {
      let suiteUuid = input.suiteUuid;
      if (!suiteUuid) {
        let projectUuid = input.projectUuid;
        if (!projectUuid) {
          const resolved = await resolveProject(client, input.projectName!);
          if ('error' in resolved) return errorResp(resolved.error, resolved.message, { candidates: (resolved as any).candidates });
          projectUuid = resolved.uuid;
        }
        const resolved = await resolveTestSuite(client, input.suiteName!, projectUuid);
        if ('error' in resolved) return errorResp(resolved.error, resolved.message, { candidates: (resolved as any).candidates });
        suiteUuid = resolved.uuid;
      }
  
      // Resolve the effective target URL — tunnel if localhost, pass-through otherwise.
      let effectiveTargetUrl = input.targetUrl;
  
      if (input.targetUrl) {
        const ctx = buildContext(input.targetUrl);
  
        if (ctx.isLocalhost) {
          const port = extractLocalhostPort(ctx.originalUrl);
          if (typeof port === 'number') {
            const probe = await probeLocalPort(port);
            if (!probe.reachable) {
              return errorResp(
                'LocalServerUnreachable',
                `No server listening on 127.0.0.1:${port}. Start your dev server before running the suite. (${probe.code}: ${probe.detail ?? 'no detail'})`,
                { port, probeCode: probe.code, elapsedMs: probe.elapsedMs },
              );
            }
          }
  
          if (config.devMode) {
            // Dev mode: local backend can reach localhost directly — no tunnel needed.
            logger.info(`run_test_suite: dev mode — using localhost URL directly: ${input.targetUrl}`);
          } else {
            // Reuse an existing tunnel for this port if one is already active.
            const reused = findExistingTunnel(ctx);
            if (reused) {
              effectiveTargetUrl = reused.targetUrl ?? input.targetUrl;
              tunnelId = reused.tunnelId;
            } else {
              // Provision a new tunnel.
              let tunnel;
              try {
                tunnel = await client.tunnels!.provisionWithRetry();
              } catch (provisionError) {
                const msg = provisionError instanceof Error ? provisionError.message : String(provisionError);
                const diag = provisionError instanceof TunnelProvisionError ? ` ${provisionError.diagnosticSuffix()}` : '';
                return errorResp(
                  'TunnelProvisionFailed',
                  `Failed to provision tunnel for ${input.targetUrl}. (Detail: ${msg})${diag}`,
                );
              }
              acquiredKeyId = tunnel.keyId;
  
              let tunneled;
              try {
                tunneled = await ensureTunnel(
                  ctx,
                  tunnel.tunnelKey,
                  tunnel.tunnelId,
                  tunnel.keyId,
                  () => client.revokeNgrokKey(tunnel.keyId),
                );
              } catch (tunnelError) {
                const msg = tunnelError instanceof Error ? tunnelError.message : String(tunnelError);
                return errorResp('TunnelCreationFailed', `Tunnel creation failed for ${input.targetUrl}. (Detail: ${msg})`);
              }
  
              // Health probe — catches ERR_NGROK_8012 and bind mismatches before
              // the remote agent wastes steps trying to reach the server.
              if (tunneled.targetUrl) {
                const health = await probeTunnelHealth(tunneled.targetUrl);
                if (!health.healthy) {
                  if (tunneled.tunnelId) {
                    tunnelManager.stopTunnel(tunneled.tunnelId).catch((err) =>
                      logger.warn(`Failed to stop broken tunnel ${tunneled.tunnelId}: ${err}`),
                    );
                  }
                  return errorResp(
                    'TunnelTrafficBlocked',
                    `Tunnel established but traffic isn't reaching the dev server. ${health.detail ?? ''}`,
                    { code: health.code, ngrokErrorCode: health.ngrokErrorCode, elapsedMs: health.elapsedMs },
                  );
                }
              }
  
              effectiveTargetUrl = tunneled.targetUrl ?? input.targetUrl;
              tunnelId = tunneled.tunnelId;
            }
  
            logger.info(`run_test_suite: localhost detected, tunneled ${input.targetUrl} → ${effectiveTargetUrl}`);
          }
        }
      }
  
      const result = await client.runTestSuite(suiteUuid, { targetUrl: effectiveTargetUrl });
      logger.toolComplete('run_test_suite', Date.now() - start);
      return {
        content: [{
          type: 'text',
          text: JSON.stringify({
            ...result,
            ...(tunnelId ? { tunnelActive: true, originalUrl: input.targetUrl } : {}),
            note: 'Tests are running asynchronously. Use get_test_suite_results to check progress.',
          }, null, 2),
        }],
      };
    } catch (error) {
      logger.toolError('run_test_suite', error as Error, Date.now() - start);
      throw handleExternalServiceError(error, 'DebuggAI', 'run_test_suite');
    } finally {
      // Tunnels are NOT torn down — reuse pattern + 55-min idle auto-shutoff.
      // Only revoke an orphaned key (acquired but tunnel creation failed).
      if (acquiredKeyId && !tunnelId) {
        client.revokeNgrokKey(acquiredKeyId).catch((err) =>
          logger.warn(`Failed to revoke unused ngrok key ${acquiredKeyId}: ${err}`),
        );
      }
    }
  }

• types/index.ts:410-416 (schema)

  Zod schema (RunTestSuiteInputSchema) and type (RunTestSuiteInput) defining input validation: suiteUuid or suiteName + projectUuid/projectName, optional targetUrl.

  export const RunTestSuiteInputSchema = z.object({
    ...suiteIdentifier,
    ...projectIdentifier,
    targetUrl: z.string().url().optional(),
  }).strict();
  
  export type RunTestSuiteInput = z.infer<typeof RunTestSuiteInputSchema>;

• tools/testSuiteTools.ts:177-198 (registration)

  Tool schema definition (buildRunTestSuiteTool) and validated tool builder (buildValidatedRunTestSuiteTool) that registers the tool with name 'run_test_suite', its input schema, description, and the handler.

  // ── run_test_suite ────────────────────────────────────────────────────────────
  
  export function buildRunTestSuiteTool(): Tool {
    return {
      name: 'run_test_suite',
      title: 'Run Test Suite',
      description: 'Trigger all test cases in a suite to run asynchronously. Accepts suiteUuid directly, or suiteName + project identifier. Optional: targetUrl to override the default test target. Returns {suiteUuid, runStatus, testsTriggered, note}. Use get_test_suite_results to poll for results.',
      inputSchema: {
        type: 'object',
        properties: {
          ...SUITE_PROPS,
          ...PROJECT_PROPS,
          targetUrl: { type: 'string', description: 'Optional URL to run tests against (overrides default). Must be a full URL.' },
        },
        additionalProperties: false,
      },
    };
  }
  
  export function buildValidatedRunTestSuiteTool(): ValidatedTool {
    return { ...buildRunTestSuiteTool(), inputSchema: RunTestSuiteInputSchema, handler: runTestSuiteHandler };
  }

• tools/index.ts:54-78 (registration)

  Registration of 'run_test_suite' in tool collections: buildRunTestSuiteTool() in the tools array (line 54) and buildValidatedRunTestSuiteTool() in the validated tools array (line 76).

    buildRunTestSuiteTool(),
    buildGetTestSuiteResultsTool(),
  ];
  const validated: ValidatedTool[] = [
    buildValidatedTestPageChangesTool(ctx),
    buildValidatedTriggerCrawlTool(ctx),
    buildValidatedProbePageTool(),
    buildValidatedSearchProjectsTool(),
    buildValidatedSearchEnvironmentsTool(),
    buildValidatedCreateEnvironmentTool(),
    buildValidatedUpdateEnvironmentTool(),
    buildValidatedDeleteEnvironmentTool(),
    buildValidatedUpdateProjectTool(),
    buildValidatedDeleteProjectTool(),
    buildValidatedSearchExecutionsTool(),
    buildValidatedCreateProjectTool(),
    buildValidatedCreateTestSuiteTool(),
    buildValidatedSearchTestSuitesTool(),
    buildValidatedDeleteTestSuiteTool(),
    buildValidatedCreateTestCaseTool(),
    buildValidatedUpdateTestCaseTool(),
    buildValidatedDeleteTestCaseTool(),
    buildValidatedRunTestSuiteTool(),
    buildValidatedGetTestSuiteResultsTool(),
  ];

• services/index.ts:631-644 (helper)

  DebuggAIServerClient.runTestSuite() — the API client method that POSTs to api/v1/test-suites/{suiteUuid}/run/ to trigger the async test suite execution.

  public async runTestSuite(
    suiteUuid: string,
    params: { targetUrl?: string },
  ): Promise<{ suiteUuid: string; runStatus: string; testsTriggered: number }> {
    if (!this.tx) throw new Error('Client not initialized — call init() first');
    const body: Record<string, any> = {};
    if (params.targetUrl) body.target_url = params.targetUrl;
    const s = await this.tx.post<any>(`api/v1/test-suites/${suiteUuid}/run/`, body);
    return {
      suiteUuid,
      runStatus: s?.runStatus ?? s?.run_status ?? 'PENDING',
      testsTriggered: (s?.tests ?? []).length,
    };
  }