run_test_scenario

Execute ordered test scenarios on Android devices with multiple steps and assertions to verify app functionality through automated actions and result checks.

Instructions

Execute an ordered test scenario with multiple steps and assertions. Each step performs an action and optionally verifies the result. Returns detailed pass/fail results per step.

Input Schema

TableJSON Schema

Name	Required	Description
`name`	Yes	Name of the test scenario
`steps`	Yes	Ordered list of test steps
`stop_on_failure`	No
`device_id`	No	Device serial number

Implementation Reference

src/automation/test-runner.ts:50-156 (handler)

The core implementation of the runTestScenario function which executes a sequence of test steps.

export async function runTestScenario(
  scenarioName: string,
  steps: TestStep[],
  options: {
    deviceId?: string;
    stopOnFailure?: boolean;
    captureScreenshotsOnFailure?: boolean;
  } = {}
): Promise<TestScenarioResult> {
  const {
    deviceId,
    stopOnFailure = false,
    captureScreenshotsOnFailure = true,
  } = options;

  const resolved = await deviceManager.resolveDeviceId(deviceId);
  const scenarioStart = Date.now();

  const result: TestScenarioResult = {
    name: scenarioName,
    totalSteps: steps.length,
    passed: 0,
    failed: 0,
    skipped: 0,
    results: [],
    durationMs: 0,
  };

  let shouldSkip = false;

  for (const step of steps) {
    if (shouldSkip) {
      result.results.push({
        stepName: step.name,
        status: 'skip',
        durationMs: 0,
      });
      result.skipped++;
      continue;
    }

    const stepStart = Date.now();
    let status: TestResult['status'] = 'pass';
    let error: string | undefined;
    let screenshot: string | undefined;

    try {
      // Execute the action
      await executeTestAction(step.action, step.args, resolved);

      // Wait if specified
      if (step.delayAfter) {
        await new Promise(resolve => setTimeout(resolve, step.delayAfter));
      }

      // Run assertion if specified
      if (step.assert) {
        await runAssertion(step.assert, resolved);
      }
    } catch (err) {
      status = 'fail';
      error = err instanceof Error ? err.message : String(err);

      // Capture screenshot on failure
      if (captureScreenshotsOnFailure) {
        try {
          const sc = await captureScreenshot(resolved);
          screenshot = sc.base64;
        } catch {
          // Ignore screenshot failure
        }
      }

      if (stopOnFailure) {
        shouldSkip = true;
      }
    }

    const stepResult: TestResult = {
      stepName: step.name,
      status,
      durationMs: Date.now() - stepStart,
      error,
      screenshot,
    };

    result.results.push(stepResult);
    if (status === 'pass') result.passed++;
    else result.failed++;

    log.info(`Step "${step.name}": ${status}`, {
      durationMs: stepResult.durationMs,
      ...(error ? { error } : {}),
    });
  }

  result.durationMs = Date.now() - scenarioStart;

  log.info(`Scenario "${scenarioName}" completed`, {
    passed: result.passed,
    failed: result.failed,
    skipped: result.skipped,
    durationMs: result.durationMs,
  });

  return result;
}

src/automation/test-runner.ts:11-27 (schema)

TypeScript interface defining the TestStep structure.

export interface TestStep {
  /** Step identifier */
  name: string;
  /** Action to perform */
  action: string;
  /** Action arguments */
  args: Record<string, unknown>;
  /** Optional assertion to verify after action */
  assert?: {
    type: 'element_exists' | 'element_not_exists' | 'text_visible';
    selector?: ElementSelector;
    text?: string;
    timeoutMs?: number;
  };
  /** Delay after action in ms */
  delayAfter?: number;
}

src/controllers/automation-tools.ts:85-96 (registration)

The registration and tool handler invocation for the 'run_test_scenario' MCP tool.

},
async ({ name, steps, stop_on_failure, device_id }) => {
  return await metrics.measure('run_test_scenario', device_id || 'default', async () => {
    const result = await runTestScenario(name, steps as TestStep[], {
      deviceId: device_id,
      stopOnFailure: stop_on_failure,
      captureScreenshotsOnFailure: false, // Don't include base64 screenshots in results to save space
    });
    return {
      content: [{
        type: 'text' as const,
        text: JSON.stringify({

Android MCP Server