app-screen-mcp

Instructions

Implementation Reference

• src/index.ts:835-878 (handler)

  The tapById method handles the tap_id tool, searching for an element by ID in the UI tree, handling specific toggle logic for switches, and executing the tap via idb.

  private async tapById(id: string, udid?: string) {
    const target = await resolveUdid(udid);
    try {
      const elements = await this.fetchUiTree(target);
      const matches = findElementsById(elements, id);
  
      if (matches.length === 0) {
        throw new McpError(ErrorCode.InvalidRequest, `No element found with identifier "${id}"`);
      }
  
      const el = matches[0];
      const cx = Math.round(el.frame.x + el.frame.w / 2);
      const cy = Math.round(el.frame.y + el.frame.h / 2);
  
      // UISwitch requires a swipe gesture; a raw coordinate tap doesn't toggle it
      if (/switch/i.test(el.type)) {
        const isOn = el.value === '1';
        const fromX = isOn
          ? Math.round(el.frame.x + el.frame.w - 6)
          : Math.round(el.frame.x + 6);
        const toX = isOn
          ? Math.round(el.frame.x + 6)
          : Math.round(el.frame.x + el.frame.w - 6);
        await execAsync(`idb ui swipe --udid ${target} ${fromX} ${cy} ${toX} ${cy} --duration 0.15`);
        return {
          content: [{
            type: 'text',
            text: `Toggled switch "${id}" (was ${isOn ? 'ON' : 'OFF'}) at (${cx}, ${cy}) on ${target}`,
          }],
        };
      }
  
      await execAsync(`idb ui tap --udid ${target} ${cx} ${cy}`);
      return {
        content: [{
          type: 'text',
          text: `Tapped element with id "${id}" at (${cx}, ${cy}) on ${target}`,
        }],
      };
    } catch (error: any) {
      if (error instanceof McpError) throw error;
      throw new McpError(ErrorCode.InternalError, `Failed to tap by id: ${error.message}`);
    }
  }

• src/index.ts:473-484 (registration)

  Definition of the tap_id tool in the ListToolsRequestSchema.

    name: 'tap_id',
    description: 'Find a UI element by its accessibility identifier and tap its center.',
    inputSchema: {
      type: 'object',
      properties: {
        id: { type: 'string', description: 'Accessibility identifier of the element (set via .accessibilityIdentifier in SwiftUI)' },
        udid: { type: 'string', description: 'Simulator UDID (optional, defaults to booted simulator)' },
      },
      required: ['id'],
      additionalProperties: false,
    },
  },

• src/index.ts:534-535 (registration)

  CallToolRequestSchema handler for tap_id.

  case 'tap_id':
    return this.tapById(args.id as string, args.udid);