AutoMobile

Instructions

Implementation Reference

• src/server/interactionTools.ts:682-688 (registration)

  Registration of the 'pressButton' tool using ToolRegistry.registerDeviceAware, linking name, description, schema, and handler.

  ToolRegistry.registerDeviceAware(
    "pressButton",
    "Press a hardware button on the device",
    pressButtonSchema,
    pressButtonHandler,
    true // Supports progress notifications
  );

• src/server/interactionTools.ts:288-301 (handler)

  The pressButtonHandler function wraps the PressButton class execution, handles errors, and formats the tool response.

  const pressButtonHandler = async (device: BootedDevice, args: PressButtonArgs, progress?: ProgressCallback) => {
    try {
      const pressButton = new PressButton(device);
      const result = await pressButton.execute(args.button, progress); // observe = true
  
      return createJSONToolResponse({
        message: `Pressed button ${args.button}`,
        observation: result.observation,
        ...result
      });
    } catch (error) {
      throw new ActionableError(`Failed to press button: ${error}`);
    }
  };

• src/server/interactionTools.ts:177-181 (schema)

  Zod schema defining input arguments for pressButton: button type and platform.

  export const pressButtonSchema = z.object({
    button: z.enum(["home", "back", "menu", "power", "volume_up", "volume_down", "recent"])
      .describe("The button to press"),
    platform: z.enum(["android", "ios"]).describe("Platform of the device")
  });

• src/features/action/PressButton.ts:6-101 (handler)

  Core PressButton class with execute method implementing platform-specific button presses (ADB keyevent for Android, Axe for iOS).

  export class PressButton extends BaseVisualChange {
    constructor(device: BootedDevice, adb: AdbUtils | null = null, axe: Axe | null = null) {
      super(device, adb, axe);
      this.device = device;
    }
  
    async execute(
      button: string,
      progress?: ProgressCallback
    ): Promise<PressButtonResult> {
      return this.observedInteraction(
        async () => {
          try {
            // Platform-specific button press execution
            switch (this.device.platform) {
              case "android":
                return await this.executeAndroidButtonPress(button);
              case "ios":
                return await this.executeiOSButtonPress(button);
              default:
                throw new Error(`Unsupported platform: ${this.device.platform}`);
            }
          } catch (error) {
            return {
              success: false,
              button,
              keyCode: -1,
              error: `Failed to press button: ${error instanceof Error ? error.message : String(error)}`
            };
          }
        },
        {
          changeExpected: false,
          timeoutMs: 2000, // Reduce timeout for faster execution
          progress
        }
      );
    }
  
    /**
     * Execute Android-specific button press
     * @param button - Button name to press
     * @returns Result of the button press operation
     */
    private async executeAndroidButtonPress(button: string): Promise<PressButtonResult> {
      const keyCodeMap: Record<string, number> = {
        "home": 3,
        "back": 4,
        "menu": 82,
        "power": 26,
        "volume_up": 24,
        "volume_down": 25,
        "recent": 187,
      };
  
      const keyCode = keyCodeMap[button.toLowerCase()];
      if (!keyCode) {
        return {
          success: false,
          button,
          keyCode: -1,
          error: `Unsupported button: ${button}`
        };
      }
  
      await this.adb.executeCommand(`shell input keyevent ${keyCode}`);
  
      return {
        success: true,
        button,
        keyCode
      };
    }
  
    /**
     * Execute iOS-specific button press
     * @param button - Button name to press
     * @returns Result of the button press operation
     */
    private async executeiOSButtonPress(button: string): Promise<PressButtonResult> {
  
      const axeButton: AxeButton = "home";
      const supportedButtons = ["apple_pay", "home", "lock", "side_button", "siri"];
      if (!supportedButtons.includes(button.toLowerCase())) {
        throw new Error(`Unsupported iOS button: ${button}. Supported buttons: ${supportedButtons.join(", ")}`);
      }
  
      await this.axe.pressButton(axeButton);
  
      return {
        success: true,
        button,
        keyCode: 0, // iOS doesn't use keycodes, using 0 as placeholder
      };
    }
  }

• src/models/PressButtonResult.ts:6-12 (schema)

  TypeScript interface defining the return type of pressButton operations.

  export interface PressButtonResult {
    success: boolean;
    button: string;
    keyCode: number;
    observation?: ObserveResult;
    error?: string;
  }