AutoMobile

Instructions

Implementation Reference

• src/server/interactionTools.ts:690-696 (registration)

  Registration of the swipeOnElement tool in the ToolRegistry with name, description, schema, and handler.

  ToolRegistry.registerDeviceAware(
    "swipeOnElement",
    "Swipe on a specific element",
    swipeOnElementSchema,
    swipeOnElementHandler,
    true // Supports progress notifications
  );

• src/server/interactionTools.ts:129-135 (schema)

  Zod schema definition for swipeOnElement tool arguments.

  export const swipeOnElementSchema = z.object({
    containerElementId: z.string().optional().describe("Container element ID to restrict the search within"),
    elementId: z.string().describe("ID of the element to swipe on"),
    direction: z.enum(["up", "down", "left", "right"]).describe("Direction to swipe"),
    duration: z.number().describe("Duration of the swipe in milliseconds"),
    platform: z.enum(["android", "ios"]).describe("Platform of the device")
  });

• src/server/interactionTools.ts:304-379 (handler)

  Main handler function for the swipeOnElement tool. Finds the element by ID, instantiates SwipeOnElement class, and executes the swipe.

  const swipeOnElementHandler = async (device: BootedDevice, args: SwipeOnElementArgs, progress?: ProgressCallback) => {
    try {
      // Validate element ID format
      const elementId = args.elementId;
  
      // Check if the elementId looks like coordinate bounds instead of a resource ID
      const boundsPattern = /^\[\d+,\d+\]\[\d+,\d+\]$/;
      if (boundsPattern.test(elementId)) {
        throw new ActionableError(
          `Invalid element ID: "${elementId}" appears to be coordinate bounds. ` +
          `Please provide a proper Android resource ID (e.g., "com.example.app:id/button") ` +
          `or use swipeOnScreen with coordinates instead.`
        );
      }
  
      // Check if elementId is empty or just whitespace
      if (!elementId || elementId.trim().length === 0) {
        throw new ActionableError("Element ID cannot be empty. Please provide a valid Android resource ID.");
      }
  
      const observeScreen = new ObserveScreen(device);
      const swipeOnElement = new SwipeOnElement(device);
  
      // First observe to find the element
      const observeResult = await observeScreen.execute();
      if (!observeResult.viewHierarchy) {
        throw new ActionableError("Could not get view hierarchy to find element for swipe.");
      }
  
      const element = elementUtils.findElementByResourceId(
        observeResult.viewHierarchy,
        elementId,
        args.containerElementId, // Search within the specific container
        true // partial match
      );
  
      if (!element) {
        // Provide helpful suggestions
        const allResourceIds: string[] = [];
        const rootNodes = elementUtils.extractRootNodes(observeResult.viewHierarchy);
  
        for (const rootNode of rootNodes) {
          elementUtils.traverseNode(rootNode, (node: any) => {
            const nodeProperties = elementUtils.extractNodeProperties(node);
            if (nodeProperties["resource-id"] && nodeProperties["resource-id"].trim()) {
              allResourceIds.push(nodeProperties["resource-id"]);
            }
          });
        }
  
        const uniqueResourceIds = [...new Set(allResourceIds)].slice(0, 5); // Show first 5 unique IDs
        const suggestion = uniqueResourceIds.length > 0
          ? ` Available resource IDs include: ${uniqueResourceIds.join(", ")}`
          : " No elements with resource IDs found on current screen.";
  
        throw new ActionableError(
          `Element not found with ID "${elementId}".${suggestion} ` +
          `Use the 'observe' command to see the current view hierarchy and find valid element IDs.`
        );
      }
  
      const result = await swipeOnElement.execute(
        element,
        args.direction,
        { duration: args.duration ?? 100 },
        progress
      );
  
      return createJSONToolResponse({
        message: `Swiped ${args.direction} on element with ID "${elementId}"`,
        observation: result.observation
      });
    } catch (error) {
      throw new ActionableError(`Failed to swipe on element: ${error}`);
    }
  };

• src/features/action/SwipeOnElement.ts:14-82 (helper)

  Core SwipeOnElement class extending BaseVisualChange, containing the execute method that performs the actual swipe gesture using calculated coordinates within element bounds.

  export class SwipeOnElement extends BaseVisualChange {
    private executeGesture: ExecuteGesture;
    private elementUtils: ElementUtils;
  
    constructor(device: BootedDevice, adb: AdbUtils | null = null, axe: Axe | null = null) {
      super(device, adb, axe);
      this.executeGesture = new ExecuteGesture(device, adb);
      this.elementUtils = new ElementUtils();
    }
  
    /**
     * Swipe on a specific element in a given direction
     * @param element - The element to swipe on
     * @param direction - Direction to swipe ('up', 'down', 'left', 'right')
     * @param options - Additional gesture options
     * @param progress - Optional progress callback
     * @returns Result of the swipe operation
     */
    async execute(
      element: Element,
      direction: "up" | "down" | "left" | "right",
      options: GestureOptions = {},
      progress?: ProgressCallback
    ): Promise<SwipeResult> {
      logger.info(`[SwipeOnElement] Starting swipe: direction=${direction}, platform=${this.device.platform}`);
      logger.info(`[SwipeOnElement] Element bounds: ${JSON.stringify(element.bounds)}`);
      logger.info(`[SwipeOnElement] Options: ${JSON.stringify(options)}`);
  
      return this.observedInteraction(
        async () => {
          logger.info(`[SwipeOnElement] In observedInteraction callback`);
  
          const { startX, startY, endX, endY } = this.elementUtils.getSwipeWithinBounds(
            direction,
            element.bounds
          );
  
          logger.info(`[SwipeOnElement] Raw swipe coordinates: start=(${startX}, ${startY}), end=(${endX}, ${endY})`);
  
          const flooredStartX = Math.floor(startX);
          const flooredStartY = Math.floor(startY);
          const flooredEndX = Math.floor(endX);
          const flooredEndY = Math.floor(endY);
  
          logger.info(`[SwipeOnElement] Floored swipe coordinates: start=(${flooredStartX}, ${flooredStartY}), end=(${flooredEndX}, ${flooredEndY})`);
  
          try {
            const result = await this.executeGesture.swipe(
              flooredStartX,
              flooredStartY,
              flooredEndX,
              flooredEndY,
              options
            );
            logger.info(`[SwipeOnElement] Swipe completed successfully: ${JSON.stringify(result)}`);
            return result;
          } catch (error) {
            logger.error(`[SwipeOnElement] Swipe execution failed: ${error}`);
            throw error;
          }
        },
        {
          changeExpected: false,
          timeoutMs: 500,
          progress
        }
      );
    }
  }