AutoMobile

Instructions

Implementation Reference

• src/server/interactionTools.ts:690-696 (registration)

  Registration of the swipeOnElement tool in the ToolRegistry with name, description, schema, and handler.
  ToolRegistry.registerDeviceAware( "swipeOnElement", "Swipe on a specific element", swipeOnElementSchema, swipeOnElementHandler, true // Supports progress notifications );
• src/server/interactionTools.ts:129-135 (schema)

  Zod schema definition for swipeOnElement tool arguments.
  export const swipeOnElementSchema = z.object({ containerElementId: z.string().optional().describe("Container element ID to restrict the search within"), elementId: z.string().describe("ID of the element to swipe on"), direction: z.enum(["up", "down", "left", "right"]).describe("Direction to swipe"), duration: z.number().describe("Duration of the swipe in milliseconds"), platform: z.enum(["android", "ios"]).describe("Platform of the device") });
• src/server/interactionTools.ts:304-379 (handler)

  Main handler function for the swipeOnElement tool. Finds the element by ID, instantiates SwipeOnElement class, and executes the swipe.
  const swipeOnElementHandler = async (device: BootedDevice, args: SwipeOnElementArgs, progress?: ProgressCallback) => { try { // Validate element ID format const elementId = args.elementId; // Check if the elementId looks like coordinate bounds instead of a resource ID const boundsPattern = /^\[\d+,\d+\]\[\d+,\d+\]$/; if (boundsPattern.test(elementId)) { throw new ActionableError( `Invalid element ID: "${elementId}" appears to be coordinate bounds. ` + `Please provide a proper Android resource ID (e.g., "com.example.app:id/button") ` + `or use swipeOnScreen with coordinates instead.` ); } // Check if elementId is empty or just whitespace if (!elementId || elementId.trim().length === 0) { throw new ActionableError("Element ID cannot be empty. Please provide a valid Android resource ID."); } const observeScreen = new ObserveScreen(device); const swipeOnElement = new SwipeOnElement(device); // First observe to find the element const observeResult = await observeScreen.execute(); if (!observeResult.viewHierarchy) { throw new ActionableError("Could not get view hierarchy to find element for swipe."); } const element = elementUtils.findElementByResourceId( observeResult.viewHierarchy, elementId, args.containerElementId, // Search within the specific container true // partial match ); if (!element) { // Provide helpful suggestions const allResourceIds: string[] = []; const rootNodes = elementUtils.extractRootNodes(observeResult.viewHierarchy); for (const rootNode of rootNodes) { elementUtils.traverseNode(rootNode, (node: any) => { const nodeProperties = elementUtils.extractNodeProperties(node); if (nodeProperties["resource-id"] && nodeProperties["resource-id"].trim()) { allResourceIds.push(nodeProperties["resource-id"]); } }); } const uniqueResourceIds = [...new Set(allResourceIds)].slice(0, 5); // Show first 5 unique IDs const suggestion = uniqueResourceIds.length > 0 ? ` Available resource IDs include: ${uniqueResourceIds.join(", ")}` : " No elements with resource IDs found on current screen."; throw new ActionableError( `Element not found with ID "${elementId}".${suggestion} ` + `Use the 'observe' command to see the current view hierarchy and find valid element IDs.` ); } const result = await swipeOnElement.execute( element, args.direction, { duration: args.duration ?? 100 }, progress ); return createJSONToolResponse({ message: `Swiped ${args.direction} on element with ID "${elementId}"`, observation: result.observation }); } catch (error) { throw new ActionableError(`Failed to swipe on element: ${error}`); } };
• src/features/action/SwipeOnElement.ts:14-82 (helper)

  Core SwipeOnElement class extending BaseVisualChange, containing the execute method that performs the actual swipe gesture using calculated coordinates within element bounds.
  export class SwipeOnElement extends BaseVisualChange { private executeGesture: ExecuteGesture; private elementUtils: ElementUtils; constructor(device: BootedDevice, adb: AdbUtils | null = null, axe: Axe | null = null) { super(device, adb, axe); this.executeGesture = new ExecuteGesture(device, adb); this.elementUtils = new ElementUtils(); } /** * Swipe on a specific element in a given direction * @param element - The element to swipe on * @param direction - Direction to swipe ('up', 'down', 'left', 'right') * @param options - Additional gesture options * @param progress - Optional progress callback * @returns Result of the swipe operation */ async execute( element: Element, direction: "up" | "down" | "left" | "right", options: GestureOptions = {}, progress?: ProgressCallback ): Promise<SwipeResult> { logger.info(`[SwipeOnElement] Starting swipe: direction=${direction}, platform=${this.device.platform}`); logger.info(`[SwipeOnElement] Element bounds: ${JSON.stringify(element.bounds)}`); logger.info(`[SwipeOnElement] Options: ${JSON.stringify(options)}`); return this.observedInteraction( async () => { logger.info(`[SwipeOnElement] In observedInteraction callback`); const { startX, startY, endX, endY } = this.elementUtils.getSwipeWithinBounds( direction, element.bounds ); logger.info(`[SwipeOnElement] Raw swipe coordinates: start=(${startX}, ${startY}), end=(${endX}, ${endY})`); const flooredStartX = Math.floor(startX); const flooredStartY = Math.floor(startY); const flooredEndX = Math.floor(endX); const flooredEndY = Math.floor(endY); logger.info(`[SwipeOnElement] Floored swipe coordinates: start=(${flooredStartX}, ${flooredStartY}), end=(${flooredEndX}, ${flooredEndY})`); try { const result = await this.executeGesture.swipe( flooredStartX, flooredStartY, flooredEndX, flooredEndY, options ); logger.info(`[SwipeOnElement] Swipe completed successfully: ${JSON.stringify(result)}`); return result; } catch (error) { logger.error(`[SwipeOnElement] Swipe execution failed: ${error}`); throw error; } }, { changeExpected: false, timeoutMs: 500, progress } ); } }