AutoMobile

Instructions

Implementation Reference

• src/server/interactionTools.ts:756-761 (registration)

  Registration of the 'swipe' tool, which uses scrollSchema for input validation and scrollHandler for execution logic.

    "swipe",
    "Unified scroll command supporting direction and speed (no index support due to reliability)",
    scrollSchema,
    scrollHandler,
    true // Supports progress notifications
  );

• src/server/interactionTools.ts:430-545 (handler)

  The scrollHandler function, directly used as the handler for the 'swipe' tool. Implements scrolling logic by performing repeated swipes on a container element until a target is found or basic scroll is done.

  const scrollHandler = async (device: BootedDevice, args: ScrollArgs, progress?: ProgressCallback) => {
    // Element-specific scrolling
    const observeScreen = new ObserveScreen(device);
    const swipe = new SwipeOnElement(device);
    const observeResult = await observeScreen.execute();
  
    if (!observeResult.viewHierarchy) {
      throw new ActionableError("Could not get view hierarchy for element scrolling");
    }
  
    // Find the element by resource ID
    const element = elementUtils.findElementByResourceId(
      observeResult.viewHierarchy,
      args.containerElementId,
      args.containerElementId,
      true // partial match
    );
  
    if (!element) {
      throw new ActionableError(`Container element not found with ID: ${args.containerElementId}`);
    }
  
    const containerElement = element;
  
    if (!args.lookFor) {
      const duration = elementUtils.getSwipeDurationFromSpeed(args.speed);
      const result = await swipe.execute(
        containerElement,
        elementUtils.getSwipeDirectionForScroll(args.direction), {
          duration: duration,
          easing: "accelerateDecelerate",
          fingers: 1,
          randomize: false,
          lift: true,
          pressure: 1
        },
        progress
      );
  
      return createJSONToolResponse({
        message: `Scrolled ${args.direction} within element ${args.containerElementId}`,
        observation: result.observation
      });
  
    } else if (!args.lookFor.text && !args.lookFor.elementId) {
      throw new ActionableError("Either text or element id must be specified to look for something in a scrollable list.");
    } else {
      let lastObservation = await observeScreen.execute();
      if (!lastObservation.viewHierarchy || !lastObservation.screenSize) {
        throw new Error("Failed to get initial observation for scrolling until visible.");
      }
  
      const direction = args.direction;
      const maxTime = 120000; // args.lookFor.maxTime ?? 120000;
      const startTime = Date.now();
      let foundElement = null;
  
      while (Date.now() - startTime < maxTime) {
        // Re-observe the screen to get current state
        lastObservation = await observeScreen.execute();
        if (!lastObservation.viewHierarchy) {
          throw new Error("Lost observation during scroll until visible.");
        }
  
        // Check if target element is now visible
        if (args.lookFor.text) {
          foundElement = elementUtils.findElementByText(
            lastObservation.viewHierarchy,
            args.lookFor.text,
            args.containerElementId, // Search within the specific container
            true, // fuzzy match
            false // case-sensitive
          );
        } else if (args.lookFor.elementId) {
          foundElement = elementUtils.findElementByResourceId(
            lastObservation.viewHierarchy,
            args.lookFor.elementId,
            args.containerElementId, // Search within the specific container
            true // partial match
          );
        }
  
        if (foundElement) {
          logger.info(`Found element after scrolling for ${Date.now() - startTime}ms.`);
          break;
        }
  
        // Use the specific container element to swipe, not any scrollable element
        const result = await swipe.execute(
          containerElement,
          elementUtils.getSwipeDirectionForScroll(direction),
          { duration: 600 },
          progress
        );
  
        // Update observation from swipe result
        if (result.observation && result.observation.viewHierarchy) {
          lastObservation = result.observation;
        } else {
          throw new Error("Lost observation after swipe during scroll until visible.");
        }
      }
  
      if (!foundElement) {
        const target = args.lookFor.text ? `text "${args.lookFor.text}"` : `element with id "${args.lookFor.elementId}"`;
        throw new ActionableError(`${target} not found after scrolling for ${maxTime}ms.`);
      }
  
      const target = args.lookFor.text ? `text "${args.lookFor.text}"` : `element with id "${args.lookFor.elementId}"`;
      return createJSONToolResponse({
        message: `Scrolled until ${target} became visible`,
        found: !!foundElement,
        observation: lastObservation
      });
    }
  };

• src/server/interactionTools.ts:157-167 (schema)

  Zod schema for input parameters to the 'swipe' tool (shared with 'scroll' tool).

  export const scrollSchema = z.object({
    containerElementId: z.string().describe("Element ID to scroll until visible"),
    direction: z.enum(["up", "down", "left", "right"]).describe("Scroll direction"),
    lookFor: z.object({
      elementId: z.string().optional().describe("ID of the element to look for while scrolling"),
      text: z.string().optional().describe("Optional text to look for while scrolling"),
      maxTime: z.number().optional().describe("Maximum amount of time to spend scrolling, (default 10 seconds)")
    }).optional().describe("What we're searching for while scrolling"),
    speed: z.enum(["slow", "normal", "fast"]).optional().describe("Scroll speed"),
    platform: z.enum(["android", "ios"]).describe("Platform of the device")
  });

• src/features/action/SwipeOnElement.ts:32-81 (helper)

  SwipeOnElement.execute method: core helper function used by scrollHandler to perform individual swipe gestures on UI elements.

  async execute(
    element: Element,
    direction: "up" | "down" | "left" | "right",
    options: GestureOptions = {},
    progress?: ProgressCallback
  ): Promise<SwipeResult> {
    logger.info(`[SwipeOnElement] Starting swipe: direction=${direction}, platform=${this.device.platform}`);
    logger.info(`[SwipeOnElement] Element bounds: ${JSON.stringify(element.bounds)}`);
    logger.info(`[SwipeOnElement] Options: ${JSON.stringify(options)}`);
  
    return this.observedInteraction(
      async () => {
        logger.info(`[SwipeOnElement] In observedInteraction callback`);
  
        const { startX, startY, endX, endY } = this.elementUtils.getSwipeWithinBounds(
          direction,
          element.bounds
        );
  
        logger.info(`[SwipeOnElement] Raw swipe coordinates: start=(${startX}, ${startY}), end=(${endX}, ${endY})`);
  
        const flooredStartX = Math.floor(startX);
        const flooredStartY = Math.floor(startY);
        const flooredEndX = Math.floor(endX);
        const flooredEndY = Math.floor(endY);
  
        logger.info(`[SwipeOnElement] Floored swipe coordinates: start=(${flooredStartX}, ${flooredStartY}), end=(${flooredEndX}, ${flooredEndY})`);
  
        try {
          const result = await this.executeGesture.swipe(
            flooredStartX,
            flooredStartY,
            flooredEndX,
            flooredEndY,
            options
          );
          logger.info(`[SwipeOnElement] Swipe completed successfully: ${JSON.stringify(result)}`);
          return result;
        } catch (error) {
          logger.error(`[SwipeOnElement] Swipe execution failed: ${error}`);
          throw error;
        }
      },
      {
        changeExpected: false,
        timeoutMs: 500,
        progress
      }
    );
  }

• src/models/SwipeResult.ts:6-17 (schema)

  TypeScript interface defining the return type for swipe operations, used throughout the swipe implementations.

  export interface SwipeResult {
    success: boolean;
    x1: number;
    y1: number;
    x2: number;
    y2: number;
    duration: number;
    path?: number;
    easing?: "linear" | "decelerate" | "accelerate" | "accelerateDecelerate";
    observation?: ObserveResult;
    error?: string;
  }