swipeOnScreen
Automate screen swipes in specific directions (up, down, left, right) on Android and iOS devices using predefined duration and system-inset inclusion for precise control.
Instructions
Swipe on screen in a specific direction
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| direction | Yes | Direction to swipe | |
| duration | Yes | Duration of the swipe in milliseconds | |
| includeSystemInsets | Yes | Whether to include system inset areas in the swipe | |
| platform | Yes | Platform of the device |
Implementation Reference
- src/server/interactionTools.ts:382-403 (handler)The swipeOnScreenHandler function that executes the tool logic by instantiating SwipeOnScreen class and calling its execute method.const swipeOnScreenHandler = async (device: BootedDevice, args: SwipeOnScreenArgs, progress?: ProgressCallback) => { try { const swipeOnScreen = new SwipeOnScreen(device); const result = await swipeOnScreen.execute( args.direction, { duration: args.duration ?? 100, includeSystemInsets: args.includeSystemInsets }, progress ); return createJSONToolResponse({ message: `Swiped ${args.direction} on screen${args.includeSystemInsets ? " including navigation areas" : ""}`, observation: result.observation, ...result }); } catch (error) { throw new ActionableError(`Failed to swipe on screen: ${error}`); } };
- Zod schema defining the input arguments for the swipeOnScreen tool.export const swipeOnScreenSchema = z.object({ direction: z.enum(["up", "down", "left", "right"]).describe("Direction to swipe"), includeSystemInsets: z.boolean().describe("Whether to include system inset areas in the swipe"), duration: z.number().describe("Duration of the swipe in milliseconds"), platform: z.enum(["android", "ios"]).describe("Platform of the device") });
- src/server/interactionTools.ts:698-704 (registration)Registration of the swipeOnScreen tool with the ToolRegistry, including name, description, schema, and handler.ToolRegistry.registerDeviceAware( "swipeOnScreen", "Swipe on screen in a specific direction", swipeOnScreenSchema, swipeOnScreenHandler, true // Supports progress notifications );
- The SwipeOnScreen class implementing the core swipe gesture logic for both Android and iOS platforms, including coordinate calculation respecting system insets.export class SwipeOnScreen extends BaseVisualChange { private executeGesture: ExecuteGesture; private elementUtils: ElementUtils; constructor(device: BootedDevice, adb: AdbUtils | null = null, axe: Axe | null = null) { super(device, adb, axe); this.executeGesture = new ExecuteGesture(device, adb); this.elementUtils = new ElementUtils(); } /** * Swipe on screen in a given direction * @param observeResult - Previous ObserveResult * @param direction - Direction to swipe ('up', 'down', 'left', 'right') * @param options - Additional gesture options * @param progress - Optional progress callback * @returns Result of the swipe operation */ async executeAndroid( observeResult: ObserveResult, direction: "up" | "down" | "left" | "right", options: GestureOptions = {}, progress?: ProgressCallback ): Promise<SwipeResult> { logger.info(`[SwipeOnScreen] In observedInteraction callback`); if (!observeResult.screenSize) { logger.error(`[SwipeOnScreen] No screen size available in observeResult`); throw new ActionableError("Could not determine screen size"); } const screenWidth = observeResult.screenSize.width; const screenHeight = observeResult.screenSize.height; const insets = observeResult.systemInsets || { top: 0, right: 0, bottom: 0, left: 0 }; logger.info(`[SwipeOnScreen] Screen dimensions: ${screenWidth}x${screenHeight}`); logger.info(`[SwipeOnScreen] System insets: ${JSON.stringify(insets)}`); // Calculate the bounds based on system insets const bounds = (options.includeSystemInsets === true) ? { left: 0, top: 0, right: screenWidth, bottom: screenHeight } : { left: insets.left, top: insets.top, right: screenWidth - insets.right, bottom: screenHeight - insets.bottom }; logger.info(`[SwipeOnScreen] Calculated bounds: ${JSON.stringify(bounds)}`); const { startX, startY, endX, endY } = this.elementUtils.getSwipeWithinBounds( direction, bounds ); logger.info(`[SwipeOnScreen] Raw swipe coordinates: start=(${startX}, ${startY}), end=(${endX}, ${endY})`); const flooredStartX = Math.floor(startX); const flooredStartY = Math.floor(startY); const flooredEndX = Math.floor(endX); const flooredEndY = Math.floor(endY); logger.info(`[SwipeOnScreen] Floored swipe coordinates: start=(${flooredStartX}, ${flooredStartY}), end=(${flooredEndX}, ${flooredEndY})`); try { const result = await this.executeGesture.swipe( flooredStartX, flooredStartY, flooredEndX, flooredEndY, options ); logger.info(`[SwipeOnScreen] Swipe completed successfully: ${JSON.stringify(result)}`); return result; } catch (error) { logger.error(`[SwipeOnScreen] Swipe execution failed: ${error}`); throw error; } } /** * Swipe on screen in a given direction * @param observeResult - Previous ObserveResult * @param direction - Direction to swipe ('up', 'down', 'left', 'right') * @param options - Additional gesture options * @param progress - Optional progress callback * @returns Result of the swipe operation */ async executeiOS( observeResult: ObserveResult, direction: "up" | "down" | "left" | "right", options: GestureOptions = {}, progress?: ProgressCallback ): Promise<SwipeResult> { logger.info(`[SwipeOnScreen] In observedInteraction callback for iOS`); if (!observeResult.screenSize) { logger.error(`[SwipeOnScreen] No screen size available in observeResult`); throw new ActionableError("Could not determine screen size"); } const screenWidth = observeResult.screenSize.width; const screenHeight = observeResult.screenSize.height; const insets = observeResult.systemInsets || { top: 0, right: 0, bottom: 0, left: 0 }; logger.info(`[SwipeOnScreen] Screen dimensions: ${screenWidth}x${screenHeight}`); logger.info(`[SwipeOnScreen] System insets: ${JSON.stringify(insets)}`); // Calculate the bounds based on system insets const bounds = (options.includeSystemInsets === true) ? { left: 0, top: 0, right: screenWidth, bottom: screenHeight } : { left: insets.left, top: insets.top, right: screenWidth - insets.right, bottom: screenHeight - insets.bottom }; logger.info(`[SwipeOnScreen] Calculated bounds: ${JSON.stringify(bounds)}`); const { startX, startY, endX, endY } = this.elementUtils.getSwipeWithinBounds( direction, bounds ); logger.info(`[SwipeOnScreen] Raw swipe coordinates: start=(${startX}, ${startY}), end=(${endX}, ${endY})`); // Ensure coordinates are bounded by screen size and always positive const boundedStartX = Math.max(0, Math.min(Math.floor(startX), screenWidth - 1)); const boundedStartY = Math.max(0, Math.min(Math.floor(startY), screenHeight - 1)); const boundedEndX = Math.max(0, Math.min(Math.floor(endX), screenWidth - 1)); const boundedEndY = Math.max(0, Math.min(Math.floor(endY), screenHeight - 1)); logger.info(`[SwipeOnScreen] Bounded swipe coordinates: start=(${boundedStartX}, ${boundedStartY}), end=(${boundedEndX}, ${boundedEndY})`); try { const result = await this.axe.swipe( boundedStartX, boundedStartY, boundedEndX, boundedEndY ); logger.info(`[SwipeOnScreen] Swipe completed successfully: ${JSON.stringify(result)}`); return result; } catch (error) { logger.error(`[SwipeOnScreen] Swipe execution failed: ${error}`); throw error; } } /** * Swipe on screen in a given direction * @param direction - Direction to swipe ('up', 'down', 'left', 'right') * @param options - Additional gesture options * @param progress - Optional progress callback * @returns Result of the swipe operation */ async execute( direction: "up" | "down" | "left" | "right", options: GestureOptions = {}, progress?: ProgressCallback ): Promise<SwipeResult> { logger.info(`[SwipeOnScreen] Starting swipe: direction=${direction}, platform=${this.device.platform}`); logger.info(`[SwipeOnScreen] Options: ${JSON.stringify(options)}`); return this.observedInteraction( async (observeResult: ObserveResult) => { logger.info(`[SwipeOnScreen] In observedInteraction callback`); switch (this.device.platform) { case "android": return this.executeAndroid(observeResult, direction, options, progress); case "ios": return this.executeiOS(observeResult, direction, options, progress); } }, { changeExpected: false, timeoutMs: 500, progress } ); } }