homeScreen
Navigate back to the home screen on Android or iOS devices using the home button. Part of the AutoMobile MCP server for mobile automation testing and control.
Instructions
Return to the home screen by pressing the home button
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| platform | Yes | Platform of the device |
Implementation Reference
- src/server/interactionTools.ts:634-647 (handler)The tool handler function for 'homeScreen' that instantiates HomeScreen class and executes itconst homeScreenHandler = async (device: BootedDevice, args: any, progress?: ProgressCallback) => { try { const homeScreen = new HomeScreen(device); const result = await homeScreen.execute(progress); return createJSONToolResponse({ message: "Pressed home button to return to the home screen", observation: result.observation, ...result }); } catch (error) { throw new ActionableError(`Failed to go to home screen: ${error}`); } };
- src/features/action/HomeScreen.ts:15-315 (handler)Core implementation of HomeScreen navigation logic, including detection of gesture/hardware/element methods and executionexport class HomeScreen extends BaseVisualChange { private static navigationCache = new Map<string, NavigationCache>(); private static readonly CACHE_DURATION_MS = 300000; // 5 minutes private elementUtils: ElementUtils; constructor(device: BootedDevice, adb: AdbUtils | null = null, axe: Axe | null = null) { super(device, adb, axe); this.device = device; this.elementUtils = new ElementUtils(); } async execute(progress?: ProgressCallback): Promise<HomeScreenResult> { // Check cache first const cachedMethod = this.getCachedNavigationMethod(this.device); if (cachedMethod) { logger.info(`[HomeScreen] Using cached navigation method: ${cachedMethod}`); // Only try the cached method, if it fails, surface the error to the caller. return await this.executeNavigationMethod(cachedMethod, progress); } // Detect navigation style and only try that, no fallback logic. const detectedMethod = await this.detectNavigationStyle(progress); // Cache the detected method (without getting device props again) this.cacheNavigationMethodSimple(this.device, detectedMethod); return await this.executeNavigationMethod(detectedMethod, progress); } private getCachedNavigationMethod(device: BootedDevice): "gesture" | "hardware" | "element" | null { const cached = HomeScreen.navigationCache.get(device.deviceId); if (!cached) {return null;} const now = Date.now(); if (now - cached.timestamp > HomeScreen.CACHE_DURATION_MS) { HomeScreen.navigationCache.delete(device.deviceId); return null; } return cached.method; } private cacheNavigationMethodSimple(device: BootedDevice, method: "gesture" | "hardware" | "element"): void { HomeScreen.navigationCache.set(device.deviceId, { method, timestamp: Date.now() }); logger.info(`[HomeScreen] Cached navigation method: ${method} for device: ${device.deviceId}`); } private async detectNavigationStyle(progress?: ProgressCallback): Promise<"gesture" | "hardware" | "element"> { if (progress) { await progress(10, 100, "Detecting navigation style..."); } // First, check device properties for navigation hints (to determine gesture navigation) const deviceProps = await this.getDeviceProperties(); const sdkVersion = parseInt(deviceProps["ro.build.version.sdk"] || "0", 10); logger.info(`[HomeScreen] SDK version: ${sdkVersion}`); // Android 10+ (API 29+) typically uses gesture navigation by default if (sdkVersion >= 29) { const hasGestureNav = await this.checkGestureNavigationEnabled(); if (hasGestureNav) { logger.info("[HomeScreen] Detected gesture navigation (Android 10+)"); return "gesture"; } } // Check view hierarchy for navigation elements if (progress) { await progress(30, 100, "Analyzing view hierarchy for navigation elements..."); } const observation = await this.observeScreen.execute(); if (observation.viewHierarchy) { const hasHomeButton = this.findHomeButton(observation.viewHierarchy); if (hasHomeButton) { logger.info("[HomeScreen] Detected navigation bar with home button"); return "element"; } } logger.info("[HomeScreen] Defaulting to hardware home button"); return "hardware"; } private async getDeviceProperties(): Promise<Record<string, string>> { try { const result = await this.adb.executeCommand("shell getprop"); const props: Record<string, string> = {}; result.stdout.split("\n").forEach(line => { const match = line.match(/\[([^\]]+)\]: \[([^\]]*)\]/); if (match) { props[match[1]] = match[2]; } }); return props; } catch (error) { logger.warn(`[HomeScreen] Failed to get device properties: ${error}`); return {}; } } private async checkGestureNavigationEnabled(): Promise<boolean> { try { // Check if gesture navigation is enabled const result = await this.adb.executeCommand("shell settings get secure navigation_mode"); const navigationMode = result.stdout.trim(); // Mode 2 typically indicates gesture navigation return navigationMode === "2"; } catch (error) { logger.debug(`[HomeScreen] Could not check gesture navigation setting: ${error}`); return false; } } private findHomeButton(viewHierarchy: any): boolean { try { // Pass the full viewHierarchy to extractRootNodes, not just the hierarchy part const rootNodes = this.elementUtils.extractRootNodes(viewHierarchy); for (const rootNode of rootNodes) { let found = false; this.elementUtils.traverseNode(rootNode, (node: any) => { const props = this.elementUtils.extractNodeProperties(node); const resourceId = props["resource-id"] || ""; const contentDesc = props["content-desc"] || ""; const className = props["class"] || ""; // Common home button resource IDs and content descriptions const homePatterns = [ "home", "launcher", "com.android.systemui:id/home", "android:id/home", "navigation_bar_home" ]; const isHomeButton = homePatterns.some(pattern => resourceId.toLowerCase().includes(pattern) || contentDesc.toLowerCase().includes(pattern) ) && (className.includes("Button") || className.includes("ImageView")); if (isHomeButton) { found = true; } }); if (found) { return true; } } return false; } catch (error) { logger.warn(`[HomeScreen] Error finding home button in view hierarchy: ${error}`); return false; } } private async executeNavigationMethod( method: "gesture" | "hardware" | "element", progress?: ProgressCallback ): Promise<HomeScreenResult> { // Each method is tried directly. If it fails, error is surfaced. switch (method) { case "gesture": return await this.observedInteraction( async (observeResult: ObserveResult) => await this.executeGestureNavigation(observeResult, progress), { changeExpected: true, timeoutMs: 5000, progress } ); case "element": return await this.observedInteraction( async (observeResult: ObserveResult) => await this.executeElementNavigation(observeResult, progress), { changeExpected: true, timeoutMs: 5000, progress } ); case "hardware": return await this.observedInteraction( async () => await this.executeHardwareNavigation(progress), { changeExpected: true, timeoutMs: 5000, progress } ); default: throw new Error(`Unknown navigation method: ${method}`); } } private async executeGestureNavigation(oberveResult: ObserveResult, progress?: ProgressCallback): Promise<HomeScreenResult> { if (progress) { await progress(60, 100, "Executing gesture navigation..."); } // Get screen dimensions for gesture calculation if (!oberveResult.screenSize) { throw new Error("Could not get screen size for gesture navigation"); } const { width, height } = oberveResult.screenSize; // Calculate swipe coordinates from bottom center const startX = width / 2; const startY = height - 50; // Start near bottom edge const endX = startX; const endY = height / 2; // Swipe to middle of screen // Execute gesture swipe await this.adb.executeCommand( `shell input swipe ${startX} ${startY} ${endX} ${endY} 300` ); return { success: true, navigationMethod: "gesture" }; } private async executeElementNavigation(oberveResult: ObserveResult, progress?: ProgressCallback): Promise<HomeScreenResult> { if (progress) { await progress(60, 100, "Executing element navigation..."); } if (!oberveResult.viewHierarchy) { throw new Error("Could not get view hierarchy for element navigation"); } // Pass the full viewHierarchy to extractRootNodes, not just the hierarchy part const rootNodes = this.elementUtils.extractRootNodes(oberveResult.viewHierarchy); for (const rootNode of rootNodes) { let homeButton: any = null; this.elementUtils.traverseNode(rootNode, (node: any) => { const props = this.elementUtils.extractNodeProperties(node); const resourceId = props["resource-id"] || ""; const contentDesc = props["content-desc"] || ""; const className = props["class"] || ""; const homePatterns = [ "home", "launcher", "com.android.systemui:id/home", "android:id/home", "navigation_bar_home" ]; const isHomeButton = homePatterns.some(pattern => resourceId.toLowerCase().includes(pattern) || contentDesc.toLowerCase().includes(pattern) ) && (className.includes("Button") || className.includes("ImageView")); if (isHomeButton && !homeButton) { homeButton = node; } }); if (homeButton) { // Parse the node to get proper Element structure with bounds const parsedElement = this.elementUtils.parseNodeBounds(homeButton); if (!parsedElement) { throw new Error("Failed to parse home button bounds"); } const center = this.elementUtils.getElementCenter(parsedElement); await this.adb.executeCommand(`shell input tap ${center.x} ${center.y}`); return { success: true, navigationMethod: "element" }; } } throw new Error("Home button element not found"); } private async executeHardwareNavigation(progress?: ProgressCallback): Promise<HomeScreenResult> { if (progress) { await progress(60, 100, "Executing hardware navigation..."); } // Press hardware home button (keycode 3) await this.adb.executeCommand("shell input keyevent 3"); return { success: true, navigationMethod: "hardware" }; } }
- Input schema validation for the homeScreen toolexport const homeScreenSchema = z.object({ platform: z.enum(["android", "ios"]).describe("Platform of the device") });
- src/server/interactionTools.ts:787-793 (registration)Registration of the homeScreen tool in the ToolRegistryToolRegistry.registerDeviceAware( "homeScreen", "Return to the home screen by pressing the home button", homeScreenSchema, homeScreenHandler, true // Supports progress notifications );
- src/models/HomeScreenResult.ts:1-6 (schema)Type definition for the output of HomeScreen executionexport interface HomeScreenResult { success: boolean; navigationMethod?: "gesture" | "hardware" | "element"; error?: string; observation?: any; }