Android Puppeteer

Instructions

Implementation Reference

• puppeteer.py:881-1063 (handler)

  The handler function for the 'scroll_element' tool. It is decorated with @mcp.tool(), which registers the tool in the FastMCP server and automatically generates the input schema from the function signature and docstring. The function locates the target UI element by index or name, calculates scroll gesture coordinates within the element's bounds, and executes a swipe gesture using ADB to simulate scrolling.
  @mcp.tool() async def scroll_element(element, direction: str, distance: int = 200, duration: int = 300, device_id: str = None) -> dict: """Scroll a specific UI element in the given direction for a specified distance. Args: element: Either an integer (element index from annotated screenshot) or string (element name) direction: Direction to scroll - 'up', 'down', 'left', 'right' distance: Distance to scroll in pixels (default: 200) duration: Duration of scroll gesture in milliseconds (default: 300) device_id: Optional device ID to target specific device/emulator """ try: # Validate direction parameter direction = direction.lower() if direction not in ['up', 'down', 'left', 'right']: return { "success": False, "error": f'Invalid direction: {direction}. Use "up", "down", "left", or "right"', "element": element } # Validate distance parameter if distance <= 0: return { "success": False, "error": "Distance must be a positive integer", "element": element, "distance": distance } # Get UI elements elements = get_ui_elements(device_id) if not elements: return { "success": False, "error": "No UI elements found on screen", "element": element } # Find target element by index or name target_element = None if isinstance(element, int): # Find by index if 0 <= element < len(elements): target_element = elements[element] else: return { "success": False, "error": f"Element index {element} is out of range (0-{len(elements)-1})", "element": element, "available_count": len(elements) } else: # Find by name element_str = str(element) for elem in elements: if elem.name == element_str: target_element = elem break if not target_element: return { "success": False, "error": f"Element with name '{element_str}' not found", "element": element, "available_elements": [elem.name for elem in elements[:10]] # Show first 10 for reference } # Check if element is likely scrollable scrollable_classes = [ "android.widget.ScrollView", "android.widget.HorizontalScrollView", "android.support.v7.widget.RecyclerView", "androidx.recyclerview.widget.RecyclerView", "android.widget.ListView", "android.widget.GridView", "androidx.viewpager.widget.ViewPager", "androidx.viewpager2.widget.ViewPager2" ] is_scrollable = target_element.class_name in scrollable_classes # Get element bounds bbox = target_element.bounding_box element_width = bbox.x2 - bbox.x1 element_height = bbox.y2 - bbox.y1 # Calculate center point of the element center_x = bbox.x1 + element_width // 2 center_y = bbox.y1 + element_height // 2 # Calculate scroll coordinates within element boundaries with margins margin = 20 # Keep scroll gesture away from element edges if direction == 'up': # Scroll up: start lower in element, end higher start_y = min(center_y + distance // 2, bbox.y2 - margin) end_y = max(center_y - distance // 2, bbox.y1 + margin) start_x = end_x = center_x elif direction == 'down': # Scroll down: start higher in element, end lower start_y = max(center_y - distance // 2, bbox.y1 + margin) end_y = min(center_y + distance // 2, bbox.y2 - margin) start_x = end_x = center_x elif direction == 'left': # Scroll left: start right in element, end left start_x = min(center_x + distance // 2, bbox.x2 - margin) end_x = max(center_x - distance // 2, bbox.x1 + margin) start_y = end_y = center_y else: # right # Scroll right: start left in element, end right start_x = max(center_x - distance // 2, bbox.x1 + margin) end_x = min(center_x + distance // 2, bbox.x2 - margin) start_y = end_y = center_y # Ensure coordinates are within element bounds start_x = max(bbox.x1 + margin, min(start_x, bbox.x2 - margin)) end_x = max(bbox.x1 + margin, min(end_x, bbox.x2 - margin)) start_y = max(bbox.y1 + margin, min(start_y, bbox.y2 - margin)) end_y = max(bbox.y1 + margin, min(end_y, bbox.y2 - margin)) # Build adb swipe command to perform scroll gesture cmd = ['adb'] if device_id: cmd.extend(['-s', device_id]) cmd.extend(['shell', 'input', 'swipe', str(start_x), str(start_y), str(end_x), str(end_y), str(duration)]) # Execute scroll command subprocess.run(cmd, capture_output=True, text=True, check=True) return { "success": True, "message": f"Successfully scrolled element '{target_element.name}' {direction} for {distance} pixels in {duration}ms", "element": { "name": target_element.name, "class_name": target_element.class_name, "bounding_box": { "x1": bbox.x1, "y1": bbox.y1, "x2": bbox.x2, "y2": bbox.y2 } }, "scroll_coordinates": { "start": {"x": start_x, "y": start_y}, "end": {"x": end_x, "y": end_y} }, "direction": direction, "distance": distance, "duration": duration, "is_scrollable_element": is_scrollable, "action_type": f"element_scroll_{direction}", "device_id": device_id or "default" } except ConnectionError as e: return { "success": False, "error": f"Device connection failed: {e}", "element": element, "action_type": "element_scroll" } except subprocess.CalledProcessError as e: return { "success": False, "error": f"Failed to execute scroll: {e}", "stderr": e.stderr if e.stderr else "", "element": element, "action_type": "element_scroll" } except FileNotFoundError: return { "success": False, "error": "ADB not found. Please ensure Android SDK is installed and adb is in PATH.", "element": element, "action_type": "element_scroll" } except Exception as e: return { "success": False, "error": f"Unexpected error: {e}", "element": element, "action_type": "element_scroll" }