List Screen Elements

Instructions

Implementation Reference

• src/server.ts:277-312 (handler)

  Full tool registration including inline handler function that fetches screen elements via Robot interface, formats them with coordinates and properties, and returns JSON string.

  tool(
  	"mobile_list_elements_on_screen",
  	"List elements on screen and their coordinates, with display text or accessibility label. Do not cache this result.",
  	{
  		noParams
  	},
  	async ({}) => {
  		requireRobot();
  		const elements = await robot!.getElementsOnScreen();
  
  		const result = elements.map(element => {
  			const out: any = {
  				type: element.type,
  				text: element.text,
  				label: element.label,
  				name: element.name,
  				value: element.value,
  				identifier: element.identifier,
  				coordinates: {
  					x: element.rect.x,
  					y: element.rect.y,
  					width: element.rect.width,
  					height: element.rect.height,
  				},
  			};
  
  			if (element.focused) {
  				out.focused = true;
  			}
  
  			return out;
  		});
  
  		return `Found these elements on screen: ${JSON.stringify(result)}`;
  	}
  );

• src/robot.ts:26-37 (schema)

  Type definition for ScreenElement used in getElementsOnScreen response, defining structure with type, labels, text, rect coordinates, etc.

  export interface ScreenElement {
  	type: string;
  	label?: string;
  	text?: string;
  	name?: string;
  	value?: string;
  	identifier?: string;
  	rect: ScreenElementRect;
  
  	// currently only on android tv
  	focused?: boolean;
  }

• src/server.ts:277-312 (registration)

  Registration of the tool using the 'tool' helper, with empty input schema (noParams), description, and handler callback.

  tool(
  	"mobile_list_elements_on_screen",
  	"List elements on screen and their coordinates, with display text or accessibility label. Do not cache this result.",
  	{
  		noParams
  	},
  	async ({}) => {
  		requireRobot();
  		const elements = await robot!.getElementsOnScreen();
  
  		const result = elements.map(element => {
  			const out: any = {
  				type: element.type,
  				text: element.text,
  				label: element.label,
  				name: element.name,
  				value: element.value,
  				identifier: element.identifier,
  				coordinates: {
  					x: element.rect.x,
  					y: element.rect.y,
  					width: element.rect.width,
  					height: element.rect.height,
  				},
  			};
  
  			if (element.focused) {
  				out.focused = true;
  			}
  
  			return out;
  		});
  
  		return `Found these elements on screen: ${JSON.stringify(result)}`;
  	}
  );

• src/android.ts:245-250 (helper)

  Android-specific implementation of getElementsOnScreen using UiAutomator XML dump parsing and collectElements helper.

  public async getElementsOnScreen(): Promise<ScreenElement[]> {
  	const parsedXml = await this.getUiAutomatorXml();
  	const hierarchy = parsedXml.hierarchy;
  	const elements = this.collectElements(hierarchy.node);
  	return elements;
  }

• src/android.ts:206-243 (helper)

  Helper function to recursively collect interactive screen elements from UiAutomator XML hierarchy in Android implementation.

  private collectElements(node: UiAutomatorXmlNode): ScreenElement[] {
  	const elements: Array<ScreenElement> = [];
  
  	if (node.node) {
  		if (Array.isArray(node.node)) {
  			for (const childNode of node.node) {
  				elements.push(...this.collectElements(childNode));
  			}
  		} else {
  			elements.push(...this.collectElements(node.node));
  		}
  	}
  
  	if (node.text || node["content-desc"] || node.hint) {
  		const element: ScreenElement = {
  			type: node.class || "text",
  			text: node.text,
  			label: node["content-desc"] || node.hint || "",
  			rect: this.getScreenElementRect(node),
  		};
  
  		if (node.focused === "true") {
  			// only provide it if it's true, otherwise don't confuse llm
  			element.focused = true;
  		}
  
  		const resourceId = node["resource-id"];
  		if (resourceId !== null && resourceId !== "") {
  			element.identifier = resourceId;
  		}
  
  		if (element.rect.width > 0 && element.rect.height > 0) {
  			elements.push(element);
  		}
  	}
  
  	return elements;
  }