Mobile Next MCP Server

Instructions

Implementation Reference

• src/server.ts:395-423 (handler)

  The handler function for the 'mobile_list_elements_on_screen' tool. It fetches the robot instance for the specified device, retrieves the screen elements using getElementsOnScreen(), processes them into a structured JSON format including type, text, label, coordinates, etc., and returns a descriptive string with the JSON data.

  async ({ device }) => {
  	const robot = getRobotFromDevice(device);
  	const elements = await robot.getElementsOnScreen();
  
  	const result = elements.map(element => {
  		const out: any = {
  			type: element.type,
  			text: element.text,
  			label: element.label,
  			name: element.name,
  			value: element.value,
  			identifier: element.identifier,
  			coordinates: {
  				x: element.rect.x,
  				y: element.rect.y,
  				width: element.rect.width,
  				height: element.rect.height,
  			},
  		};
  
  		if (element.focused) {
  			out.focused = true;
  		}
  
  		return out;
  	});
  
  	return `Found these elements on screen: ${JSON.stringify(result)}`;
  }

• src/server.ts:388-424 (registration)

  The registration of the tool using the 'tool()' function, which includes the tool name, title, description, input schema (requiring 'device' string), and the inline handler function.

  tool(
  	"mobile_list_elements_on_screen",
  	"List Screen Elements",
  	"List elements on screen and their coordinates, with display text or accessibility label. Do not cache this result.",
  	{
  		device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
  	},
  	async ({ device }) => {
  		const robot = getRobotFromDevice(device);
  		const elements = await robot.getElementsOnScreen();
  
  		const result = elements.map(element => {
  			const out: any = {
  				type: element.type,
  				text: element.text,
  				label: element.label,
  				name: element.name,
  				value: element.value,
  				identifier: element.identifier,
  				coordinates: {
  					x: element.rect.x,
  					y: element.rect.y,
  					width: element.rect.width,
  					height: element.rect.height,
  				},
  			};
  
  			if (element.focused) {
  				out.focused = true;
  			}
  
  			return out;
  		});
  
  		return `Found these elements on screen: ${JSON.stringify(result)}`;
  	}
  );

• src/server.ts:392-394 (schema)

  The input schema for the tool, defined using Zod, requiring a 'device' string parameter.

  {
  	device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
  },

• src/server.ts:142-179 (helper)

  Helper function that resolves a device ID to the appropriate Robot implementation (IosRobot, AndroidRobot, or MobileDevice), ensuring mobilecli is available and throwing an error if device not found. Used in the tool handler to get the robot instance.

  const getRobotFromDevice = (deviceId: string): Robot => {
  
  	// from now on, we must have mobilecli working
  	ensureMobilecliAvailable();
  
  	// Check if it's an iOS device
  	const iosManager = new IosManager();
  	const iosDevices = iosManager.listDevices();
  	const iosDevice = iosDevices.find(d => d.deviceId === deviceId);
  	if (iosDevice) {
  		return new IosRobot(deviceId);
  	}
  
  	// Check if it's an Android device
  	const androidManager = new AndroidDeviceManager();
  	const androidDevices = androidManager.getConnectedDevices();
  	const androidDevice = androidDevices.find(d => d.deviceId === deviceId);
  	if (androidDevice) {
  		return new AndroidRobot(deviceId);
  	}
  
  	// Check if it's a simulator (will later replace all other device types as well)
  	const response = mobilecli.getDevices({
  		platform: "ios",
  		type: "simulator",
  		includeOffline: false,
  	});
  
  	if (response.status === "ok" && response.data && response.data.devices) {
  		for (const device of response.data.devices) {
  			if (device.id === deviceId) {
  				return new MobileDevice(deviceId);
  			}
  		}
  	}
  
  	throw new ActionableError(`Device "${deviceId}" not found. Use the mobile_list_available_devices tool to see available devices.`);
  };