Android MCP Server

Instructions

Implementation Reference

• src/vision/visual-analyzer.ts:24-51 (handler)

  The logic for capturing the screenshot and UI tree to analyze the screen.

  export async function analyzeScreen(deviceId?: string): Promise<ScreenAnalysis> {
    const resolved = await deviceManager.resolveDeviceId(deviceId);
  
    // Capture screenshot
    const screenshot = await captureScreenshot(resolved, {
      resize: { width: 1080, height: 1920 },
    });
  
    // Try to get UI tree summary (best effort)
    let uiSummary = '';
    try {
      const tree = await getUITree(resolved);
      uiSummary = summarizeTree(tree);
    } catch (error) {
      log.warn('UI tree dump failed during analysis, proceeding with screenshot only', {
        error: error instanceof Error ? error.message : String(error),
      });
    }
  
    const screenSize = await deviceManager.getScreenSize(resolved);
  
    return {
      screenshot,
      uiSummary,
      screenSize,
      deviceInfo: `Screen: ${screenSize.width}x${screenSize.height}`,
    };
  }

• src/controllers/vision-tools.ts:56-89 (registration)

  The MCP tool registration for the 'analyze_screen' tool.

  server.registerTool(
    'analyze_screen',
    {
      description: 'Capture a screenshot and analyze the screen content. Returns both the visual screenshot and a text summary of the UI tree for comprehensive screen understanding. Use this when you need to understand the full screen context.',
      inputSchema: {
        device_id: z.string().optional().describe('Device serial number'),
      },
    },
    async ({ device_id }) => {
      return await metrics.measure('analyze_screen', device_id || 'default', async () => {
        const analysis = await analyzeScreen(device_id);
  
        const content: Array<{ type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string }> = [];
  
        content.push({
          type: 'image' as const,
          data: analysis.screenshot.base64,
          mimeType: 'image/png',
        });
  
        content.push({
          type: 'text' as const,
          text: JSON.stringify({
            success: true,
            screenSize: analysis.screenSize,
            deviceInfo: analysis.deviceInfo,
            uiSummary: analysis.uiSummary,
          }, null, 2),
        });
  
        return { content };
      });
    }
  );