Android MCP Server

Instructions

Implementation Reference

• src/controllers/vision-tools.ts:126-174 (handler)

  The MCP tool 'compare_screenshots' is registered here. It handles input validation, retrieves the previous screenshot from memory, captures the current one, and invokes the comparison logic.

  server.registerTool(
    'compare_screenshots',
    {
      description: 'Compare the current screen with the previously captured screenshot to detect changes. Returns the percentage of pixels that changed and whether the screen has significantly changed. Useful for verifying that an action had an effect.',
      inputSchema: {
        threshold: z.number().optional().default(5).describe('Minimum change percentage to be considered "changed" (default: 5)'),
        device_id: z.string().optional().describe('Device serial number'),
      },
    },
    async ({ threshold, device_id }) => {
      return await metrics.measure('compare_screenshots', device_id || 'default', async () => {
        const deviceKey = device_id || 'default';
        const previousBuffer = lastScreenshots.get(deviceKey);
  
        if (!previousBuffer) {
          return {
            content: [{
              type: 'text' as const,
              text: JSON.stringify({
                success: false,
                error: 'No previous screenshot to compare with. Call capture_screenshot first.',
              }, null, 2),
            }],
          };
        }
  
        const currentBuffer = await captureScreenshotBuffer(device_id);
        const diff = await compareScreenshots(previousBuffer, currentBuffer, threshold);
  
        // Update stored screenshot
        lastScreenshots.set(deviceKey, currentBuffer);
  
        return {
          content: [{
            type: 'text' as const,
            text: JSON.stringify({
              success: true,
              diff: {
                changePercentage: diff.changePercentage,
                hasChanged: diff.hasChanged,
                totalPixels: diff.totalPixels,
                changedPixels: diff.changedPixels,
              },
            }, null, 2),
          }],
        };
      });
    }
  );

• src/vision/screen-diff.ts:33-83 (helper)

  The core logic for comparing two screenshot buffers. It resizes images to a standard size, compares pixels raw, and calculates the change percentage.

  export async function compareScreenshots(
    buffer1: Buffer,
    buffer2: Buffer,
    threshold: number = 5 // minimum change % to be considered "changed"
  ): Promise<DiffResult> {
    // Normalize both images to same size and raw format
    const size = { width: 360, height: 640 }; // Downscale for fast comparison
  
    const [raw1, raw2] = await Promise.all([
      sharp(buffer1)
        .resize(size.width, size.height, { fit: 'fill' })
        .raw()
        .toBuffer(),
      sharp(buffer2)
        .resize(size.width, size.height, { fit: 'fill' })
        .raw()
        .toBuffer(),
    ]);
  
    const totalPixels = size.width * size.height;
    let changedPixels = 0;
  
    // Compare pixel by pixel (each pixel = 3 bytes for RGB)
    const bytesPerPixel = 3;
    const pixelThreshold = 30; // Per-channel difference threshold
  
    for (let i = 0; i < totalPixels; i++) {
      const offset = i * bytesPerPixel;
      const dr = Math.abs(raw1[offset] - raw2[offset]);
      const dg = Math.abs(raw1[offset + 1] - raw2[offset + 1]);
      const db = Math.abs(raw1[offset + 2] - raw2[offset + 2]);
  
      if (dr > pixelThreshold || dg > pixelThreshold || db > pixelThreshold) {
        changedPixels++;
      }
    }
  
    const changePercentage = (changedPixels / totalPixels) * 100;
    const screenHash = computeScreenHash(buffer2);
  
    const result: DiffResult = {
      changePercentage: Math.round(changePercentage * 100) / 100,
      hasChanged: changePercentage >= threshold,
      screenHash,
      totalPixels,
      changedPixels,
    };
  
    log.debug('Screen diff computed', result);
    return result;
  }