app-screen-mcp

Instructions

Implementation Reference

• src/index.ts:672-715 (handler)

  The 'getScreenSummary' method acts as the handler for the 'get_screen_summary' MCP tool. It resolves the target device, fetches the UI tree, and optionally captures a screenshot based on provided parameters.

  private async getScreenSummary(
    udid?: string,
    includeImage?: boolean,
    maxDimInput?: number,
    qualityInput?: number,
    onlyIfChanged?: boolean,
    previousImageHash?: string,
    compactTree?: boolean
  ) {
    const target = await resolveUdid(udid);
    const shouldIncludeImage = includeImage !== false;
    const options = normalizeScreenshotOptions(maxDimInput, qualityInput);
  
    const promises: Promise<any>[] = [this.fetchUiTree(target)];
    if (shouldIncludeImage) {
      promises.push(this.captureScreenshot(target, options));
    }
  
    const settled = await Promise.allSettled(promises);
    const treeResult = settled[0];
    const screenshotResult = shouldIncludeImage ? settled[1] : undefined;
  
    const meta: Record<string, any> = {
      timestamp: Date.now(),
      udid: target,
      include_image: shouldIncludeImage,
      compact_tree: compactTree === true,
    };
  
    if (treeResult.status === 'fulfilled') {
      const elements = treeResult.value as UIElement[];
      meta.element_count = elements.length;
      meta.elements = compactTree === true ? this.compactElements(elements) : elements;
    } else {
      meta.ui_tree_error = String(treeResult.reason?.message ?? treeResult.reason);
    }
  
    const content: any[] = [{ type: 'text', text: JSON.stringify(meta, null, 2) }];
  
    if (shouldIncludeImage && screenshotResult) {
      if (screenshotResult.status === 'fulfilled') {
        const shot = screenshotResult.value as ScreenshotPayload;
        const compareHash = previousImageHash ?? (
          onlyIfChanged ? this.lastScreenshotHashByUdid.get(target) : undefined

• src/index.ts:351-368 (registration)

  The 'get_screen_summary' tool is defined and registered with its schema (description and input parameters) within the MCP tool list.

    name: 'get_screen_summary',
    description: 'Get screen context (UI tree and optional screenshot) with token-saving options.',
    inputSchema: {
      type: 'object',
      properties: {
        udid: { type: 'string', description: 'Simulator UDID (optional, defaults to booted simulator)' },
        include_image: { type: 'boolean', description: 'Include screenshot image content (default: true)' },
        max_dim: { type: 'number', description: 'Max width/height for output image (default: 960)' },
        quality: { type: 'number', description: 'JPEG quality 20..95 (default: 60)' },
        only_if_changed: { type: 'boolean', description: 'Do not return image content if hash is unchanged' },
        previous_image_hash: { type: 'string', description: 'Compare against this hash for unchanged detection' },
        compact_tree: { type: 'boolean', description: 'Return compact tree format for fewer tokens' },
      },
      additionalProperties: false,
    },
  },
  
  // Interaction