ui-annotator-mcp

Instructions

Implementation Reference

• src/index.js:60-97 (registration)

  The MCP tool 'get_elements' is registered in src/index.js. It calls proxy.getElements() to retrieve elements and formats the output for the user.

  mcp.tool(
    'get_elements',
    'Get all UI elements detected on the currently annotated page. Returns element names, CSS selectors, positions, and sizes. Use this to understand what the user is referring to when they describe a UI element.',
    {},
    async () => {
      const els = proxy.getElements();
      if (els.length === 0) {
        return {
          content: [{
            type: 'text',
            text: 'No elements detected. Make sure the user has opened the annotated proxy URL in their browser. Use the annotate tool first to get the proxy URL.',
          }],
        };
      }
  
      // Group elements by source type for readability
      const grouped = {};
      for (const el of els) {
        const key = el.source || 'other';
        if (!grouped[key]) grouped[key] = [];
        grouped[key].push(el);
      }
  
      let output = `Found ${els.length} UI elements on the page:\n\n`;
  
      for (const [source, items] of Object.entries(grouped)) {
        output += `## ${source.toUpperCase()} (${items.length})\n`;
        for (const el of items) {
          output += `- **${el.name}** — \`${el.selector}\` (${el.rect.w}×${el.rect.h}px at ${el.rect.x},${el.rect.y})`;
          if (el.text && el.text !== el.name) output += ` "${el.text.slice(0, 50)}"`;
          output += '\n';
        }
        output += '\n';
      }
  
      return { content: [{ type: 'text', text: output }] };
    }
  );

• src/proxy.js:176-176 (handler)

  The underlying implementation of proxy.getElements(), which simply returns the currentElements array in src/proxy.js.

  getElements: () => currentElements,