Pilot

Instructions

Implementation Reference

• src/tools/snapshot-tools.ts:65-92 (handler)

  Handler for the pilot_annotated_screenshot tool, which calls annotateScreenshot and returns the result as an image.

  server.tool(
    'pilot_annotated_screenshot',
    'Take a screenshot with red overlay boxes and ref labels at each @eN/@cN position. Requires a prior pilot_snapshot call.',
    {
      output_path: z.string().optional().describe('Output file path for the screenshot'),
    },
    async ({ output_path }) => {
      await bm.ensureBrowser();
      try {
        const screenshotPath = await annotateScreenshot(bm, output_path);
        bm.resetFailures();
  
        // Read the image and return as base64
        const imageData = fs.readFileSync(screenshotPath);
        const base64 = imageData.toString('base64');
  
        return {
          content: [
            { type: 'text' as const, text: `Annotated screenshot saved: ${screenshotPath}` },
            { type: 'image' as const, data: base64, mimeType: 'image/png' },
          ],
        };
      } catch (err) {
        bm.incrementFailures();
        return { content: [{ type: 'text' as const, text: wrapError(err) }], isError: true };
      }
    }
  );