Robot Resources Scraper

Instructions

Implementation Reference

• src/server.ts:93-130 (handler)

  The 'compressUrl' function is the actual implementation/handler for the 'scraper_compress_url' tool. It fetches the content, extracts it, and converts it to markdown.

  export async function compressUrl({
    url,
    mode,
    timeout,
    maxRetries,
  }: {
    url: string;
    mode?: FetchMode;
    timeout?: number;
    maxRetries?: number;
  }) {
    try {
      const fetchResult = await fetchWithMode(url, mode ?? 'auto', { timeout, maxRetries });
      const originalTokens = estimateTokens(fetchResult.html);
      const extractResult = await extractContent(fetchResult);
      const convertResult = await convertToMarkdown(extractResult);
  
      const compressionRatio =
        originalTokens > 0
          ? Math.round((1 - convertResult.tokenCount / originalTokens) * 100)
          : 0;
  
      return {
        content: [{ type: 'text' as const, text: convertResult.markdown }],
        structuredContent: {
          markdown: convertResult.markdown,
          tokenCount: convertResult.tokenCount,
          title: extractResult.title ?? null,
          author: extractResult.author ?? null,
          siteName: extractResult.siteName ?? null,
          url: fetchResult.url,
          compressionRatio,
        },
      };
    } catch (error) {
      return formatError(url, error);
    }
  }

• src/server.ts:25-48 (registration)

  Registration of the 'scraper_compress_url' tool in the McpServer.

  server.tool(
    'scraper_compress_url',
    'Compress web content from a URL for reduced token usage. Returns markdown with 70-90% fewer tokens than raw HTML.',
    {
      url: z.string().url().describe('URL to compress'),
      mode: z
        .enum(['fast', 'stealth', 'render', 'auto'])
        .optional()
        .describe("Fetch mode: 'fast' (plain HTTP), 'stealth' (TLS fingerprint), 'render' (headless browser), 'auto' (fast with fallback). Default: 'auto'"),
      timeout: z
        .number()
        .positive()
        .optional()
        .describe('Fetch timeout in milliseconds (default: 10000)'),
      maxRetries: z
        .number()
        .int()
        .min(0)
        .max(10)
        .optional()
        .describe('Max retry attempts (default: 3)'),
    },
    async (args) => compressUrl(args),
  );