Web-curl MCP Server

Instructions

Implementation Reference

• src/index.ts:619-738 (handler)

  Core handler function that launches Puppeteer browser, navigates to URL, extracts and cleans HTML content, handles slicing based on startIndex and maxLength, supports pagination, resource blocking, authentication, headers, and script evaluation.

  private async fetchWebpage(url: string, options: {
      blockResources: boolean;
      resourceTypesToBlock?: string[];
      timeout: number;
      maxLength: number;
      startIndex: number;
      headers?: Record<string, string>;
      username?: string;
      password?: string;
      nextPageSelector?: string;
      maxPages: number;
      evaluateScript?: string; // Add evaluateScript to options
  }) {
      const browser = await puppeteer.launch({
          headless: true,
          args: ['--incognito', '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
      });
  
      let page;
      try {
          page = await browser.newPage();
          await page.setDefaultNavigationTimeout(options.timeout);
  
          if (options.headers) {
              await page.setExtraHTTPHeaders(options.headers);
          }
  
          if (options.username && options.password) {
              await page.authenticate({ username: options.username, password: options.password });
          }
  
          if (options.blockResources || options.resourceTypesToBlock) {
              await page.setRequestInterception(true);
              page.on('request', req => {
                  const resourceType = req.resourceType();
                  const typesToBlock = options.resourceTypesToBlock ?? [];
                  if (typesToBlock.includes(resourceType)) {
                      req.abort();
                  } else {
                      req.continue();
                  }
              });
          }
  
          const results: any[] = [];
          let currentUrl = url;
          let currentPageNum = 1;
  
          while (currentPageNum <= options.maxPages) {
              console.error(`Fetching content from: ${currentUrl} (page ${currentPageNum})`);
              await page.goto(currentUrl, { waitUntil: 'domcontentloaded', timeout: options.timeout });
  
              const title = await page.title();
  
              // Efficiently extract and slice content within the browser context
              // 1. Get the full content from the page
              const html = await page.content();
  
              // Step 1: Remove all whitespace from the raw HTML content
              const cleanedHtml = html.replace(/\s/g, '');
  
              // Step 2: Calculate the total character count of the cleaned HTML
              const totalLength = cleanedHtml.length;
  
              // Step 3: Slice the cleaned HTML based on startIndex and maxLength
              const slicedContent = cleanedHtml.substring(options.startIndex, options.startIndex + options.maxLength);
  
              let scriptResult: any = null; // Declare scriptResult here with explicit 'any' type
              if (options.evaluateScript) {
                  try {
                      scriptResult = await page.evaluate(options.evaluateScript);
                  } catch (scriptError: any) {
                      scriptResult = `Error executing script: ${scriptError.message}`;
                  }
              }
  
              results.push({
                  url: currentUrl,
                  title,
                  content: slicedContent, // Renamed from slicedContent for consistency with tool handler
                  contentLength: totalLength, // Renamed from totalCharacters for consistency with tool handler
                  fetchedAt: new Date().toISOString(),
                  evaluatedScriptResult: scriptResult, // Include the script result
              });
              console.log(`DEBUG: totalLength (after whitespace removal): ${totalLength}, startIndex: ${options.startIndex}, slicedContent length: ${slicedContent.length}`);
              
              if (!options.nextPageSelector || currentPageNum >= options.maxPages) {
                  break;
              }
  
              // Pagination logic
              const nextHref = await page.evaluate((sel) => {
                  const el = document.querySelector(sel) as HTMLAnchorElement | HTMLElement;
                  if (el) {
                      if ('href' in el && (el as HTMLAnchorElement).href) {
                          return (el as HTMLAnchorElement).href;
                      }
                      el.click(); // Fallback for non-anchor elements
                  }
                  return null;
              }, options.nextPageSelector);
  
              if (nextHref && nextHref !== currentUrl) {
                  currentUrl = nextHref;
                  currentPageNum++;
              } else {
                  break; // No more pages
              }
          }
  
          return results.length === 1 ? results[0] : results;
      } catch (err) {
          throw err;
      } finally {
          // Always close browser to prevent temp file leaks
          if (browser) {
            try { await browser.close(); } catch (e) {}
          }
      }
  }

• src/index.ts:278-390 (handler)

  MCP CallToolRequest handler dispatch for fetch_webpage: validates arguments, maps legacy params, enforces required params, calls fetchWebpage, processes results with slicing info and pagination instructions, handles errors.

  if (toolName === 'fetch_webpage') {
    // Cast args to a more specific type after validation
    if (!isValidFetchWebpageArgs(args)) {
        throw new McpError(ErrorCode.InvalidParams, 'Invalid fetch_webpage arguments');
    }
  
    // Map legacy/alias params to current names:
    // - 'index' -> startIndex
    try {
      if (typeof (args as any).index === 'number' && typeof (args as any).startIndex !== 'number') {
        (args as any).startIndex = (args as any).index;
      }
    } catch (e) {
      // ignore mapping errors; validation below will catch invalid shapes
    }
  
    // Enforce required parameters (runtime validation).
    const hasStartIndex = typeof (args as any).startIndex === 'number';
    const hasMaxLength = typeof (args as any).maxLength === 'number';
  
    if (!hasStartIndex) {
      throw new McpError(ErrorCode.InvalidParams, "fetch_webpage: required parameter 'startIndex' (or alias 'index') is missing or not a number");
    }
    if (!hasMaxLength) {
      throw new McpError(ErrorCode.InvalidParams, "fetch_webpage: required parameter 'maxLength' is missing or not a number");
    }
  
    const validatedArgs = args as FetchWebpageArgs & {
        nextPageSelector?: string;
        maxPages?: number;
    };
  
    const {
        url,
        blockResources = false,
        resourceTypesToBlock,
        timeout: rawTimeout,
        maxLength = 4000,
        startIndex = 0,
        headers,
        username,
        password,
        nextPageSelector,
        maxPages = 1,
        evaluateScript // Destructure evaluateScript
    } = validatedArgs;
  
    const timeout = Math.min(rawTimeout || 60000, 300000);
  
    // Helper function to perform the fetch and process the result
    const performFetchAndProcess = async (currentArgs: typeof validatedArgs) => {
        const result: any = await this.fetchWebpage(currentArgs.url, {
            blockResources: currentArgs.blockResources ?? false, // Provide default
            resourceTypesToBlock: currentArgs.resourceTypesToBlock,
            timeout: currentArgs.timeout ?? 30000, // Provide default
            maxLength: currentArgs.maxLength ?? 4000, // Provide default
            startIndex: currentArgs.startIndex ?? 0, // Provide default
            headers: currentArgs.headers,
            username: currentArgs.username,
            password: currentArgs.password,
            nextPageSelector: currentArgs.nextPageSelector,
            maxPages: currentArgs.maxPages ?? 1, // Provide default
            evaluateScript: currentArgs.evaluateScript // Pass evaluateScript
        });
  
        const totalCharacters = result.contentLength; // total characters of the whitespace-removed HTML
        const slicedContentLength = result.content.length; // length of the sliced part
  
        const remainingCharacters = Math.max(0, totalCharacters - (currentArgs.startIndex + slicedContentLength));
        const nextStartIndex = currentArgs.startIndex + slicedContentLength; // The next start index should be right after the current sliced content
  
        const instruction = remainingCharacters === 0
            ? "All content has been fetched."
            : `To fetch more content, call fetch_webpage again with startIndex=${nextStartIndex}. If you have enough information, you can stop.`;
  
        const finalResponse = {
            url: result.url,
            title: result.title,
            content: result.content, // This is the sliced, whitespace-removed HTML
            fetchedAt: result.fetchedAt,
            startIndex: currentArgs.startIndex,
            maxLength: currentArgs.maxLength,
            remainingCharacters: remainingCharacters,
            instruction: instruction,
            evaluatedScriptResult: result.evaluatedScriptResult, // Include the script result
        };
  
        return {
            content: [{ type: 'text', text: JSON.stringify(finalResponse, null, 2) }],
        };
    };
  
    try {
        return await performFetchAndProcess(validatedArgs);
    } catch (error: any) { // Catch block for fetch_webpage
        let errorMessage = `Error fetching webpage: ${error.message}`;
  
        // Check for timeout error and suggest disabling blockResources
        // Removed automatic retry logic as blockResources is now always false.
        // The default behavior is now to not block resources.
  
        // If still a timeout, you might consider a generic message or further options.
        // For now, it will just return the original error message.
  
        // Note: The previous logic for suggesting 'blockResources: false' is removed
        // because blockResources is now forced to false by default for all calls.
  
        console.error('Error fetching webpage:', error);
        return {
            content: [{ type: 'text', text: `Error fetching webpage: ${errorMessage}` }],
            isError: true,
        };
    }

• src/index.ts:106-168 (registration)

  Tool registration in ListToolsRequest handler, defining name, description, and detailed JSON inputSchema with properties, required fields, and usage description.

  {
    name: 'fetch_webpage',
    description: 'Retrieve text content from a web page',
    inputSchema: {
      type: 'object',
      properties: {
        url: {
          type: 'string',
          description: 'The URL of the webpage to fetch'
        },
        blockResources: {
          type: 'boolean',
          description: 'Whether to block images, stylesheets, and fonts to improve performance (default: true)'
        },
        resourceTypesToBlock: {
          type: 'array',
          items: {
            type: 'string'
          },
          description: 'List of resource types to block (e.g., "image", "stylesheet", "font")'
        },
        timeout: {
          type: 'number',
          description: 'Navigation timeout in milliseconds (default: 120000)'
        },
        maxLength: {
          type: 'number',
          description: 'Maximum number of characters to return (default: 10000).'
        },
        startIndex: {
          type: 'number',
          description: 'Start character index for content extraction (required; default: 0).'
        },
        headers: {
          type: 'object',
          description: 'Custom headers to include in the request'
        },
        username: {
          type: 'string',
          description: 'Username for basic authentication'
        },
        password: {
          type: 'string',
          description: 'Password for basic authentication'
        },
        nextPageSelector: {
          type: 'string',
          description: 'CSS selector for next page button/link (for auto-pagination, optional)'
        },
        maxPages: {
          type: 'number',
          description: 'Maximum number of pages to crawl (for auto-pagination, optional, default: 1)'
        },
        evaluateScript: { // Add evaluateScript to the input schema
          type: 'string',
          description: 'JavaScript code to execute on the page after loading. The result of the script will be returned.'
        }
      },
      required: ['url', 'startIndex'],
      additionalProperties: false,
      description: 'Fetch web content (text, html, mainContent, metadata, supports multi-page crawling, and AI-friendly regex extraction). Debug option for verbose output/logging.'
    },
  },

• src/index.ts:23-37 (schema)

  TypeScript interface defining the structure of arguments for the fetch_webpage tool.

  // Define the interface for the fetch_webpage tool arguments
  interface FetchWebpageArgs {
    url: string;
    blockResources?: boolean;
    resourceTypesToBlock?: string[];
    timeout?: number;
    maxLength?: number;
   startIndex?: number;
   nextPageSelector?: string;
   maxPages?: number;
   headers?: Record<string, string>;
   username?: string;
   password?: string;
   evaluateScript?: string; // Add evaluateScript parameter
  }

• src/index.ts:59-71 (schema)

  Runtime validation function for fetch_webpage arguments, type-guarding to FetchWebpageArgs interface.

  // Validate the arguments for fetch_webpage tool
  const isValidFetchWebpageArgs = (args: any): args is FetchWebpageArgs =>
    typeof args === 'object' &&
    args !== null &&
    typeof args.url === 'string' &&
    (args.blockResources === undefined || typeof args.blockResources === 'boolean') &&
    (args.resourceTypesToBlock === undefined || (Array.isArray(args.resourceTypesToBlock) && args.resourceTypesToBlock.every((item: any) => typeof item === 'string'))) &&
    (args.timeout === undefined || typeof args.timeout === 'number') &&
    (args.maxLength === undefined || typeof args.maxLength === 'number') &&
    (args.startIndex === undefined || typeof args.startIndex === 'number') &&
    (args.headers === undefined || (typeof args.headers === 'object' && args.headers !== null && !Array.isArray(args.headers))) &&
    (args.username === undefined || typeof args.username === 'string') &&
    (args.password === undefined || typeof args.password === 'string');