MCP URL Fetcher

Instructions

Implementation Reference

• src/index.ts:99-154 (handler)

  Core handler logic for the 'fetch' tool: fetches the URL, buffers content, detects type, converts to specified format using helper converters, records the fetch, and returns formatted content or error response.

  async ({ url, format = "auto" }) => {
    try {
      const response = await fetchUrl(url);
      const contentBuffer = await response.buffer();
      const contentText = contentBuffer.toString();
      const detectedType = detectContentType(response, url);
  
      // If format is auto, use the detected type or default to text
      let outputFormat = format;
      if (format === "auto") {
        outputFormat = detectedType;
      }
  
      let processedContent;
  
      // Convert to the desired output format
      switch (outputFormat) {
        case "json":
          processedContent = await convertToJson(contentText, detectedType, url);
          break;
        case "markdown":
          processedContent = await convertToMarkdown(contentText, detectedType, url);
          break;
        case "html":
          processedContent = await convertToHtml(contentText, detectedType, url);
          break;
        case "text":
        default:
          processedContent = await convertToText(contentText, detectedType, url);
          outputFormat = "text";
          break;
      }
  
      // Record this fetch
      recordUrlFetch(url, outputFormat);
  
      return {
        content: [
          {
            type: "text",
            text: `# Content from ${url} converted to ${outputFormat}:\n\n${processedContent}`,
          },
        ],
      };
    } catch (error) {
      return {
        isError: true,
        content: [
          {
            type: "text",
            text: `Error fetching content from URL: ${error instanceof Error ? error.message : String(error)}`,
          },
        ],
      };
    }
  }

• src/index.ts:95-98 (schema)

  Zod input schema for the 'fetch' tool, validating 'url' as a string URL and optional 'format' enum.

  {
    url: z.string().url().describe("URL to fetch content from"),
    format: z.enum(["auto", "html", "json", "markdown", "text"]).optional().describe("Format to convert to (default: auto)"),
  },

• src/index.ts:92-94 (registration)

  Registration of the 'fetch' tool on the MCP server with name and description.

  server.tool(
    "fetch",
    "Fetch content from a URL with automatic content type detection",

• src/index.ts:40-51 (helper)

  Helper function that performs the actual HTTP fetch using node-fetch, handles errors, and returns the response.

  async function fetchUrl(url: string) {
    try {
      const response = await fetch(url);
      if (!response.ok) {
        throw new Error(`HTTP error! status: ${response.status}`);
      }
      return response;
    } catch (error) {
      console.error(`Error fetching URL: ${url}`, error);
      throw error;
    }
  }

• src/index.ts:54-73 (helper)

  Helper function to detect the content type based on response headers or URL extension for automatic format selection.

  function detectContentType(response, url: string): string {
    const contentType = response.headers.get("content-type") || "";
  
    // Check based on content-type header
    if (contentType.includes("json")) return "json";
    if (contentType.includes("html")) return "html";
    if (contentType.includes("markdown") || contentType.includes("md")) return "markdown";
    if (contentType.includes("xml")) return "xml";
    if (contentType.includes("csv")) return "csv";
  
    // If no clear content-type, check the URL extension
    if (url.endsWith(".json")) return "json";
    if (url.endsWith(".html") || url.endsWith(".htm")) return "html";
    if (url.endsWith(".md") || url.endsWith(".markdown")) return "markdown";
    if (url.endsWith(".xml")) return "xml";
    if (url.endsWith(".csv")) return "csv";
  
    // Default to text
    return "text";
  }