fetch
Fetch and process web content from any URL with automatic format detection, supporting HTML, JSON, Markdown, and text for efficient data extraction and integration.
Instructions
Fetch content from a URL with automatic content type detection
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| format | No | Format to convert to (default: auto) | |
| url | Yes | URL to fetch content from |
Implementation Reference
- src/index.ts:99-154 (handler)Core handler logic for the 'fetch' tool: fetches the URL, buffers content, detects type, converts to specified format using helper converters, records the fetch, and returns formatted content or error response.async ({ url, format = "auto" }) => { try { const response = await fetchUrl(url); const contentBuffer = await response.buffer(); const contentText = contentBuffer.toString(); const detectedType = detectContentType(response, url); // If format is auto, use the detected type or default to text let outputFormat = format; if (format === "auto") { outputFormat = detectedType; } let processedContent; // Convert to the desired output format switch (outputFormat) { case "json": processedContent = await convertToJson(contentText, detectedType, url); break; case "markdown": processedContent = await convertToMarkdown(contentText, detectedType, url); break; case "html": processedContent = await convertToHtml(contentText, detectedType, url); break; case "text": default: processedContent = await convertToText(contentText, detectedType, url); outputFormat = "text"; break; } // Record this fetch recordUrlFetch(url, outputFormat); return { content: [ { type: "text", text: `# Content from ${url} converted to ${outputFormat}:\n\n${processedContent}`, }, ], }; } catch (error) { return { isError: true, content: [ { type: "text", text: `Error fetching content from URL: ${error instanceof Error ? error.message : String(error)}`, }, ], }; } }
- src/index.ts:95-98 (schema)Zod input schema for the 'fetch' tool, validating 'url' as a string URL and optional 'format' enum.{ url: z.string().url().describe("URL to fetch content from"), format: z.enum(["auto", "html", "json", "markdown", "text"]).optional().describe("Format to convert to (default: auto)"), },
- src/index.ts:92-94 (registration)Registration of the 'fetch' tool on the MCP server with name and description.server.tool( "fetch", "Fetch content from a URL with automatic content type detection",
- src/index.ts:40-51 (helper)Helper function that performs the actual HTTP fetch using node-fetch, handles errors, and returns the response.async function fetchUrl(url: string) { try { const response = await fetch(url); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } return response; } catch (error) { console.error(`Error fetching URL: ${url}`, error); throw error; } }
- src/index.ts:54-73 (helper)Helper function to detect the content type based on response headers or URL extension for automatic format selection.function detectContentType(response, url: string): string { const contentType = response.headers.get("content-type") || ""; // Check based on content-type header if (contentType.includes("json")) return "json"; if (contentType.includes("html")) return "html"; if (contentType.includes("markdown") || contentType.includes("md")) return "markdown"; if (contentType.includes("xml")) return "xml"; if (contentType.includes("csv")) return "csv"; // If no clear content-type, check the URL extension if (url.endsWith(".json")) return "json"; if (url.endsWith(".html") || url.endsWith(".htm")) return "html"; if (url.endsWith(".md") || url.endsWith(".markdown")) return "markdown"; if (url.endsWith(".xml")) return "xml"; if (url.endsWith(".csv")) return "csv"; // Default to text return "text"; }