fetch
Retrieve web content as markdown, including image URLs. Extract specific sections or raw data from any URL using customizable parameters and proxy support.
Instructions
Retrieves URLs from the Internet and extracts their content as markdown. If images are found, their URLs will be included in the response.
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| maxLength | No | ||
| raw | No | ||
| startIndex | No | ||
| url | Yes |
Implementation Reference
- index.ts:183-248 (handler)The request handler for 'tools/call' which dispatches to the 'fetch' tool implementation: validates input with FetchArgsSchema, calls fetchUrl, truncates content if needed, appends image list, returns formatted text content.server.setRequestHandler( CallToolSchema, async ( request: { method: "tools/call" params: { name: string; arguments?: Record<string, unknown> } }, extra: RequestHandlerExtra, ) => { try { const { name, arguments: args } = request.params if (name !== "fetch") { throw new Error(`Unknown tool: ${name}`) } const parsed = FetchArgsSchema.safeParse(args) if (!parsed.success) { throw new Error(`Invalid arguments: ${parsed.error}`) } const { content, prefix, imageUrls } = await fetchUrl( parsed.data.url, DEFAULT_USER_AGENT_AUTONOMOUS, parsed.data.raw, ) let finalContent = content if (finalContent.length > parsed.data.maxLength) { finalContent = finalContent.slice( parsed.data.startIndex, parsed.data.startIndex + parsed.data.maxLength, ) finalContent += `\n\n<e>Content truncated. Call the fetch tool with a start_index of ${ parsed.data.startIndex + parsed.data.maxLength } to get more content.</e>` } let imagesSection = "" if (imageUrls && imageUrls.length > 0) { imagesSection = "\n\nImages found in article:\n" + imageUrls.map((url) => `- ${url}`).join("\n") } return { content: [ { type: "text", text: `${prefix}Contents of ${parsed.data.url}:\n${finalContent}${imagesSection}`, }, ], } } catch (error) { return { content: [ { type: "text", text: `Error: ${error instanceof Error ? error.message : String(error)}`, }, ], isError: true, } } }, )
- index.ts:41-46 (schema)Zod schema for 'fetch' tool input: url (required), optional maxLength, startIndex for pagination, raw flag.const FetchArgsSchema = z.object({ url: z.string().url(), maxLength: z.number().positive().max(1000000).default(20000), startIndex: z.number().min(0).default(0), raw: z.boolean().default(false), })
- index.ts:168-180 (registration)Registration of the 'fetch' tool via the 'tools/list' handler, providing name, description, and input schema.server.setRequestHandler( ListToolsSchema, async (request: { method: "tools/list" }, extra: RequestHandlerExtra) => { const tools = [ { name: "fetch", description: "Retrieves URLs from the Internet and extracts their content as markdown. If images are found, their URLs will be included in the response.", inputSchema: zodToJsonSchema(FetchArgsSchema), }, ] return { tools } },
- index.ts:108-149 (helper)Core helper function for fetching URL content: uses proxy-aware fetch, detects HTML, extracts readable markdown and images via extractContentFromHtml, or returns raw with prefix.async function fetchUrl( url: string, userAgent: string, forceRaw = false, ): Promise<FetchResult> { const response = await fetch(url, { headers: { "User-Agent": userAgent }, }) if (!response.ok) { throw new Error(`Failed to fetch ${url} - status code ${response.status}`) } const contentType = response.headers.get("content-type") || "" const text = await response.text() const isHtml = text.toLowerCase().includes("<html") || contentType.includes("text/html") if (isHtml && !forceRaw) { const result = extractContentFromHtml(text, url) if (typeof result === "string") { return { content: result, prefix: "", } } const { markdown, images } = result const imageUrls = images.map((img) => img.src) return { content: markdown, prefix: "", imageUrls, } } return { content: text, prefix: `Content type ${contentType} cannot be simplified to markdown, but here is the raw content:\n`, } }
- index.ts:60-91 (helper)Helper to convert HTML to markdown and extract images: uses Mozilla Readability for article extraction, Turndown for markdown, JSDOM for image parsing.function extractContentFromHtml( html: string, url: string, ): ExtractedContent | string { const dom = new JSDOM(html, { url }) const reader = new Readability(dom.window.document) const article = reader.parse() if (!article || !article.content) { return "<e>Page failed to be simplified from HTML</e>" } // Extract images from the article content only const articleDom = new JSDOM(article.content) const imgElements = Array.from( articleDom.window.document.querySelectorAll("img"), ) const images: Image[] = imgElements.map((img) => { const src = img.src const alt = img.alt || "" return { src, alt } }) const turndownService = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", }) const markdown = turndownService.turndown(article.content) return { markdown, images } }