fetch_url
Fetch web content from URLs with configurable response formats including text, JSON, HTML, and Markdown, while handling errors and timeouts.
Instructions
Fetch content from a URL with proper error handling and response processing
Input Schema
TableJSON Schema
| Name | Required | Description | Default |
|---|---|---|---|
| url | Yes | The URL to fetch | |
| responseType | No | Expected response type | text |
| timeout | No | Request timeout in milliseconds |
Implementation Reference
- src/url-fetcher.ts:280-372 (handler)The core handler logic for the 'fetch_url' tool. It implements URL fetching with AbortController for timeout, browser-like headers, retry logic for failures and rate limits, response processing via processResponse helper, and returns MCP-compatible content structure with metadata.
async (params) => { for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), params.timeout); const response = await fetch(params.url.toString(), { signal: controller.signal, headers: BROWSER_HEADERS, redirect: 'follow' }); clearTimeout(timeout); // Handle different status codes if (!response.ok) { if (response.status === 429) { if (attempt === MAX_RETRIES - 1) { return { content: [{ type: "text", text: "Rate limit exceeded. Please try again later." }], isError: true }; } await new Promise(resolve => setTimeout(resolve, getRetryDelay(attempt))); continue; } return { content: [{ type: "text", text: `HTTP ${response.status}: ${response.statusText}` }], isError: true }; } // Process the response const processedContent = await processResponse(response, params.responseType, params.url); // Always return as text type with appropriate metadata return { content: [{ type: "text", text: processedContent, mimeType: params.responseType === 'json' ? 'application/json' : params.responseType === 'markdown' ? 'text/markdown' : params.responseType === 'html' ? 'text/html' : 'text/plain' }], metadata: { url: params.url.toString(), contentType: response.headers.get('content-type'), contentLength: response.headers.get('content-length'), isGoogleSearch: params.url.origin + params.url.pathname === GOOGLE_SEARCH_URL, responseType: params.responseType } }; } catch (error) { if (error instanceof Error && error.name === 'AbortError') { return { content: [{ type: "text", text: `Request timed out after ${params.timeout}ms` }], isError: true }; } if (attempt === MAX_RETRIES - 1) { return { content: [{ type: "text", text: `Failed to fetch URL: ${error instanceof Error ? error.message : 'Unknown error'}` }], isError: true }; } await new Promise(resolve => setTimeout(resolve, getRetryDelay(attempt))); } } return { content: [{ type: "text", text: "Failed to fetch URL after all retry attempts" }], isError: true }; } - src/url-fetcher.ts:46-59 (schema)Zod validation schema for 'fetch_url' tool parameters, used in server.tool registration.
const UrlFetcherSchema = z.object({ url: z.string() .url() .transform(url => new URL(url)) .describe("The URL to fetch"), responseType: z.enum(['text', 'json', 'html', 'markdown']) .default('text') .describe("Expected response type"), timeout: z.number() .min(1000) .max(60000) .default(DEFAULT_TIMEOUT) .describe("Request timeout in milliseconds") }); - src/url-fetcher.ts:275-374 (registration)Function that registers the 'fetch_url' tool on the MCP server instance using server.tool(), specifying name, description, input schema, and handler.
export function registerUrlFetcherTool(server: McpServer) { server.tool( "fetch_url", "Fetch content from a URL with proper error handling and response processing", UrlFetcherSchema.shape, async (params) => { for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), params.timeout); const response = await fetch(params.url.toString(), { signal: controller.signal, headers: BROWSER_HEADERS, redirect: 'follow' }); clearTimeout(timeout); // Handle different status codes if (!response.ok) { if (response.status === 429) { if (attempt === MAX_RETRIES - 1) { return { content: [{ type: "text", text: "Rate limit exceeded. Please try again later." }], isError: true }; } await new Promise(resolve => setTimeout(resolve, getRetryDelay(attempt))); continue; } return { content: [{ type: "text", text: `HTTP ${response.status}: ${response.statusText}` }], isError: true }; } // Process the response const processedContent = await processResponse(response, params.responseType, params.url); // Always return as text type with appropriate metadata return { content: [{ type: "text", text: processedContent, mimeType: params.responseType === 'json' ? 'application/json' : params.responseType === 'markdown' ? 'text/markdown' : params.responseType === 'html' ? 'text/html' : 'text/plain' }], metadata: { url: params.url.toString(), contentType: response.headers.get('content-type'), contentLength: response.headers.get('content-length'), isGoogleSearch: params.url.origin + params.url.pathname === GOOGLE_SEARCH_URL, responseType: params.responseType } }; } catch (error) { if (error instanceof Error && error.name === 'AbortError') { return { content: [{ type: "text", text: `Request timed out after ${params.timeout}ms` }], isError: true }; } if (attempt === MAX_RETRIES - 1) { return { content: [{ type: "text", text: `Failed to fetch URL: ${error instanceof Error ? error.message : 'Unknown error'}` }], isError: true }; } await new Promise(resolve => setTimeout(resolve, getRetryDelay(attempt))); } } return { content: [{ type: "text", text: "Failed to fetch URL after all retry attempts" }], isError: true }; } ); } - src/index.ts:24-30 (schema)Declarative tool schema in MCP server capabilities for client discovery.
fetch_url: { description: "Fetch content from a URL with proper error handling and response processing", parameters: { url: "The URL to fetch", responseType: "Expected response type (text, json, html, markdown)", timeout: "Request timeout in milliseconds (optional)" } - src/url-fetcher.ts:225-270 (helper)Helper function called by handler to process fetched response: size check, Google special case, format conversion based on responseType.
async function processResponse(response: Response, responseType: 'text' | 'json' | 'html' | 'markdown', url: URL): Promise<string> { const contentType = response.headers.get('content-type') || ''; // Check response size const contentLength = parseInt(response.headers.get('content-length') || '0'); if (contentLength > MAX_RESPONSE_SIZE) { throw new Error('Response too large'); } let text = await response.text(); // Special handling for Google search results if (url.origin + url.pathname === GOOGLE_SEARCH_URL) { const mappedType = responseType === 'json' || responseType === 'markdown' ? responseType : 'json'; const results = await extractGoogleResults(text, mappedType); return typeof results === 'string' ? results : JSON.stringify(results, null, 2); } switch (responseType) { case 'json': if (!contentType.includes('application/json')) { throw new Error('Response is not JSON'); } // Pretty print JSON return JSON.stringify(JSON.parse(text), null, 2); case 'html': if (!contentType.includes('text/html')) { throw new Error('Response is not HTML'); } return text; case 'markdown': if (contentType.includes('text/html')) { return htmlToMarkdown(text); } else if (contentType.includes('text/markdown')) { return text; } // If not HTML or Markdown, convert plain text to markdown return `\`\`\`\n${text}\n\`\`\``; case 'text': default: return text; } }