read-website-fast

Instructions

Implementation Reference

• src/serve.ts:52-85 (schema)

  Defines the MCP Tool object for 'read_website' including name, description, input schema (url required, optional pages and cookiesFile), and annotations indicating behavior.
  const READ_WEBSITE_TOOL: Tool = { name: 'read_website', description: 'Fast, token-efficient web content extraction - ideal for reading documentation, analyzing content, and gathering information from websites. Converts to clean Markdown while preserving links and structure.', inputSchema: { type: 'object', properties: { url: { type: 'string', description: 'HTTP/HTTPS URL to fetch and convert to markdown', }, pages: { type: 'number', description: 'Maximum number of pages to crawl (default: 1)', default: 1, minimum: 1, maximum: 100, }, cookiesFile: { type: 'string', description: 'Path to Netscape cookie file for authenticated pages', optional: true, }, }, required: ['url'], }, annotations: { title: 'Read Website', readOnlyHint: true, // Only reads content destructiveHint: false, idempotentHint: true, // Same URL returns same content (with cache) openWorldHint: true, // Interacts with external websites }, };
• src/serve.ts:104-114 (registration)

  Registers the 'read_website' tool by including it in the ListTools response.
  server.setRequestHandler(ListToolsRequestSchema, async () => { logger.debug('Received ListTools request'); const response = { tools: [READ_WEBSITE_TOOL], }; logger.debug( 'Returning tools:', response.tools.map(t => t.name) ); return response; });
• src/serve.ts:117-198 (handler)

  MCP server handler for tool calls: dispatches 'read_website' requests, lazy loads core module, calls fetchMarkdown with parameters, formats response as MCP content.
  server.setRequestHandler(CallToolRequestSchema, async request => { logger.info('Received CallTool request:', request.params.name); logger.debug('Request params:', JSON.stringify(request.params, null, 2)); if (request.params.name !== 'read_website') { const error = `Unknown tool: ${request.params.name}`; logger.error(error); throw new Error(error); } try { // Lazy load the module on first use if (!fetchMarkdownModule) { logger.debug('Lazy loading fetchMarkdown module...'); fetchMarkdownModule = await import('./internal/fetchMarkdown.js'); logger.info('fetchMarkdown module loaded successfully'); } const args = request.params.arguments as any; // Validate URL if (!args.url || typeof args.url !== 'string') { throw new Error('URL parameter is required and must be a string'); } logger.info(`Processing read request for URL: ${args.url}`); logger.debug('Read parameters:', { url: args.url, pages: args.pages, cookiesFile: args.cookiesFile, }); logger.debug('Calling fetchMarkdown...'); // Convert pages to depth (pages - 1 = depth) // pages: 1 = depth: 0 (single page) // pages: 2+ = depth: 1 (crawl one level to get multiple pages) const depth = args.pages > 1 ? 1 : 0; const result = await fetchMarkdownModule.fetchMarkdown(args.url, { depth: depth, respectRobots: false, // Default to not respecting robots.txt maxPages: args.pages ?? 1, cookiesFile: args.cookiesFile, }); logger.info('Content fetched successfully'); // If there's an error but we still have some content, return it with a note if (result.error && result.markdown) { return { content: [ { type: 'text', text: `${result.markdown}\n\n---\n*Note: ${result.error}*`, }, ], }; } // If there's an error and no content, throw it if (result.error && !result.markdown) { throw new Error(result.error); } return { content: [{ type: 'text', text: result.markdown }], }; } catch (error: any) { logger.error('Error fetching content:', error.message); logger.debug('Error stack:', error.stack); logger.debug('Error details:', { name: error.name, code: error.code, ...error, }); // Re-throw with more context throw new Error( `Failed to fetch content: ${error instanceof Error ? error.message : 'Unknown error'}` ); } });
• src/internal/fetchMarkdown.ts:23-127 (handler)

  Core tool handler: iteratively crawls up to maxPages using @just-every/crawl (single depth), extracts same-origin links, combines markdown from multiple pages with sources indicated.
  export async function fetchMarkdown( url: string, options: FetchMarkdownOptions = {} ): Promise<FetchMarkdownResult> { try { const maxPages = options.maxPages ?? 1; const visited = new Set<string>(); const toVisit = [url]; const allResults: any[] = []; // If we want multiple pages, we need to crawl iteratively while (toVisit.length > 0 && allResults.length < maxPages) { const currentUrl = toVisit.shift()!; // Skip if already visited if (visited.has(currentUrl)) continue; visited.add(currentUrl); // Fetch single page const crawlOptions: CrawlOptions = { depth: 0, // Always single page maxConcurrency: options.maxConcurrency ?? 3, respectRobots: options.respectRobots ?? true, sameOriginOnly: options.sameOriginOnly ?? true, userAgent: options.userAgent, cacheDir: options.cacheDir ?? '.cache', timeout: options.timeout ?? 30000, }; if (options.cookiesFile) { (crawlOptions as any).cookiesFile = options.cookiesFile; } const results = await fetch(currentUrl, crawlOptions); if (results && results.length > 0) { const result = results[0]; allResults.push(result); // Extract links from markdown if we need more pages if (allResults.length < maxPages && result.markdown) { const links = extractMarkdownLinks(result.markdown, currentUrl); const filteredLinks = options.sameOriginOnly !== false ? filterSameOriginLinks(links, currentUrl) : links; // Add new links to visit queue for (const link of filteredLinks) { if (!visited.has(link) && !toVisit.includes(link)) { toVisit.push(link); } } } } } if (allResults.length === 0) { return { markdown: '', error: 'No results returned', }; } // Process results as before const pagesToReturn = allResults; // Combine all pages into a single markdown document const combinedMarkdown = pagesToReturn .map((result, index) => { if (result.error) { return `<!-- Error fetching ${result.url}: ${result.error} -->`; } let pageContent = ''; // Add page separator for multiple pages if (pagesToReturn.length > 1 && index > 0) { pageContent += '\n\n---\n\n'; } // Add source URL as a comment pageContent += `<!-- Source: ${result.url} -->\n`; // Add the content pageContent += result.markdown || ''; return pageContent; }) .join('\n'); // Return combined results return { markdown: combinedMarkdown, title: pagesToReturn[0].title, links: pagesToReturn.flatMap(r => r.links || []), error: pagesToReturn.some(r => r.error) ? `Some pages had errors: ${pagesToReturn.filter(r => r.error).map(r => r.url).join(', ')}` : undefined, }; } catch (error) { return { markdown: '', error: error instanceof Error ? error.message : 'Unknown error', }; } }
• src/utils/extractMarkdownLinks.ts:1-68 (helper)

  Utility functions used by fetchMarkdown to extract links from markdown for crawling and filter to same-origin only.
  /** * Extract all HTTP/HTTPS links from markdown content * @param markdown The markdown content to extract links from * @param baseUrl The base URL to resolve relative links against * @returns Array of absolute URLs found in the markdown */ export function extractMarkdownLinks(markdown: string, baseUrl: string): string[] { const links: string[] = []; // Match markdown links: [text](url) const markdownLinkRegex = /\[([^\]]+)\]\(([^)]+)\)/g; // Match bare URLs const bareUrlRegex = /https?:\/\/[^\s<>)\]]+/g; // Extract markdown links let match; while ((match = markdownLinkRegex.exec(markdown)) !== null) { const url = match[2]; if (url && !url.startsWith('#') && !url.startsWith('mailto:') && !url.startsWith('tel:')) { links.push(url); } } // Extract bare URLs while ((match = bareUrlRegex.exec(markdown)) !== null) { links.push(match[0]); } // Convert relative URLs to absolute const absoluteLinks = links.map(link => { try { // If it's already absolute, return as-is if (link.startsWith('http://') || link.startsWith('https://')) { return link; } // Otherwise, resolve relative to base URL return new URL(link, baseUrl).href; } catch { // If URL parsing fails, skip this link return null; } }).filter(Boolean) as string[]; // Remove duplicates and return return [...new Set(absoluteLinks)]; } /** * Filter links to only include those from the same origin * @param links Array of URLs to filter * @param baseUrl The base URL to compare against * @returns Filtered array of URLs from the same origin */ export function filterSameOriginLinks(links: string[], baseUrl: string): string[] { try { const baseOrigin = new URL(baseUrl).origin; return links.filter(link => { try { return new URL(link).origin === baseOrigin; } catch { return false; } }); } catch { return []; } }