Better Fetch

Instructions

Implementation Reference

• src/server.ts:393-431 (handler)

  Handler for executing the fetch_website_nested tool. Extracts parameters, creates options, calls scraper.scrapeWebsite, and returns markdown content.

  case "fetch_website_nested": {
    const {
      url,
      maxDepth = 2,
      maxPages = 50,
      sameDomainOnly = true,
      excludePatterns = [],
      includePatterns = [],
      timeout = 10000,
    } = args as any;
  
    if (!url) {
      throw new Error("URL is required");
    }
  
    try {
      const options: FetchOptions = {
        maxDepth,
        maxPages,
        sameDomainOnly,
        excludePatterns,
        includePatterns,
        timeout,
      };
  
      const markdown = await scraper.scrapeWebsite(url, options);
  
      return {
        content: [
          {
            type: "text",
            text: markdown,
          },
        ],
      };
    } catch (error) {
      throw new Error(`Failed to fetch website: ${error}`);
    }
  }

• src/server.ts:304-343 (schema)

  Input schema defining parameters for the fetch_website_nested tool, including URL, crawling depth, page limits, and filtering options.

  inputSchema: {
    type: "object",
    properties: {
      url: {
        type: "string",
        description: "The starting URL to fetch and crawl",
      },
      maxDepth: {
        type: "number",
        description: "Maximum depth to crawl (default: 2)",
        default: 2,
      },
      maxPages: {
        type: "number", 
        description: "Maximum number of pages to fetch (default: 50)",
        default: 50,
      },
      sameDomainOnly: {
        type: "boolean",
        description: "Only crawl URLs from the same domain (default: true)",
        default: true,
      },
      excludePatterns: {
        type: "array",
        items: { type: "string" },
        description: "Regex patterns for URLs to exclude",
      },
      includePatterns: {
        type: "array", 
        items: { type: "string" },
        description: "Regex patterns for URLs to include (if specified, only matching URLs will be processed)",
      },
      timeout: {
        type: "number",
        description: "Request timeout in milliseconds (default: 10000)",
        default: 10000,
      },
    },
    required: ["url"],
  },

• src/server.ts:301-344 (registration)

  Registration of the fetch_website_nested tool in the TOOLS array, returned by list tools handler.

  {
    name: "fetch_website_nested",
    description: "Fetch website content with nested URL crawling and convert to clean markdown",
    inputSchema: {
      type: "object",
      properties: {
        url: {
          type: "string",
          description: "The starting URL to fetch and crawl",
        },
        maxDepth: {
          type: "number",
          description: "Maximum depth to crawl (default: 2)",
          default: 2,
        },
        maxPages: {
          type: "number", 
          description: "Maximum number of pages to fetch (default: 50)",
          default: 50,
        },
        sameDomainOnly: {
          type: "boolean",
          description: "Only crawl URLs from the same domain (default: true)",
          default: true,
        },
        excludePatterns: {
          type: "array",
          items: { type: "string" },
          description: "Regex patterns for URLs to exclude",
        },
        includePatterns: {
          type: "array", 
          items: { type: "string" },
          description: "Regex patterns for URLs to include (if specified, only matching URLs will be processed)",
        },
        timeout: {
          type: "number",
          description: "Request timeout in milliseconds (default: 10000)",
          default: 10000,
        },
      },
      required: ["url"],
    },
  },

• src/server.ts:219-260 (helper)

  Core helper method implementing nested website crawling, content extraction, and markdown formatting.

  async scrapeWebsite(startUrl: string, options: FetchOptions = {}): Promise<string> {
    const {
      maxDepth = 2,
      maxPages = 50,
      sameDomainOnly = true,
      timeout = 10000
    } = options;
  
    this.baseUrl = startUrl;
    this.visitedUrls.clear();
  
    const allContent: PageContent[] = [];
    const urlsToProcess: Array<{ url: string; depth: number }> = [{ url: startUrl, depth: 0 }];
  
    while (urlsToProcess.length > 0 && allContent.length < maxPages) {
      const { url, depth } = urlsToProcess.shift()!;
  
      if (depth > maxDepth || this.visitedUrls.has(url)) {
        continue;
      }
  
      const pageContent = await this.fetchPageContent(url, depth, options);
      
      if (pageContent) {
        allContent.push(pageContent);
  
        // Add child URLs for processing
        if (depth < maxDepth) {
          for (const link of pageContent.links) {
            if (!this.visitedUrls.has(link)) {
              urlsToProcess.push({ url: link, depth: depth + 1 });
            }
          }
        }
      }
  
      // Small delay to be respectful
      await new Promise(resolve => setTimeout(resolve, 500));
    }
  
    return this.formatAsMarkdown(allContent, startUrl);
  }

• src/server.ts:30-297 (helper)

  AdvancedWebScraper class providing all helper utilities for web scraping, link extraction, content cleaning, and markdown conversion. Used by the tool handler.

  class AdvancedWebScraper {
    private turndownService: TurndownService;
    private visitedUrls: Set<string> = new Set();
    private baseUrl: string = '';
  
    constructor() {
      this.turndownService = new TurndownService({
        headingStyle: 'atx',
        codeBlockStyle: 'fenced',
        bulletListMarker: '-',
      });
  
      // Custom rules for better markdown conversion
      this.turndownService.addRule('removeScripts', {
        filter: ['script', 'style', 'nav', 'header', 'footer', 'aside'],
        replacement: () => ''
      });
  
      this.turndownService.addRule('cleanCodeBlocks', {
        filter: 'pre',
        replacement: (content, node) => {
          const code = node.textContent || '';
          return '\n```\n' + code + '\n```\n';
        }
      });
    }
  
    private async fetchWithTimeout(url: string, timeout: number = 10000): Promise<Response> {
      const controller = new AbortController();
      const timeoutId = setTimeout(() => controller.abort(), timeout);
  
      try {
        const response = await fetch(url, {
          signal: controller.signal,
          headers: {
            'User-Agent': 'Mozilla/5.0 (compatible; MCP-WebScraper/1.0)',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
          }
        });
        clearTimeout(timeoutId);
        return response as unknown as Response;
      } catch (error) {
        clearTimeout(timeoutId);
        throw error;
      }
    }
  
    private extractLinks($: cheerio.CheerioAPI, baseUrl: string): string[] {
      const links: string[] = [];
      
      $('a[href]').each((_, element) => {
        const href = $(element).attr('href');
        if (href) {
          try {
            const absoluteUrl = new URL(href, baseUrl).toString();
            links.push(absoluteUrl);
          } catch (error) {
            // Invalid URL, skip
          }
        }
      });
  
      return [...new Set(links)]; // Remove duplicates
    }
  
    private shouldProcessUrl(url: string, options: FetchOptions): boolean {
      const urlObj = new URL(url);
      const baseUrlObj = new URL(this.baseUrl);
  
      // Check if same domain only
      if (options.sameDomainOnly && urlObj.hostname !== baseUrlObj.hostname) {
        return false;
      }
  
      // Check exclude patterns
      if (options.excludePatterns) {
        for (const pattern of options.excludePatterns) {
          if (url.match(new RegExp(pattern, 'i'))) {
            return false;
          }
        }
      }
  
      // Check include patterns
      if (options.includePatterns && options.includePatterns.length > 0) {
        let matches = false;
        for (const pattern of options.includePatterns) {
          if (url.match(new RegExp(pattern, 'i'))) {
            matches = true;
            break;
          }
        }
        if (!matches) return false;
      }
  
      // Skip common non-content URLs
      const skipPatterns = [
        /\.(pdf|doc|docx|xls|xlsx|ppt|pptx|zip|rar|tar|gz)$/i,
        /^mailto:/,
        /^tel:/,
        /^javascript:/,
        /#$/,
        /\/search\?/,
        /\/login/,
        /\/register/,
        /\/cart/,
        /\/checkout/,
      ];
  
      return !skipPatterns.some(pattern => pattern.test(url));
    }
  
    private cleanContent($: cheerio.CheerioAPI): string {
      // Remove unwanted elements
      $('script, style, nav, header, footer, aside, .advertisement, .ads, .sidebar, .menu, .navigation').remove();
      
      // Find main content area
      let contentElement = $('main, article, .content, .main-content, #content, #main').first();
      
      if (contentElement.length === 0) {
        // Fallback to body if no main content area found
        contentElement = $('body');
      }
  
      return contentElement.html() || '';
    }
  
    private generateSectionTitle(url: string, title: string, depth: number): string {
      const urlObj = new URL(url);
      const pathParts = urlObj.pathname.split('/').filter(part => part && part !== 'index.html');
      
      let sectionTitle = title || pathParts[pathParts.length - 1] || urlObj.hostname;
      
      // Clean up title
      sectionTitle = sectionTitle
        .replace(/[-_]/g, ' ')
        .replace(/\b\w/g, l => l.toUpperCase())
        .trim();
  
      const headerLevel = '#'.repeat(Math.min(depth + 1, 6));
      return `${headerLevel} ${sectionTitle}`;
    }
  
    private async fetchPageContent(url: string, depth: number, options: FetchOptions): Promise<PageContent | null> {
      if (this.visitedUrls.has(url)) {
        return null;
      }
  
      this.visitedUrls.add(url);
  
      try {
        console.error(`Fetching: ${url} (depth: ${depth})`);
        
        const response = await this.fetchWithTimeout(url, options.timeout);
        
        if (!response.ok) {
          console.error(`Failed to fetch ${url}: ${response.status}`);
          return null;
        }
  
        const html = await response.text();
        const $ = cheerio.load(html);
  
        // Extract title
        const title = $('title').text().trim() || 
                     $('h1').first().text().trim() || 
                     'Untitled Page';
  
        // Clean and extract content
        const cleanHtml = this.cleanContent($);
        const markdownContent = this.turndownService.turndown(cleanHtml);
  
        // Extract links for potential further processing
        const links = this.extractLinks($, url);
  
        return {
          url,
          title,
          content: markdownContent,
          links: links.filter(link => this.shouldProcessUrl(link, options)),
          depth
        };
  
      } catch (error) {
        console.error(`Error fetching ${url}:`, error);
        return null;
      }
    }
  
    async scrapeWebsite(startUrl: string, options: FetchOptions = {}): Promise<string> {
      const {
        maxDepth = 2,
        maxPages = 50,
        sameDomainOnly = true,
        timeout = 10000
      } = options;
  
      this.baseUrl = startUrl;
      this.visitedUrls.clear();
  
      const allContent: PageContent[] = [];
      const urlsToProcess: Array<{ url: string; depth: number }> = [{ url: startUrl, depth: 0 }];
  
      while (urlsToProcess.length > 0 && allContent.length < maxPages) {
        const { url, depth } = urlsToProcess.shift()!;
  
        if (depth > maxDepth || this.visitedUrls.has(url)) {
          continue;
        }
  
        const pageContent = await this.fetchPageContent(url, depth, options);
        
        if (pageContent) {
          allContent.push(pageContent);
  
          // Add child URLs for processing
          if (depth < maxDepth) {
            for (const link of pageContent.links) {
              if (!this.visitedUrls.has(link)) {
                urlsToProcess.push({ url: link, depth: depth + 1 });
              }
            }
          }
        }
  
        // Small delay to be respectful
        await new Promise(resolve => setTimeout(resolve, 500));
      }
  
      return this.formatAsMarkdown(allContent, startUrl);
    }
  
    private formatAsMarkdown(contents: PageContent[], startUrl: string): string {
      const urlObj = new URL(startUrl);
      const siteName = urlObj.hostname;
      
      let markdown = `# ${siteName} Documentation\n\n`;
      markdown += `*Scraped from: ${startUrl}*\n`;
      markdown += `*Generated on: ${new Date().toISOString()}*\n\n`;
      
      // Table of contents
      markdown += `## Table of Contents\n\n`;
      contents.forEach((content, index) => {
        const indent = '  '.repeat(content.depth);
        markdown += `${indent}- [${content.title}](#${this.slugify(content.title)})\n`;
      });
      markdown += '\n---\n\n';
  
      // Content sections
      contents.forEach(content => {
        const sectionTitle = this.generateSectionTitle(content.url, content.title, content.depth);
        markdown += `${sectionTitle}\n\n`;
        markdown += `*Source: [${content.url}](${content.url})*\n\n`;
        markdown += content.content;
        markdown += '\n\n---\n\n';
      });
  
      return markdown;
    }
  
    private slugify(text: string): string {
      return text
        .toLowerCase()
        .replace(/[^\w\s-]/g, '')
        .replace(/[\s_-]+/g, '-')
        .replace(/^-+|-+$/g, '');
    }
  }