fetch_website_nested

Crawl websites with nested URL navigation to extract and convert content into clean markdown format for structured data analysis.

Instructions

Fetch website content with nested URL crawling and convert to clean markdown

Input Schema

TableJSON Schema

Name	Required	Description
`url`	Yes	The starting URL to fetch and crawl
`maxDepth`	No	Maximum depth to crawl (default: 2)
`maxPages`	No	Maximum number of pages to fetch (default: 50)
`sameDomainOnly`	No	Only crawl URLs from the same domain (default: true)
`excludePatterns`	No	Regex patterns for URLs to exclude
`includePatterns`	No	Regex patterns for URLs to include (if specified, only matching URLs will be processed)
`timeout`	No	Request timeout in milliseconds (default: 10000)

Implementation Reference

src/server.ts:393-431 (handler)

Handler for executing the fetch_website_nested tool. Extracts parameters, creates options, calls scraper.scrapeWebsite, and returns markdown content.

case "fetch_website_nested": {
  const {
    url,
    maxDepth = 2,
    maxPages = 50,
    sameDomainOnly = true,
    excludePatterns = [],
    includePatterns = [],
    timeout = 10000,
  } = args as any;

  if (!url) {
    throw new Error("URL is required");
  }

  try {
    const options: FetchOptions = {
      maxDepth,
      maxPages,
      sameDomainOnly,
      excludePatterns,
      includePatterns,
      timeout,
    };

    const markdown = await scraper.scrapeWebsite(url, options);

    return {
      content: [
        {
          type: "text",
          text: markdown,
        },
      ],
    };
  } catch (error) {
    throw new Error(`Failed to fetch website: ${error}`);
  }
}

src/server.ts:304-343 (schema)

Input schema defining parameters for the fetch_website_nested tool, including URL, crawling depth, page limits, and filtering options.

inputSchema: {
  type: "object",
  properties: {
    url: {
      type: "string",
      description: "The starting URL to fetch and crawl",
    },
    maxDepth: {
      type: "number",
      description: "Maximum depth to crawl (default: 2)",
      default: 2,
    },
    maxPages: {
      type: "number", 
      description: "Maximum number of pages to fetch (default: 50)",
      default: 50,
    },
    sameDomainOnly: {
      type: "boolean",
      description: "Only crawl URLs from the same domain (default: true)",
      default: true,
    },
    excludePatterns: {
      type: "array",
      items: { type: "string" },
      description: "Regex patterns for URLs to exclude",
    },
    includePatterns: {
      type: "array", 
      items: { type: "string" },
      description: "Regex patterns for URLs to include (if specified, only matching URLs will be processed)",
    },
    timeout: {
      type: "number",
      description: "Request timeout in milliseconds (default: 10000)",
      default: 10000,
    },
  },
  required: ["url"],
},

src/server.ts:301-344 (registration)

Registration of the fetch_website_nested tool in the TOOLS array, returned by list tools handler.

{
  name: "fetch_website_nested",
  description: "Fetch website content with nested URL crawling and convert to clean markdown",
  inputSchema: {
    type: "object",
    properties: {
      url: {
        type: "string",
        description: "The starting URL to fetch and crawl",
      },
      maxDepth: {
        type: "number",
        description: "Maximum depth to crawl (default: 2)",
        default: 2,
      },
      maxPages: {
        type: "number", 
        description: "Maximum number of pages to fetch (default: 50)",
        default: 50,
      },
      sameDomainOnly: {
        type: "boolean",
        description: "Only crawl URLs from the same domain (default: true)",
        default: true,
      },
      excludePatterns: {
        type: "array",
        items: { type: "string" },
        description: "Regex patterns for URLs to exclude",
      },
      includePatterns: {
        type: "array", 
        items: { type: "string" },
        description: "Regex patterns for URLs to include (if specified, only matching URLs will be processed)",
      },
      timeout: {
        type: "number",
        description: "Request timeout in milliseconds (default: 10000)",
        default: 10000,
      },
    },
    required: ["url"],
  },
},

src/server.ts:219-260 (helper)

Core helper method implementing nested website crawling, content extraction, and markdown formatting.

async scrapeWebsite(startUrl: string, options: FetchOptions = {}): Promise<string> {
  const {
    maxDepth = 2,
    maxPages = 50,
    sameDomainOnly = true,
    timeout = 10000
  } = options;

  this.baseUrl = startUrl;
  this.visitedUrls.clear();

  const allContent: PageContent[] = [];
  const urlsToProcess: Array<{ url: string; depth: number }> = [{ url: startUrl, depth: 0 }];

  while (urlsToProcess.length > 0 && allContent.length < maxPages) {
    const { url, depth } = urlsToProcess.shift()!;

    if (depth > maxDepth || this.visitedUrls.has(url)) {
      continue;
    }

    const pageContent = await this.fetchPageContent(url, depth, options);
    
    if (pageContent) {
      allContent.push(pageContent);

      // Add child URLs for processing
      if (depth < maxDepth) {
        for (const link of pageContent.links) {
          if (!this.visitedUrls.has(link)) {
            urlsToProcess.push({ url: link, depth: depth + 1 });
          }
        }
      }
    }

    // Small delay to be respectful
    await new Promise(resolve => setTimeout(resolve, 500));
  }

  return this.formatAsMarkdown(allContent, startUrl);
}

src/server.ts:30-297 (helper)

AdvancedWebScraper class providing all helper utilities for web scraping, link extraction, content cleaning, and markdown conversion. Used by the tool handler.

class AdvancedWebScraper {
  private turndownService: TurndownService;
  private visitedUrls: Set<string> = new Set();
  private baseUrl: string = '';

  constructor() {
    this.turndownService = new TurndownService({
      headingStyle: 'atx',
      codeBlockStyle: 'fenced',
      bulletListMarker: '-',
    });

    // Custom rules for better markdown conversion
    this.turndownService.addRule('removeScripts', {
      filter: ['script', 'style', 'nav', 'header', 'footer', 'aside'],
      replacement: () => ''
    });

    this.turndownService.addRule('cleanCodeBlocks', {
      filter: 'pre',
      replacement: (content, node) => {
        const code = node.textContent || '';
        return '\n```\n' + code + '\n```\n';
      }
    });
  }

  private async fetchWithTimeout(url: string, timeout: number = 10000): Promise<Response> {
    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), timeout);

    try {
      const response = await fetch(url, {
        signal: controller.signal,
        headers: {
          'User-Agent': 'Mozilla/5.0 (compatible; MCP-WebScraper/1.0)',
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        }
      });
      clearTimeout(timeoutId);
      return response as unknown as Response;
    } catch (error) {
      clearTimeout(timeoutId);
      throw error;
    }
  }

  private extractLinks($: cheerio.CheerioAPI, baseUrl: string): string[] {
    const links: string[] = [];
    
    $('a[href]').each((_, element) => {
      const href = $(element).attr('href');
      if (href) {
        try {
          const absoluteUrl = new URL(href, baseUrl).toString();
          links.push(absoluteUrl);
        } catch (error) {
          // Invalid URL, skip
        }
      }
    });

    return [...new Set(links)]; // Remove duplicates
  }

  private shouldProcessUrl(url: string, options: FetchOptions): boolean {
    const urlObj = new URL(url);
    const baseUrlObj = new URL(this.baseUrl);

    // Check if same domain only
    if (options.sameDomainOnly && urlObj.hostname !== baseUrlObj.hostname) {
      return false;
    }

    // Check exclude patterns
    if (options.excludePatterns) {
      for (const pattern of options.excludePatterns) {
        if (url.match(new RegExp(pattern, 'i'))) {
          return false;
        }
      }
    }

    // Check include patterns
    if (options.includePatterns && options.includePatterns.length > 0) {
      let matches = false;
      for (const pattern of options.includePatterns) {
        if (url.match(new RegExp(pattern, 'i'))) {
          matches = true;
          break;
        }
      }
      if (!matches) return false;
    }

    // Skip common non-content URLs
    const skipPatterns = [
      /\.(pdf|doc|docx|xls|xlsx|ppt|pptx|zip|rar|tar|gz)$/i,
      /^mailto:/,
      /^tel:/,
      /^javascript:/,
      /#$/,
      /\/search\?/,
      /\/login/,
      /\/register/,
      /\/cart/,
      /\/checkout/,
    ];

    return !skipPatterns.some(pattern => pattern.test(url));
  }

  private cleanContent($: cheerio.CheerioAPI): string {
    // Remove unwanted elements
    $('script, style, nav, header, footer, aside, .advertisement, .ads, .sidebar, .menu, .navigation').remove();
    
    // Find main content area
    let contentElement = $('main, article, .content, .main-content, #content, #main').first();
    
    if (contentElement.length === 0) {
      // Fallback to body if no main content area found
      contentElement = $('body');
    }

    return contentElement.html() || '';
  }

  private generateSectionTitle(url: string, title: string, depth: number): string {
    const urlObj = new URL(url);
    const pathParts = urlObj.pathname.split('/').filter(part => part && part !== 'index.html');
    
    let sectionTitle = title || pathParts[pathParts.length - 1] || urlObj.hostname;
    
    // Clean up title
    sectionTitle = sectionTitle
      .replace(/[-_]/g, ' ')
      .replace(/\b\w/g, l => l.toUpperCase())
      .trim();

    const headerLevel = '#'.repeat(Math.min(depth + 1, 6));
    return `${headerLevel} ${sectionTitle}`;
  }

  private async fetchPageContent(url: string, depth: number, options: FetchOptions): Promise<PageContent | null> {
    if (this.visitedUrls.has(url)) {
      return null;
    }

    this.visitedUrls.add(url);

    try {
      console.error(`Fetching: ${url} (depth: ${depth})`);
      
      const response = await this.fetchWithTimeout(url, options.timeout);
      
      if (!response.ok) {
        console.error(`Failed to fetch ${url}: ${response.status}`);
        return null;
      }

      const html = await response.text();
      const $ = cheerio.load(html);

      // Extract title
      const title = $('title').text().trim() || 
                   $('h1').first().text().trim() || 
                   'Untitled Page';

      // Clean and extract content
      const cleanHtml = this.cleanContent($);
      const markdownContent = this.turndownService.turndown(cleanHtml);

      // Extract links for potential further processing
      const links = this.extractLinks($, url);

      return {
        url,
        title,
        content: markdownContent,
        links: links.filter(link => this.shouldProcessUrl(link, options)),
        depth
      };

    } catch (error) {
      console.error(`Error fetching ${url}:`, error);
      return null;
    }
  }

  async scrapeWebsite(startUrl: string, options: FetchOptions = {}): Promise<string> {
    const {
      maxDepth = 2,
      maxPages = 50,
      sameDomainOnly = true,
      timeout = 10000
    } = options;

    this.baseUrl = startUrl;
    this.visitedUrls.clear();

    const allContent: PageContent[] = [];
    const urlsToProcess: Array<{ url: string; depth: number }> = [{ url: startUrl, depth: 0 }];

    while (urlsToProcess.length > 0 && allContent.length < maxPages) {
      const { url, depth } = urlsToProcess.shift()!;

      if (depth > maxDepth || this.visitedUrls.has(url)) {
        continue;
      }

      const pageContent = await this.fetchPageContent(url, depth, options);
      
      if (pageContent) {
        allContent.push(pageContent);

        // Add child URLs for processing
        if (depth < maxDepth) {
          for (const link of pageContent.links) {
            if (!this.visitedUrls.has(link)) {
              urlsToProcess.push({ url: link, depth: depth + 1 });
            }
          }
        }
      }

      // Small delay to be respectful
      await new Promise(resolve => setTimeout(resolve, 500));
    }

    return this.formatAsMarkdown(allContent, startUrl);
  }

  private formatAsMarkdown(contents: PageContent[], startUrl: string): string {
    const urlObj = new URL(startUrl);
    const siteName = urlObj.hostname;
    
    let markdown = `# ${siteName} Documentation\n\n`;
    markdown += `*Scraped from: ${startUrl}*\n`;
    markdown += `*Generated on: ${new Date().toISOString()}*\n\n`;
    
    // Table of contents
    markdown += `## Table of Contents\n\n`;
    contents.forEach((content, index) => {
      const indent = '  '.repeat(content.depth);
      markdown += `${indent}- [${content.title}](#${this.slugify(content.title)})\n`;
    });
    markdown += '\n---\n\n';

    // Content sections
    contents.forEach(content => {
      const sectionTitle = this.generateSectionTitle(content.url, content.title, content.depth);
      markdown += `${sectionTitle}\n\n`;
      markdown += `*Source: [${content.url}](${content.url})*\n\n`;
      markdown += content.content;
      markdown += '\n\n---\n\n';
    });

    return markdown;
  }

  private slugify(text: string): string {
    return text
      .toLowerCase()
      .replace(/[^\w\s-]/g, '')
      .replace(/[\s_-]+/g, '-')
      .replace(/^-+|-+$/g, '');
  }
}

Better Fetch