@ragrabbit/mcp

Verified
MIT License
OverviewInspectSchema Related Servers Reviews Score
packages
rag
scraping
import { UserError } from "@repo/core";
import { logger } from "@repo/logger";
import * as cheerio from "cheerio";
import * as contentTypeHelper from "content-type";
import { NodeHtmlMarkdown } from "node-html-markdown";

const CHUNK_SIZE = 1024;
const MAX_CHUNKS = 20;

export async function scrapeUrl(
  url: string,
  options: {
    stripLinks?: boolean;
    stripImages?: boolean;
    stripHeader?: boolean;
    stripFooter?: boolean;
    stripQueries?: string;
    allowSubdomains?: boolean;
    supportedContentTypes?: string[];
  }
) {
  options = {
    stripLinks: false,
    stripImages: true,
    stripHeader: true,
    stripFooter: true,
    allowSubdomains: false,
    stripQueries: "aside, nav",
    supportedContentTypes: ["text/html", "application/xhtml+xml", "text/plain", "text/markdown", "text/x-markdown"],
    ...options,
  };
  logger.info("Scraping URL", { url, options });
  const urlParts = new URL(url);
  const hostname = urlParts.hostname.toLowerCase().replace("www.", "");
  const domain = hostname.split(".").slice(-2).join(".");
  const response = await fetch(url);
  let redirectedUrl = url;
  if (response.redirected) {
    redirectedUrl = response.url;
  }
  if (response.status !== 200) {
    throw new UserError("Failed to fetch, error from page: " + response.statusText + " status: " + response.status);
  }

  const contentType = contentTypeHelper.parse(response.headers.get("content-type"));
  if (!options.supportedContentTypes.includes(contentType.type)) {
    return { unsuportedContent: true, contentType: contentType.type };
  }
  const html = await response.text();

  const $ = cheerio.load(html);
  const title = $("title").first().text();
  logger.debug("Scraped HTML for page", { title });

  const canonicalUrl = $("link[rel='canonical']").first().attr("href") || redirectedUrl;

  // Collect links to be scraped later:
  let links: string[] = [];
  $("a").each((_, el) => {
    const href = $(el).attr("href");
    if (!href || !URL.canParse(href, url)) {
      return;
    }
    const urlHref = new URL(href, urlParts);

    if (options.allowSubdomains) {
      // Same domain regardless of subdomain:
      const urlDomain = urlHref.hostname.toLowerCase().split(".").slice(-2).join(".");
      if (urlDomain != domain) {
        return;
      }
    } else if (urlHref.hostname.toLowerCase().replace("www.", "") != hostname) {
      // Only exact domain:
      return;
    }

    // Skip image file extensions
    const ext = urlHref.pathname.split(".").pop()?.toLowerCase();
    if (ext && ["jpg", "jpeg", "png", "gif", "svg", "webp", "ico"].includes(ext)) {
      return;
    }

    // Collect url with protocol hostname and path
    let newUrl = urlHref.protocol + "//" + urlHref.hostname + urlHref.pathname;
    links.push(newUrl.toLowerCase());
  });

  // Clean up and shorten the content:
  // Strip links:
  if (options.stripLinks) {
    $("a").attr("href", "");
  } else {
    $("a").each((_, el) => {
      const href = $(el).attr("href");
      if (!href || !URL.canParse(href, url)) return;
      const urlHref = new URL(href, urlParts);
      if (urlHref.hostname.replace("www.", "") == hostname) {
        const relativeUrl = urlHref.pathname + urlHref.search + urlHref.hash;
        $(el).attr("href", relativeUrl);
      }
    });
  }
  if (options.stripImages) {
    $("img").attr("src", "");
    $("svg").attr("src", "");
  } else {
    $("img").each((_, el) => {
      const src = $(el).attr("src");
      if (!src || !URL.canParse(src, url)) return;
      const urlSrc = new URL(src, urlParts.origin);
      if (urlSrc.hostname.replace("www.", "") == hostname) {
        const relativeUrl = urlSrc.pathname.split("/").slice(-1)[0] + urlSrc.search + urlSrc.hash;
        $(el).attr("src", relativeUrl);
        if (!$(el).attr("alt")) {
          console.log("#ALT", $(el).attr("alt"));
          $(el).attr("alt", "Image");
        }
      }
    });
  }
  if (options.stripHeader) {
    $("header").remove();
  }
  if (options.stripFooter) {
    $("footer").remove();
  }

  if (options.stripQueries) {
    $(options.stripQueries).remove();
  }

  const body = $("body").html()?.toString();
  if (!body) {
    throw new UserError("No body content found in page");
  }

  // Translate to markdown:
  let content = NodeHtmlMarkdown.translate(body, {
    ignore: ["script", "style", "noscript", "svg", "img"],
    keepDataImages: false,
    useLinkReferenceDefinitions: false,
    useInlineLinks: false,
    codeBlockStyle: "fenced",
  });

  const description = $("meta[name='description']").first().attr("content") || "";
  if (content.length > CHUNK_SIZE * MAX_CHUNKS) {
    logger.warn("Content too large, truncating", { bytes: content.length, maxBytes: CHUNK_SIZE * MAX_CHUNKS });
    content = content.slice(0, CHUNK_SIZE * MAX_CHUNKS);
  }

  return { title, content, description, links, canonicalUrl, contentType: contentType.type };
}