Web3 Research MCP

Implementation Reference

• src/tools/researchTools.ts:334-397 (handler)

  Executes the 'fetch-content' tool logic: determines if URL is local resource or web, fetches content using appropriate helper, stores full content as a resource in storage, returns a text preview (truncated if long).

    url,
    format,
  }: {
    url: string;
    format: "text" | "html" | "markdown" | "json";
  }) => {
    storage.addLogEntry(`Fetching content from: ${url} (format: ${format})`);
  
    try {
      let content;
  
      if (url.startsWith("research://resource/")) {
        content = await getResourceContent(url, storage);
      } else {
        content = await fetchContent(url, format);
      }
  
      const resourceId = url.startsWith("research://resource/")
        ? `derived_${url.replace(
            "research://resource/",
            ""
          )}_${new Date().getTime()}`
        : url
            .replace(/https?:\/\//, "")
            .replace(/[^\w]/g, "_")
            .substring(0, 30);
  
      storage.addToSection("resources", {
        [resourceId]: {
          url,
          format,
          content,
          fetchedAt: new Date().toISOString(),
        },
      });
  
      return {
        content: [
          {
            type: "text",
            text: `Fetched content from ${url} (${format}):\n\n${content.substring(
              0,
              1000
            )}${
              content.length > 1000
                ? "...\n\n[Content truncated, full version saved as resource]"
                : ""
            }`,
          },
        ],
      };
    } catch (error) {
      storage.addLogEntry(`Error fetching content from ${url}: ${error}`);
      return {
        isError: true,
        content: [
          {
            type: "text",
            text: `Error fetching content: ${error}`,
          },
        ],
      };
    }
  }

• src/tools/researchTools.ts:324-332 (schema)

  Input schema using Zod for the 'fetch-content' tool: url (string, supports research://), format (enum with default 'markdown').

  {
    url: z
      .string()
      .describe("URL to fetch content from (can be a resource:// URL)"),
    format: z
      .enum(["text", "html", "markdown", "json"])
      .default("markdown")
      .describe("Output format"),
  },

• src/tools/researchTools.ts:322-398 (registration)

  Registers the 'fetch-content' tool on the MCP server within registerResearchTools function.

  server.tool(
    "fetch-content",
    {
      url: z
        .string()
        .describe("URL to fetch content from (can be a resource:// URL)"),
      format: z
        .enum(["text", "html", "markdown", "json"])
        .default("markdown")
        .describe("Output format"),
    },
    async ({
      url,
      format,
    }: {
      url: string;
      format: "text" | "html" | "markdown" | "json";
    }) => {
      storage.addLogEntry(`Fetching content from: ${url} (format: ${format})`);
  
      try {
        let content;
  
        if (url.startsWith("research://resource/")) {
          content = await getResourceContent(url, storage);
        } else {
          content = await fetchContent(url, format);
        }
  
        const resourceId = url.startsWith("research://resource/")
          ? `derived_${url.replace(
              "research://resource/",
              ""
            )}_${new Date().getTime()}`
          : url
              .replace(/https?:\/\//, "")
              .replace(/[^\w]/g, "_")
              .substring(0, 30);
  
        storage.addToSection("resources", {
          [resourceId]: {
            url,
            format,
            content,
            fetchedAt: new Date().toISOString(),
          },
        });
  
        return {
          content: [
            {
              type: "text",
              text: `Fetched content from ${url} (${format}):\n\n${content.substring(
                0,
                1000
              )}${
                content.length > 1000
                  ? "...\n\n[Content truncated, full version saved as resource]"
                  : ""
              }`,
            },
          ],
        };
      } catch (error) {
        storage.addLogEntry(`Error fetching content from ${url}: ${error}`);
        return {
          isError: true,
          content: [
            {
              type: "text",
              text: `Error fetching content: ${error}`,
            },
          ],
        };
      }
    }
  );

• src/utils/searchUtils.ts:61-140 (helper)

  Supporting utility that performs the actual HTTP fetch of web content, handles JSON/HTML, parses to requested format using cheerio (removes scripts/styles), with retry logic and realistic headers.

  export async function fetchContent(
    url: string,
    format: "text" | "html" | "markdown" | "json" = "text",
    retries = 2
  ): Promise<string> {
    if (url.startsWith("research://")) {
      throw new Error("Only HTTP(S) protocols are supported");
    }
  
    for (let i = 0; i <= retries; i++) {
      try {
        await sleep(1000 + Math.random() * 2000);
  
        const response = await fetch(url, {
          headers: {
            "User-Agent":
              "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
            Accept:
              "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.9",
            Connection: "keep-alive",
            "Upgrade-Insecure-Requests": "1",
            Referer: "https://www.google.com/",
            "Cache-Control": "max-age=0",
            "sec-ch-ua":
              '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": '"Windows"',
            "Sec-Fetch-Dest": "document",
            "Sec-Fetch-Mode": "navigate",
            "Sec-Fetch-Site": "none",
            "Sec-Fetch-User": "?1",
          },
          redirect: "follow",
        });
  
        if (!response.ok) {
          throw new Error(
            `HTTP error ${response.status}: ${response.statusText}`
          );
        }
  
        const contentType = response.headers.get("content-type") || "";
  
        if (contentType.includes("application/json")) {
          const json = await response.json();
          return format === "json"
            ? JSON.stringify(json, null, 2)
            : JSON.stringify(json);
        } else {
          const html = await response.text();
  
          switch (format) {
            case "html":
              return html;
            case "markdown":
              const $ = cheerio.load(html);
              $("script, style, meta, link, noscript, iframe").remove();
              const title = $("title").text();
              const body = $("body").text().replace(/\s+/g, " ").trim();
              return `# ${title}\n\n${body}`;
            case "text":
            default:
              const $text = cheerio.load(html);
              $text("script, style, meta, link").remove();
              return $text("body").text().replace(/\s+/g, " ").trim();
          }
        }
      } catch (error) {
        if (i < retries) {
          const delay = 3000 * Math.pow(2, i);
          await sleep(delay);
        } else {
          throw error;
        }
      }
    }
  
    throw new Error(`Failed to fetch ${url} after ${retries} retries`);
  }

• src/tools/researchTools.ts:12-28 (helper)

  Helper function to retrieve content from locally stored research resources via research://resource/ URLs.

  export async function getResourceContent(
    url: string,
    storage: ResearchStorage
  ): Promise<string> {
    if (url.startsWith("research://resource/")) {
      const resourceId = url.replace("research://resource/", "");
      const resource = storage.getResource(resourceId);
  
      if (!resource) {
        throw new Error(`Resource not found: ${resourceId}`);
      }
  
      return resource.content;
    }
  
    return fetchContent(url, "markdown");
  }