robotstxt-ai-mcp

Instructions

Implementation Reference

• mcp-server/src/index.ts:458-524 (handler)

  The implementation of the `fetch_robots` tool handler, which fetches a robots.txt file, parses it, and analyzes bot statuses.

  server.tool(
    "fetch_robots",
    "Fetch and analyze a robots.txt file from a URL. Returns which AI bots are blocked or allowed.",
    {
      url: z
        .string()
        .url()
        .describe(
          "The website URL to fetch robots.txt from (e.g. https://example.com)"
        ),
    },
    async ({ url }) => {
      try {
        // Normalize URL to get robots.txt
        const parsedUrl = new URL(url);
        const robotsUrl = `${parsedUrl.protocol}//${parsedUrl.host}/robots.txt`;
  
        const response = await fetch(robotsUrl, {
          headers: {
            "User-Agent": "robotstxt-ai-mcp/1.0",
          },
          signal: AbortSignal.timeout(10000),
        });
  
        if (!response.ok) {
          return {
            content: [
              {
                type: "text" as const,
                text: `Failed to fetch robots.txt from ${robotsUrl}: HTTP ${response.status} ${response.statusText}`,
              },
            ],
          };
        }
  
        const content = await response.text();
        const parsed = parseRobotsTxt(content);
        const statuses = analyzeBots(parsed);
  
        const sitemapInfo =
          parsed.sitemaps.length > 0
            ? `\n## Sitemaps Found\n${parsed.sitemaps.map((s) => `- ${s}`).join("\n")}`
            : "";
  
        return {
          content: [
            {
              type: "text" as const,
              text: `# robots.txt Analysis for ${parsedUrl.host}\n\n${formatBotAnalysis(statuses)}${sitemapInfo}\n\n## Raw robots.txt\n\`\`\`\n${content}\n\`\`\``,
            },
          ],
        };
      } catch (error) {
        const message =
          error instanceof Error ? error.message : String(error);
        return {
          content: [
            {
              type: "text" as const,
              text: `Error fetching robots.txt from ${url}: ${message}`,
            },
          ],
          isError: true,
        };
      }
    }
  );