robotstxt-ai-mcp

Instructions

Implementation Reference

• mcp-server/src/index.ts:670-758 (handler)

  The `check_bot_status` tool fetches a robots.txt file, parses it, and determines if a specified bot is allowed or blocked based on the rules found.

  // Tool 5: check_bot_status
  server.tool(
    "check_bot_status",
    "Check if a specific bot is blocked or allowed on a given website by fetching and analyzing its robots.txt.",
    {
      url: z
        .string()
        .url()
        .describe("The website URL to check (e.g. https://example.com)"),
      bot_name: z
        .string()
        .describe(
          "The bot user-agent string to check (e.g. 'GPTBot', 'ClaudeBot'). Use list_ai_bots to see available names."
        ),
    },
    async ({ url, bot_name }) => {
      try {
        const parsedUrl = new URL(url);
        const robotsUrl = `${parsedUrl.protocol}//${parsedUrl.host}/robots.txt`;
  
        const response = await fetch(robotsUrl, {
          headers: {
            "User-Agent": "robotstxt-ai-mcp/1.0",
          },
          signal: AbortSignal.timeout(10000),
        });
  
        if (!response.ok) {
          return {
            content: [
              {
                type: "text" as const,
                text: `Could not fetch robots.txt from ${robotsUrl}: HTTP ${response.status}. If no robots.txt exists, the bot is allowed by default.`,
              },
            ],
          };
        }
  
        const content = await response.text();
        const parsed = parseRobotsTxt(content);
        const statuses = analyzeBots(parsed);
  
        // Find the specific bot
        const botStatus = statuses.find(
          (s) =>
            s.bot.userAgent.toLowerCase() === bot_name.toLowerCase() ||
            s.bot.name.toLowerCase() === bot_name.toLowerCase()
        );
  
        if (!botStatus) {
          return {
            content: [
              {
                type: "text" as const,
                text: `Bot "${bot_name}" is not in the known bot database. Use list_ai_bots to see available bots.\n\nHowever, I can check the raw rules. Here are the user-agent directives found:\n${parsed.rules.map((r) => `- User-agent: ${r.userAgent}`).join("\n")}`,
              },
            ],
          };
        }
  
        const status = botStatus.blocked ? "BLOCKED" : "ALLOWED";
        const rulesInfo =
          botStatus.rules.length > 0
            ? `\nMatching rules:\n${botStatus.rules.map((r) => `  - ${r}`).join("\n")}`
            : "\nNo specific rules found for this bot (allowed by default).";
  
        return {
          content: [
            {
              type: "text" as const,
              text: `# Bot Status: ${botStatus.bot.name} on ${parsedUrl.host}\n\n**Status: ${status}**\n\n- Bot: ${botStatus.bot.name}\n- User-Agent: \`${botStatus.bot.userAgent}\`\n- Company: ${botStatus.bot.company}\n- Type: ${botStatus.bot.type}${rulesInfo}`,
            },
          ],
        };
      } catch (error) {
        const message =
          error instanceof Error ? error.message : String(error);
        return {
          content: [
            {
              type: "text" as const,
              text: `Error checking bot status: ${message}`,
            },
          ],
          isError: true,
        };
      }
    }
  );