robotstxt-ai-mcp

Instructions

Implementation Reference

• mcp-server/src/index.ts:620-666 (handler)

  The implementation of the `list_ai_bots` tool handler, which iterates over `ALL_BOTS` to format and return a list of known AI bots.

  server.tool(
    "list_ai_bots",
    "List all known AI bots with their user-agents, companies, and descriptions. Useful for deciding which bots to block.",
    {},
    async () => {
      const lines: string[] = [];
  
      lines.push("# Known AI & Search Engine Bots\n");
  
      const groups: Record<string, Bot[]> = {};
      for (const bot of ALL_BOTS) {
        const key = bot.type;
        if (!groups[key]) groups[key] = [];
        groups[key].push(bot);
      }
  
      const typeLabels: Record<string, string> = {
        "ai-crawler": "AI Crawlers (Training Data)",
        "ai-search": "AI Search Bots",
        "search-engine": "Search Engines",
        other: "Other Bots",
        social: "Social Bots",
      };
  
      for (const [type, bots] of Object.entries(groups)) {
        lines.push(`## ${typeLabels[type] || type}\n`);
        for (const bot of bots) {
          lines.push(`- **${bot.name}**`);
          lines.push(`  - User-Agent: \`${bot.userAgent}\``);
          lines.push(`  - Company: ${bot.company}`);
          lines.push(`  - ${bot.description}`);
          lines.push(
            `  - Default blocked: ${bot.defaultBlocked ? "Yes" : "No"}`
          );
          lines.push(`  - Docs: ${bot.docsUrl}`);
        }
        lines.push("");
      }
  
      return {
        content: [
          {
            type: "text" as const,
            text: lines.join("\n"),
          },
        ],
      };