robotstxt-ai-mcp

Instructions

Implementation Reference

• mcp-server/src/index.ts:526-552 (handler)

  The implementation of the `analyze_robots` tool handler, which uses `parseRobotsTxt` and `analyzeBots` to process input content and return a formatted analysis.

  // Tool 2: analyze_robots
  server.tool(
    "analyze_robots",
    "Analyze pasted robots.txt content. Returns which AI bots are blocked or allowed based on the rules.",
    {
      content: z
        .string()
        .describe("The robots.txt content to analyze"),
    },
    async ({ content }) => {
      const parsed = parseRobotsTxt(content);
      const statuses = analyzeBots(parsed);
  
      const sitemapInfo =
        parsed.sitemaps.length > 0
          ? `\n## Sitemaps Found\n${parsed.sitemaps.map((s) => `- ${s}`).join("\n")}`
          : "";
  
      return {
        content: [
          {
            type: "text" as const,
            text: `# robots.txt Analysis\n\n${formatBotAnalysis(statuses)}${sitemapInfo}`,
          },
        ],
      };
    }