MCP LLMS.txt Explorer

Instructions

Implementation Reference

• src/index.ts:156-344 (handler)

  The primary handler function that executes the tool logic: checks a website for llms.txt and llms-full.txt files, fetches their contents, handles linked @ URLs, with caching and timeouts.

  async function checkWebsite(domain: string): Promise<WebsiteCheckResult> {
    console.error('Starting website check for:', domain);
  
    // Return cached result if available
    if (websiteCheckCache[domain]) {
      console.error('Returning cached result for:', domain);
      return websiteCheckCache[domain];
    }
  
    const result: WebsiteCheckResult = {
      hasLlmsTxt: false,
      hasLlmsFullTxt: false
    };
  
    // Create an overall timeout for the entire operation
    const globalTimeout = new Promise<never>((_, reject) => {
      setTimeout(() => {
        reject(new Error('Global timeout exceeded'));
      }, 15000); // 15 second global timeout
    });
  
    try {
      // Normalize domain and add protocol if missing
      let normalizedDomain = domain;
      if (!domain.startsWith('http://') && !domain.startsWith('https://')) {
        normalizedDomain = `https://${domain}`;
      }
      console.error('Normalized domain:', normalizedDomain);
  
      // Validate URL format
      let url: URL;
      try {
        url = new URL(normalizedDomain);
      } catch (e) {
        console.error('Invalid URL:', domain);
        throw new Error(`Invalid URL format: ${domain}`);
      }
  
      // Use the normalized URL
      const baseUrl = url.origin;
      console.error('Base URL:', baseUrl);
  
      // Helper function to fetch with timeout
      async function fetchWithTimeout(url: string, timeout = 5000) { // Reduced to 5 seconds
        console.error(`Fetching ${url} with ${timeout}ms timeout`);
        const controller = new AbortController();
        const timeoutId = setTimeout(() => {
          controller.abort();
          console.error(`Timeout after ${timeout}ms for ${url}`);
        }, timeout);
  
        try {
          const startTime = Date.now();
          const response = await fetch(url, {
            signal: controller.signal,
            headers: {
              'User-Agent': 'llms-txt-explorer/0.1.0'
            }
          });
          const endTime = Date.now();
          console.error(`Fetch completed in ${endTime - startTime}ms for ${url}`);
          clearTimeout(timeoutId);
          return response;
        } catch (error) {
          clearTimeout(timeoutId);
          console.error(`Fetch error for ${url}:`, error);
          throw error;
        }
      }
  
      const checkPromise = (async () => {
        // Check for llms.txt
        try {
          const llmsTxtUrl = `${baseUrl}/llms.txt`;
          console.error('Fetching llms.txt from:', llmsTxtUrl);
          const llmsTxtRes = await fetchWithTimeout(llmsTxtUrl);
          console.error('llms.txt response status:', llmsTxtRes.status);
  
          if (llmsTxtRes.ok) {
            result.hasLlmsTxt = true;
            result.llmsTxtUrl = llmsTxtUrl;
            const content = await llmsTxtRes.text();
            console.error(`llms.txt content length: ${content.length} bytes`);
            result.llmsTxtContent = content;
            console.error('Successfully fetched llms.txt');
  
            // Extract and fetch linked contents in parallel with timeout
            const linkedUrls = extractLinkedUrls(content).slice(0, 3); // Reduced to 3 linked contents
            if (linkedUrls.length > 0) {
              console.error(`Found ${linkedUrls.length} linked URLs in llms.txt (limited to 3)`);
              result.linkedContents = [];
  
              const fetchPromises = linkedUrls.map(async (url) => {
                console.error(`Fetching linked content from: ${url}`);
                try {
                  const linkedRes = await fetchWithTimeout(url);
                  if (!linkedRes.ok) {
                    throw new Error(`Failed to fetch content: ${linkedRes.status}`);
                  }
                  const linkedContent = await linkedRes.text();
                  console.error(`Linked content length: ${linkedContent.length} bytes`);
                  return {
                    url,
                    content: linkedContent
                  };
                } catch (error) {
                  console.error(`Error fetching linked content from ${url}:`, error);
                  return {
                    url,
                    error: error instanceof Error ? error.message : 'Unknown error'
                  };
                }
              });
  
              // Wait for all fetches to complete with a 10 second timeout
              const linkedContentTimeout = new Promise<never>((_, reject) => {
                setTimeout(() => {
                  reject(new Error('Linked content fetch timeout'));
                }, 10000);
              });
  
              try {
                result.linkedContents = await Promise.race([
                  Promise.all(fetchPromises),
                  linkedContentTimeout
                ]);
              } catch (error) {
                console.error('Error fetching linked contents:', error);
                result.linkedContents = linkedUrls.map(url => ({
                  url,
                  error: 'Timeout fetching linked contents'
                }));
              }
            }
          }
        } catch (error: unknown) {
          console.error('Error in main llms.txt fetch:', error);
          if (error instanceof Error) {
            result.error = error.message;
          } else {
            result.error = 'Unknown error fetching llms.txt';
          }
        }
  
        // Only check llms-full.txt if llms.txt was successful
        if (result.hasLlmsTxt && !result.error) {
          try {
            const llmsFullTxtUrl = `${baseUrl}/llms-full.txt`;
            console.error('Fetching llms-full.txt from:', llmsFullTxtUrl);
            const llmsFullTxtRes = await fetchWithTimeout(llmsFullTxtUrl);
            console.error('llms-full.txt response status:', llmsFullTxtRes.status);
  
            if (llmsFullTxtRes.ok) {
              result.hasLlmsFullTxt = true;
              result.llmsFullTxtUrl = llmsFullTxtUrl;
              const content = await llmsFullTxtRes.text();
              console.error(`llms-full.txt content length: ${content.length} bytes`);
              result.llmsFullTxtContent = content;
              console.error('Successfully fetched llms-full.txt');
            }
          } catch (error) {
            console.error('Error fetching llms-full.txt:', error);
            // Don't fail the whole operation for llms-full.txt errors
          }
        }
  
        return result;
      })();
  
      // Race between the check operation and the global timeout
      const finalResult = await Promise.race([checkPromise, globalTimeout]);
  
      // Cache successful results only
      if (!finalResult.error) {
        websiteCheckCache[domain] = finalResult;
      }
  
      console.error('Final result:', JSON.stringify(finalResult, null, 2));
      return finalResult;
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
      console.error('Error checking website:', errorMessage);
      return {
        hasLlmsTxt: false,
        hasLlmsFullTxt: false,
        error: errorMessage
      };
    }
  }

• src/index.ts:389-402 (registration)

  Tool registration in the ListToolsRequestSchema handler, defining name, description, and input schema.

  {
    name: "check_website",
    description: "Check if a website has llms.txt files",
    inputSchema: {
      type: "object",
      properties: {
        url: {
          type: "string",
          description: "URL of the website to check"
        }
      },
      required: ["url"]
    }
  },

• src/index.ts:431-464 (handler)

  MCP tool dispatch handler in CallToolRequestSchema that validates input, calls checkWebsite, and formats the response.

  case "check_website": {
    const url = String(request.params.arguments?.url);
    console.error('Checking website:', url);
  
    if (!url) {
      console.error('URL is required');
      return {
        content: [{
          type: "text",
          text: JSON.stringify({ error: "URL is required" }, null, 2)
        }]
      };
    }
  
    try {
      const result = await checkWebsite(url);
      console.error('Tool returning result:', JSON.stringify(result, null, 2));
      return {
        content: [{
          type: "text",
          text: JSON.stringify(result, null, 2)
        }]
      };
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
      console.error('Tool returning error:', errorMessage);
      return {
        content: [{
          type: "text",
          text: JSON.stringify({ error: errorMessage }, null, 2)
        }]
      };
    }
  }

• src/index.ts:44-53 (schema)

  TypeScript interface defining the output structure of the check_website tool.

  interface WebsiteCheckResult {
    hasLlmsTxt: boolean;
    hasLlmsFullTxt: boolean;
    llmsTxtUrl?: string;
    llmsFullTxtUrl?: string;
    llmsTxtContent?: string;
    llmsFullTxtContent?: string;
    linkedContents?: LinkedContent[];
    error?: string;
  }

• src/index.ts:136-151 (helper)

  Helper function to extract linked URLs from llms.txt content starting with '@'.

  function extractLinkedUrls(content: string): string[] {
    const urls: string[] = [];
    const lines = content.split('\n');
  
    for (const line of lines) {
      const trimmedLine = line.trim();
      if (trimmedLine.startsWith('@')) {
        const url = trimmedLine.slice(1).trim();
        if (url) {
          urls.push(url);
        }
      }
    }
  
    return urls;
  }