#!/usr/bin/env bun
import { SpiderService } from './src/spider/index.js';
import { logger, LogLevel } from './src/utils/logger.js';
// Set verbose logging to see what's happening
logger.setLevel(LogLevel.DEBUG);
async function testSpider() {
const spider = new SpiderService();
// Test URL - using a better documentation site for testing
const testUrl = process.argv[2] || 'https://docs.anthropic.com';
console.log(`\nš·ļø Testing Spider MCP Server on: ${testUrl}\n`);
try {
// Test basic crawling
console.log('š„ Starting crawl...');
const results = await spider.crawlDocs(
testUrl,
2, // maxDepth
['/docs/*', '/api/*'], // includePatterns
[], // excludePatterns - remove the restrictive ones
true, // enableLLMAnalysis
'code_examples' // llmAnalysisType
);
console.log(`\nā
Crawl completed! Found ${results.length} pages\n`);
// Show results
for (const result of results) {
console.log(`š ${result.title}`);
console.log(` URL: ${result.url}`);
console.log(` Content: ${result.content.substring(0, 200)}...`);
console.log(` Words: ${result.metadata.wordCount}`);
console.log(` Links: ${result.metadata.links.length}`);
if (result.llmAnalysis) {
console.log(` š¤ LLM Analysis:`);
console.log(` Summary: ${result.llmAnalysis.summary.substring(0, 150)}...`);
console.log(` Content Type: ${result.llmAnalysis.contentType}`);
console.log(` Key Points: ${result.llmAnalysis.keyPoints.length}`);
console.log(` Relevant Links: ${result.llmAnalysis.relevantLinks.length}`);
console.log(` Code Examples: ${result.llmAnalysis.codeExamples.length}`);
console.log(` Confidence: ${result.llmAnalysis.confidence}`);
if (result.llmAnalysis.codeExamples.length > 0) {
console.log(` š» Code Examples:`);
for (const example of result.llmAnalysis.codeExamples.slice(0, 2)) {
console.log(` - ${example.language}: ${example.description}`);
console.log(` ${example.code.substring(0, 100)}${example.code.length > 100 ? '...' : ''}`);
}
}
} else {
console.log(` š¤ LLM Analysis: Not available (no API key or disabled)`);
}
console.log('');
}
// Test getting a specific page
if (results.length > 0) {
console.log('š Testing get_page...');
const page = await spider.getPage(results[0].url);
if (page) {
console.log(` Retrieved: ${page.title}`);
}
}
// Test search
console.log('š Testing search...');
const searchResults = await spider.searchDocs('anthropic', 3);
console.log(` Found ${searchResults.length} search results`);
for (const result of searchResults) {
console.log(` - ${result.title} (relevance: ${result.relevance.toFixed(2)})`);
}
// Test list pages
console.log('š Testing list_pages...');
const allPages = await spider.listPages();
console.log(` Total cached pages: ${allPages.length}`);
for (const page of allPages.slice(0, 3)) {
console.log(` - ${page.title} (${page.wordCount} words)`);
}
// Show cache stats
console.log('š Cache stats...');
const stats = await spider.getCacheStats();
console.log(` Total entries: ${stats.totalEntries}`);
console.log(` Total size: ${(stats.totalSize / 1024).toFixed(1)} KB`);
console.log('\nš All tests completed successfully!');
} catch (error) {
console.error('ā Error during testing:', error);
console.error('Stack trace:', error.stack);
process.exit(1);
}
}
// Handle graceful shutdown
process.on('SIGINT', () => {
console.log('\nš Shutting down...');
process.exit(0);
});
if (import.meta.main) {
testSpider();
}
