Spider MCP

import express from 'express'; import axios from 'axios'; import searchService from '../services/searchService.js'; import { logger } from '../utils/logger.js'; const router = express.Router(); // General search endpoint - currently only supports Bing router.post('/multi', async (req, res) => { try { const { query, engines = ['bing'], maxResults = 10 } = req.body; if (!query || typeof query !== 'string') { return res.status(400).json({ error: 'Invalid query parameter', message: 'Query must be a non-empty string' }); } if (maxResults < 1 || maxResults > 50) { return res.status(400).json({ error: 'Invalid maxResults parameter', message: 'maxResults must be between 1 and 50' }); } logger.info(`Multi-search request received: "${query}" with engines: ${engines.join(', ')}`); const results = await searchService.multiSearch(query, engines, maxResults); res.json({ success: true, data: results }); } catch (error) { logger.error('Multi-search error:', error); res.status(500).json({ error: 'Search failed', message: error.message }); } }); // Google search endpoint router.get('/google', async (req, res) => { try { const { q: query, max = 10 } = req.query; if (!query) { return res.status(400).json({ error: 'Missing query parameter', message: 'Query parameter "q" is required' }); } if (max < 1 || max > 20) { return res.status(400).json({ error: 'Invalid max parameter', message: 'max must be between 1 and 20' }); } logger.info(`Google search request received: "${query}"`); const results = await searchService.searchGoogle(query, parseInt(max)); res.json({ success: true, data: results }); } catch (error) { logger.error('Google search error:', error); res.status(500).json({ error: 'Google search failed', message: error.message }); } }); // Bing search endpoint (supports web and news search) router.get('/bing', async (req, res) => { try { const { q: query, max = 10, type = 'web', time = 'past_24_hours' } = req.query; if (!query) { return res.status(400).json({ error: 'Missing query parameter', message: 'Query parameter "q" is required' }); } if (max < 1 || max > 20) { return res.status(400).json({ error: 'Invalid max parameter', message: 'max must be between 1 and 20' }); } if (!['web', 'news'].includes(type)) { return res.status(400).json({ error: 'Invalid type parameter', message: 'type must be "web" or "news"' }); } if (type === 'news' && !['past_hour', 'past_24_hours', 'past_7_days', 'past_30_days'].includes(time)) { return res.status(400).json({ error: 'Invalid time parameter', message: 'time must be one of: past_hour, past_24_hours, past_7_days, past_30_days' }); } logger.info(`Bing ${type} search request received: "${query}"${type === 'news' ? ` with time filter: ${time}` : ''}`); let results; if (type === 'news') { results = await searchService.searchBingNews(query, parseInt(max), time); } else { results = await searchService.searchBing(query, parseInt(max)); } res.json({ success: true, data: results }); } catch (error) { logger.error('Bing search error:', error); res.status(500).json({ error: 'Bing search failed', message: error.message }); } }); // DuckDuckGo search endpoint router.get('/duckduckgo', async (req, res) => { try { const { q: query, max = 10 } = req.query; if (!query) { return res.status(400).json({ error: 'Missing query parameter', message: 'Query parameter "q" is required' }); } if (max < 1 || max > 20) { return res.status(400).json({ error: 'Invalid max parameter', message: 'max must be between 1 and 20' }); } logger.info(`DuckDuckGo search request received: "${query}"`); const results = await searchService.searchDuckDuckGo(query, parseInt(max)); res.json({ success: true, data: results }); } catch (error) { logger.error('DuckDuckGo search error:', error); res.status(500).json({ error: 'DuckDuckGo search failed', message: error.message }); } }); // Webpage scraping endpoint router.post('/scrape', async (req, res) => { try { const { url } = req.body; if (!url || typeof url !== 'string') { return res.status(400).json({ error: 'Invalid URL parameter', message: 'URL must be a valid string' }); } // Simple URL validation try { new URL(url); } catch (error) { return res.status(400).json({ error: 'Invalid URL format', message: 'Please provide a valid URL' }); } logger.info(`Webpage scraping request received for: ${url}`); const results = await searchService.scrapeWebpage(url); res.json({ success: true, data: results }); } catch (error) { logger.error('Webpage scraping error:', error); res.status(500).json({ error: 'Webpage scraping failed', message: error.message }); } }); // Batch webpage scraping endpoint router.post('/scrape/batch', async (req, res) => { try { const { urls, maxConcurrent = 3 } = req.body; if (!Array.isArray(urls) || urls.length === 0) { return res.status(400).json({ error: 'Invalid URLs parameter', message: 'URLs must be a non-empty array' }); } if (urls.length > 20) { return res.status(400).json({ error: 'Too many URLs', message: 'Maximum 20 URLs allowed per batch' }); } logger.info(`Batch webpage scraping request received for ${urls.length} URLs`); const results = []; const errors = []; // Process in batches to avoid opening too many connections at once for (let i = 0; i < urls.length; i += maxConcurrent) { const batch = urls.slice(i, i + maxConcurrent); const batchPromises = batch.map(async (url, index) => { try { const result = await searchService.scrapeWebpage(url); return { success: true, data: result }; } catch (error) { return { success: false, url, error: error.message }; } }); const batchResults = await Promise.allSettled(batchPromises); batchResults.forEach((result, index) => { if (result.status === 'fulfilled') { if (result.value.success) { results.push(result.value.data); } else { errors.push(result.value); } } else { errors.push({ url: batch[index], error: result.reason?.message || 'Unknown error' }); } }); } res.json({ success: true, data: { totalUrls: urls.length, successful: results.length, failed: errors.length, results, errors } }); } catch (error) { logger.error('Batch webpage scraping error:', error); res.status(500).json({ error: 'Batch webpage scraping failed', message: error.message }); } }); // Search suggestions endpoint router.get('/suggestions', async (req, res) => { try { const { q: query, engine = 'bing' } = req.query; if (!query) { return res.status(400).json({ error: 'Missing query parameter', message: 'Query parameter "q" is required' }); } if (!['bing'].includes(engine)) { return res.status(400).json({ error: 'Invalid engine parameter', message: 'Engine must be "bing"' }); } logger.info(`Search suggestions request received: "${query}" from ${engine}`); // Here you can implement real search suggestions // Currently returns simple suggestions const suggestions = [ query, `${query} tutorial`, `${query} examples`, `${query} guide`, `${query} documentation` ]; res.json({ success: true, data: { query, engine, suggestions, timestamp: new Date().toISOString() } }); } catch (error) { logger.error('Search suggestions error:', error); res.status(500).json({ error: 'Search suggestions failed', message: error.message }); } }); // Get webpage source and convert to Markdown router.post('/markdown', async (req, res) => { try { const { url } = req.body; if (!url || typeof url !== 'string') { return res.status(400).json({ error: 'Invalid URL parameter', message: 'URL must be a valid string' }); } // Simple URL validation try { new URL(url); } catch (error) { return res.status(400).json({ error: 'Invalid URL format', message: 'Please provide a valid URL' }); } logger.info(`Markdown conversion request received for: ${url}`); const results = await searchService.getWebpageMarkdown(url); res.json({ success: true, data: results }); } catch (error) { logger.error('Markdown conversion error:', error); res.status(500).json({ error: 'Markdown conversion failed', message: error.message }); } }); // Get webpage source (raw HTML) router.post('/source', async (req, res) => { try { const { url } = req.body; if (!url || typeof url !== 'string') { return res.status(400).json({ error: 'Invalid URL parameter', message: 'URL must be a valid string' }); } // Simple URL validation try { new URL(url); } catch (error) { return res.status(400).json({ error: 'Invalid URL format', message: 'Please provide a valid URL' }); } logger.info(`Source code request received for: ${url}`); const response = await axios.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive' }, timeout: 15000 }); res.json({ success: true, data: { url, htmlSource: response.data, contentType: response.headers['content-type'], timestamp: new Date().toISOString() } }); } catch (error) { logger.error('Source code retrieval error:', error); res.status(500).json({ error: 'Source code retrieval failed', message: error.message }); } }); export { router as searchRouter };