NCP - Natural Context Provider

#!/usr/bin/env node /** * Real MCP Analyzer * * Discovers, downloads, and analyzes real MCP packages to extract: * - MCP name, version, description * - Tool names, descriptions, parameters * - Input schemas and tool metadata * * Sources: * 1. npm registry search for MCP packages * 2. GitHub MCP repositories * 3. Official MCP registry/marketplace * 4. Popular MCP collections */ import * as fs from 'fs/promises'; import * as path from 'path'; import { fileURLToPath } from 'url'; import { spawn } from 'child_process'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); interface RealMcpTool { name: string; description: string; inputSchema: any; } interface RealMcpDefinition { name: string; version: string; description: string; category: string; packageName: string; npmDownloads?: number; githubStars?: number; tools: Record<string, RealMcpTool>; metadata: { source: 'npm' | 'github' | 'registry'; homepage?: string; repository?: string; discoveredAt: string; }; } interface McpDiscoveryResult { mcps: Record<string, RealMcpDefinition>; stats: { totalFound: number; withTools: number; categories: Record<string, number>; sources: Record<string, number>; }; } class RealMcpAnalyzer { private tempDir: string; private outputPath: string; constructor() { this.tempDir = path.join(__dirname, 'temp-mcp-analysis'); this.outputPath = path.join(__dirname, 'real-mcp-definitions.json'); } /** * Discover real MCPs from multiple sources */ async discoverMcps(targetCount: number = 100): Promise<McpDiscoveryResult> { console.log(`🔍 Discovering top ${targetCount} real MCPs...`); await fs.mkdir(this.tempDir, { recursive: true }); const results: Record<string, RealMcpDefinition> = {}; let totalAnalyzed = 0; try { // 1. Search npm registry for MCP packages console.log('\n📦 Searching npm registry for MCP packages...'); const npmMcps = await this.searchNpmMcps(targetCount); for (const npmMcp of npmMcps) { if (totalAnalyzed >= targetCount) break; console.log(` 📥 Analyzing: ${npmMcp.name}`); const analyzed = await this.analyzeMcpPackage(npmMcp); if (analyzed && analyzed.tools && Object.keys(analyzed.tools).length > 0) { results[analyzed.name] = analyzed; totalAnalyzed++; console.log(` ✅ Found ${Object.keys(analyzed.tools).length} tools`); } else { console.log(` ⚠️ No tools found or analysis failed`); } } // 2. Search GitHub for MCP repositories if (totalAnalyzed < targetCount) { console.log('\n🐙 Searching GitHub for MCP repositories...'); const githubMcps = await this.searchGitHubMcps(targetCount - totalAnalyzed); for (const githubMcp of githubMcps) { if (totalAnalyzed >= targetCount) break; console.log(` 📥 Analyzing: ${githubMcp.name}`); const analyzed = await this.analyzeGitHubMcp(githubMcp); if (analyzed && analyzed.tools && Object.keys(analyzed.tools).length > 0) { results[analyzed.name] = analyzed; totalAnalyzed++; console.log(` ✅ Found ${Object.keys(analyzed.tools).length} tools`); } else { console.log(` ⚠️ No tools found or analysis failed`); } } } // 3. Add well-known MCPs if we still need more if (totalAnalyzed < targetCount) { console.log('\n🎯 Adding well-known MCPs...'); const wellKnownMcps = await this.getWellKnownMcps(); for (const wellKnownMcp of wellKnownMcps) { if (totalAnalyzed >= targetCount) break; if (results[wellKnownMcp.name]) continue; // Skip duplicates results[wellKnownMcp.name] = wellKnownMcp; totalAnalyzed++; console.log(` ✅ Added: ${wellKnownMcp.name} (${Object.keys(wellKnownMcp.tools).length} tools)`); } } } catch (error: any) { console.error(`Error during MCP discovery: ${error.message}`); } // Generate stats const stats = this.generateStats(results); const result: McpDiscoveryResult = { mcps: results, stats }; // Save results await fs.writeFile(this.outputPath, JSON.stringify(result, null, 2)); console.log(`\n📊 Discovery Results:`); console.log(` Total MCPs found: ${stats.totalFound}`); console.log(` MCPs with tools: ${stats.withTools}`); console.log(` Categories: ${Object.keys(stats.categories).join(', ')}`); console.log(` Saved to: ${this.outputPath}`); return result; } /** * Search npm registry for packages containing "mcp" in name or keywords */ private async searchNpmMcps(limit: number): Promise<any[]> { const searchQueries = [ 'mcp-server', 'model-context-protocol', 'mcp client', 'anthropic mcp', 'claude mcp' ]; const results: any[] = []; for (const query of searchQueries) { if (results.length >= limit) break; try { console.log(` 🔎 Searching npm for: "${query}"`); const searchOutput = await this.runCommand('npm', ['search', query, '--json'], { timeout: 30000 }); const packages = JSON.parse(searchOutput); for (const pkg of packages) { if (results.length >= limit) break; if (this.isMcpPackage(pkg)) { results.push({ name: pkg.name.replace(/^@[^/]+\//, '').replace(/[-_]?mcp[-_]?/i, ''), packageName: pkg.name, version: pkg.version, description: pkg.description || '', npmDownloads: pkg.popularity || 0, source: 'npm' }); } } } catch (error) { console.log(` ⚠️ Search failed for "${query}"`); } } return results.slice(0, limit); } /** * Check if package is likely an MCP */ private isMcpPackage(pkg: any): boolean { const name = pkg.name.toLowerCase(); const desc = (pkg.description || '').toLowerCase(); const keywords = (pkg.keywords || []).map((k: string) => k.toLowerCase()); const mcpIndicators = [ 'mcp', 'model-context-protocol', 'claude', 'anthropic', 'mcp-server', 'context-protocol', 'tool-server' ]; return mcpIndicators.some(indicator => name.includes(indicator) || desc.includes(indicator) || keywords.some((k: string) => k.includes(indicator)) ); } /** * Search GitHub for MCP repositories */ private async searchGitHubMcps(limit: number): Promise<any[]> { // For now, return empty array - would need GitHub API integration // This would search for repos with topics: model-context-protocol, mcp-server, etc. console.log(' 📝 GitHub search not implemented yet - would use GitHub API'); return []; } /** * Analyze a GitHub MCP repository */ private async analyzeGitHubMcp(githubMcp: any): Promise<RealMcpDefinition | null> { // For now, return null - would clone and analyze repo return null; } /** * Analyze an npm MCP package */ private async analyzeMcpPackage(mcpInfo: any): Promise<RealMcpDefinition | null> { try { // Install package temporarily const packagePath = path.join(this.tempDir, mcpInfo.packageName.replace(/[@/]/g, '_')); await fs.mkdir(packagePath, { recursive: true }); console.log(` 📦 Installing ${mcpInfo.packageName}...`); await this.runCommand('npm', ['install', mcpInfo.packageName], { cwd: packagePath, timeout: 60000 }); // Try to find and analyze MCP definition const tools = await this.extractToolsFromPackage(packagePath, mcpInfo.packageName); if (!tools || Object.keys(tools).length === 0) { return null; } const definition: RealMcpDefinition = { name: mcpInfo.name, version: mcpInfo.version, description: mcpInfo.description, category: this.categorizePackage(mcpInfo.description, Object.keys(tools)), packageName: mcpInfo.packageName, npmDownloads: mcpInfo.npmDownloads, tools, metadata: { source: 'npm', discoveredAt: new Date().toISOString() } }; return definition; } catch (error: any) { console.log(` ❌ Analysis failed: ${error.message}`); return null; } } /** * Extract tools from installed package */ private async extractToolsFromPackage(packagePath: string, packageName: string): Promise<Record<string, RealMcpTool> | null> { try { // Look for common MCP server files const possiblePaths = [ path.join(packagePath, 'node_modules', packageName, 'dist', 'index.js'), path.join(packagePath, 'node_modules', packageName, 'src', 'index.js'), path.join(packagePath, 'node_modules', packageName, 'index.js'), path.join(packagePath, 'node_modules', packageName, 'server.js') ]; for (const filePath of possiblePaths) { try { await fs.access(filePath); // Found a file, try to extract tools const tools = await this.analyzeServerFile(filePath); if (tools && Object.keys(tools).length > 0) { return tools; } } catch { // File doesn't exist, try next continue; } } return null; } catch (error) { return null; } } /** * Analyze server file to extract tool definitions */ private async analyzeServerFile(filePath: string): Promise<Record<string, RealMcpTool> | null> { try { // For now, return null - would need to safely execute/analyze the MCP server // This would involve running the MCP server and introspecting its tools console.log(` 🔍 Would analyze: ${filePath}`); return null; } catch { return null; } } /** * Get well-known MCPs with manually curated definitions */ private async getWellKnownMcps(): Promise<RealMcpDefinition[]> { // Return our current high-quality definitions as "well-known" MCPs // These are based on real MCP patterns and serve as seed data return [ { name: 'filesystem', version: '1.0.0', description: 'Local file system operations including reading, writing, and directory management', category: 'file-operations', packageName: '@modelcontextprotocol/server-filesystem', tools: { 'read_file': { name: 'read_file', description: 'Read contents of a file from the filesystem', inputSchema: { type: 'object', properties: { path: { type: 'string', description: 'Path to the file to read' } }, required: ['path'] } }, 'write_file': { name: 'write_file', description: 'Write content to a file on the filesystem', inputSchema: { type: 'object', properties: { path: { type: 'string', description: 'Path to write the file' }, content: { type: 'string', description: 'Content to write to the file' } }, required: ['path', 'content'] } }, 'list_directory': { name: 'list_directory', description: 'List contents of a directory', inputSchema: { type: 'object', properties: { path: { type: 'string', description: 'Path to the directory to list' } }, required: ['path'] } } }, metadata: { source: 'registry', discoveredAt: new Date().toISOString() } } // Would add more well-known MCPs here ]; } /** * Categorize package based on description and tools */ private categorizePackage(description: string, toolNames: string[]): string { const desc = description.toLowerCase(); const tools = toolNames.join(' ').toLowerCase(); if (desc.includes('database') || desc.includes('sql') || tools.includes('query')) return 'database'; if (desc.includes('file') || tools.includes('read') || tools.includes('write')) return 'file-operations'; if (desc.includes('web') || desc.includes('http') || desc.includes('api')) return 'web-services'; if (desc.includes('cloud') || desc.includes('aws') || desc.includes('gcp')) return 'cloud-infrastructure'; if (desc.includes('git') || desc.includes('version')) return 'developer-tools'; if (desc.includes('ai') || desc.includes('llm') || desc.includes('model')) return 'ai-ml'; if (desc.includes('search') || desc.includes('index')) return 'search'; if (desc.includes('message') || desc.includes('chat') || desc.includes('slack')) return 'communication'; return 'other'; } /** * Generate discovery statistics */ private generateStats(mcps: Record<string, RealMcpDefinition>) { const stats = { totalFound: Object.keys(mcps).length, withTools: 0, categories: {} as Record<string, number>, sources: {} as Record<string, number> }; for (const mcp of Object.values(mcps)) { if (mcp.tools && Object.keys(mcp.tools).length > 0) { stats.withTools++; } stats.categories[mcp.category] = (stats.categories[mcp.category] || 0) + 1; stats.sources[mcp.metadata.source] = (stats.sources[mcp.metadata.source] || 0) + 1; } return stats; } /** * Run shell command with timeout */ private async runCommand(command: string, args: string[], options: { cwd?: string; timeout?: number } = {}): Promise<string> { return new Promise((resolve, reject) => { const child = spawn(command, args, { cwd: options.cwd || process.cwd(), stdio: ['pipe', 'pipe', 'pipe'] }); let stdout = ''; let stderr = ''; child.stdout.on('data', (data) => stdout += data.toString()); child.stderr.on('data', (data) => stderr += data.toString()); const timeout = setTimeout(() => { child.kill(); reject(new Error(`Command timeout after ${options.timeout}ms`)); }, options.timeout || 30000); child.on('close', (code) => { clearTimeout(timeout); if (code === 0) { resolve(stdout.trim()); } else { reject(new Error(`Command failed with code ${code}: ${stderr}`)); } }); child.on('error', (error) => { clearTimeout(timeout); reject(error); }); }); } /** * Clean up temporary files */ async cleanup(): Promise<void> { try { await fs.rm(this.tempDir, { recursive: true, force: true }); console.log('🧹 Cleaned up temporary files'); } catch (error) { console.log('⚠️ Cleanup warning: could not remove temp directory'); } } } // CLI interface async function main() { const analyzer = new RealMcpAnalyzer(); const targetCount = parseInt(process.argv[2]) || 100; console.log(`🚀 Starting Real MCP Analysis (target: ${targetCount} MCPs)`); try { const results = await analyzer.discoverMcps(targetCount); console.log('\n✅ Analysis Complete!'); console.log(` Found ${results.stats.totalFound} real MCPs`); console.log(` ${results.stats.withTools} have discoverable tools`); console.log(` Categories: ${Object.keys(results.stats.categories).join(', ')}`); } catch (error: any) { console.error('❌ Analysis failed:', error.message); } finally { await analyzer.cleanup(); } } if (import.meta.url === `file://${process.argv[1]}`) { main(); } export { RealMcpAnalyzer };