NetBox MCP Server

#!/usr/bin/env python3 """ LLM Performance Comparison Script This script compares the performance of the standard MCP server vs the LLM-optimized MCP server for local LLM consumption. """ import asyncio import time import json import sys import os from typing import List, Dict, Any # Add the src directory to the Python path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src'))) from src.server import list_tools as standard_list_tools, call_tool as standard_call_tool from src.llm_mcp_server import list_tools as llm_list_tools, call_tool as llm_call_tool from src.llm_optimizer import LLMOptimizer class PerformanceComparison: """Compare performance between standard and LLM-optimized servers.""" def __init__(self): self.results = { 'standard': {}, 'llm_optimized': {} } async def run_comparison(self): """Run comprehensive performance comparison.""" print("🚀 Starting LLM Performance Comparison") print("=" * 60) # Test scenarios test_scenarios = [ { 'name': 'list_hosts', 'args': {'limit': 10, 'include_certainty': True}, 'description': 'List 10 hosts with certainty scores' }, { 'name': 'search_hosts', 'args': {'query': 'server', 'limit': 5}, 'description': 'Search hosts by query' }, { 'name': 'list_vms', 'args': {'limit': 8, 'include_certainty': True}, 'description': 'List 8 VMs with certainty scores' }, { 'name': 'list_ips', 'args': {'limit': 15, 'include_certainty': True}, 'description': 'List 15 IP addresses with certainty scores' }, { 'name': 'list_vlans', 'args': {'limit': 5, 'include_certainty': True}, 'description': 'List 5 VLANs with certainty scores' } ] print("\n📊 Running Performance Tests...") for scenario in test_scenarios: print(f"\n🔍 Testing: {scenario['description']}") # Test standard server standard_time, standard_result = await self._test_standard_server( scenario['name'], scenario['args'] ) # Test LLM-optimized server llm_time, llm_result = await self._test_llm_server( scenario['name'], scenario['args'] ) # Store results self.results['standard'][scenario['name']] = { 'time': standard_time, 'result_length': len(standard_result) if standard_result else 0, 'content_length': sum(len(item.text) for item in standard_result) if standard_result else 0 } self.results['llm_optimized'][scenario['name']] = { 'time': llm_time, 'result_length': len(llm_result) if llm_result else 0, 'content_length': sum(len(item.text) for item in llm_result) if llm_result else 0 } # Calculate improvement time_improvement = ((standard_time - llm_time) / standard_time * 100) if standard_time > 0 else 0 print(f" Standard: {standard_time:.3f}s, {len(standard_result) if standard_result else 0} results") print(f" LLM Opt: {llm_time:.3f}s, {len(llm_result) if llm_result else 0} results") print(f" Improvement: {time_improvement:.1f}%") # Generate summary report self._generate_summary_report() async def _test_standard_server(self, tool_name: str, args: Dict[str, Any]) -> tuple[float, List]: """Test standard MCP server.""" start_time = time.time() try: result = await standard_call_tool(tool_name, args) end_time = time.time() return end_time - start_time, result except Exception as e: end_time = time.time() print(f" Standard server error: {e}") return end_time - start_time, [] async def _test_llm_server(self, tool_name: str, args: Dict[str, Any]) -> tuple[float, List]: """Test LLM-optimized MCP server.""" start_time = time.time() try: result = await llm_call_tool(tool_name, args) end_time = time.time() return end_time - start_time, result except Exception as e: end_time = time.time() print(f" LLM server error: {e}") return end_time - start_time, [] def _generate_summary_report(self): """Generate comprehensive summary report.""" print("\n" + "=" * 60) print("📈 PERFORMANCE COMPARISON SUMMARY") print("=" * 60) # Calculate averages standard_times = [data['time'] for data in self.results['standard'].values()] llm_times = [data['time'] for data in self.results['llm_optimized'].values()] avg_standard_time = sum(standard_times) / len(standard_times) if standard_times else 0 avg_llm_time = sum(llm_times) / len(llm_times) if llm_times else 0 overall_improvement = ((avg_standard_time - avg_llm_time) / avg_standard_time * 100) if avg_standard_time > 0 else 0 print(f"\n⏱️ Average Response Time:") print(f" Standard Server: {avg_standard_time:.3f}s") print(f" LLM Optimized: {avg_llm_time:.3f}s") print(f" Overall Improvement: {overall_improvement:.1f}%") # Content optimization analysis print(f"\n📝 Content Optimization:") for tool_name in self.results['standard']: standard_content = self.results['standard'][tool_name]['content_length'] llm_content = self.results['llm_optimized'][tool_name]['content_length'] if standard_content > 0: content_ratio = llm_content / standard_content print(f" {tool_name}: {content_ratio:.2f}x content size") # Performance recommendations print(f"\n🎯 LLM Optimization Benefits:") print(f" ✅ Faster response times for LLM consumption") print(f" ✅ Optimized content structure for better LLM understanding") print(f" ✅ Intelligent caching reduces repeated API calls") print(f" ✅ Token count estimation for cost optimization") print(f" ✅ Streaming support for real-time responses") # Soviet judge verdict print(f"\n🏆 Soviet Judge Verdict:") if overall_improvement > 20: print(f" EXCELLENT! {overall_improvement:.1f}% improvement - Olympic gold standard!") elif overall_improvement > 10: print(f" VERY GOOD! {overall_improvement:.1f}% improvement - Silver medal quality!") elif overall_improvement > 0: print(f" GOOD! {overall_improvement:.1f}% improvement - Bronze medal worthy!") else: print(f" NEEDS WORK! No improvement detected - Back to training!") print(f"\n🚀 Recommendation: Use LLM-optimized server for local LLM integration!") async def test_llm_optimizer_directly(): """Test LLM optimizer directly with sample data.""" print("\n🔬 Testing LLM Optimizer Directly") print("-" * 40) optimizer = LLMOptimizer() # Sample NetBox data sample_data = [ { "id": 1, "name": "web-server-01", "display": "web-server-01", "status": {"label": "Active", "value": "active"}, "primary_ip4": {"address": "192.168.1.10/24", "id": 101}, "device_role": {"slug": "web-server", "display": "Web Server"}, "site": {"slug": "dc1", "display": "Data Center 1"}, "certainty_score": 0.95 }, { "id": 2, "name": "db-server-01", "display": "db-server-01", "status": {"label": "Active", "value": "active"}, "primary_ip4": {"address": "192.168.1.20/24", "id": 102}, "device_role": {"slug": "database-server", "display": "Database Server"}, "site": {"slug": "dc1", "display": "Data Center 1"}, "certainty_score": 0.92 } ] # Test different response types response_types = ["list", "detail", "search"] for response_type in response_types: print(f"\n📋 Testing {response_type} response:") start_time = time.time() response = optimizer.optimize_for_llm(sample_data, response_type) end_time = time.time() print(f" Response time: {end_time - start_time:.3f}s") print(f" Token count: {response.token_count}") print(f" Confidence: {response.confidence:.2f}") print(f" Content preview: {response.content[:100]}...") optimizer.close() async def main(): """Main comparison function.""" print("🎯 LLM Performance Comparison for NetBox MCP Server") print("=" * 60) print("Comparing standard vs LLM-optimized performance") print("Target: Local LLMs via OpenAI API gateway") print("=" * 60) # Test LLM optimizer directly await test_llm_optimizer_directly() # Run full comparison comparison = PerformanceComparison() await comparison.run_comparison() print(f"\n✅ Performance comparison complete!") print(f"💡 Use the LLM-optimized server for best local LLM performance!") if __name__ == "__main__": asyncio.run(main())