code2mcp
by blas0
- code2mcp
- test
token-comparison.ts•12.3 kB
#!/usr/bin/env tsx
/**
* Token Comparison Test: Scenario A vs Scenario B
*
* Scenario A: Load all MCP servers directly into Claude Code
* Scenario B: Load only code2mcp MCP server
*
* Measures:
* - Preload token usage
* - Tool usage token usage
* - Speed (ms)
*/
import { MCPOrchestrator } from '../src/orchestrator/MCPOrchestrator.js';
import type { MCPServerConfig } from '../src/types/index.js';
const MCP_SERVERS: MCPServerConfig[] = [
{
name: 'context7',
transport: 'stdio',
command: 'npx',
args: ['-y', '@upstash/context7-mcp'],
env: {},
},
{
name: 'playwright',
transport: 'stdio',
command: 'npx',
args: ['-y', '@executeautomation/playwright-mcp-server'],
env: {},
},
{
name: 'bright-data',
transport: 'sse',
url: 'https://mcp.brightdata.com/sse?token=83ca9984ed40d7edc01326acf1c4326c3b09ea906db764a56e1d0f421b1c22d5',
},
{
name: 'chrome-devtools',
transport: 'stdio',
command: 'npx',
args: ['-y', '@modelcontextprotocol/server-chrome-devtools'],
env: {},
},
{
name: 'firecrawl-mcp',
transport: 'stdio',
command: 'npx',
args: ['-y', 'firecrawl-mcp'],
env: {
FIRECRAWL_API_KEY: 'fc-634d95ad14754322904098823deac29b'
},
},
];
// Simple token counter (approximation: 1 token ≈ 4 characters)
function estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
}
// Generate tool description for direct MCP loading
function generateDirectMCPDescription(tools: any[]): string {
let description = '';
for (const tool of tools) {
const params = tool.inputSchema?.properties
? Object.entries(tool.inputSchema.properties).map(([name, schema]: [string, any]) => {
const type = schema.type || 'any';
const required = tool.inputSchema?.required?.includes(name) ? '' : '?';
return `${name}${required}: ${type}`;
})
: [];
description += `## ${tool.name}\n`;
description += `${tool.description || 'No description'}\n`;
if (params.length > 0) {
description += `**Parameters:** ${params.join(', ')}\n`;
}
description += `\n`;
}
return description;
}
// Generate code2mcp execute_code description
function generateCode2MCPDescription(tools: any[]): string {
let description = `# Execute TypeScript Code with MCP Tool Access
You can execute TypeScript code that calls MCP tools using the execute_code tool.
**Available APIs:**
`;
// Group tools by server
const serverTools = new Map<string, any[]>();
for (const tool of tools) {
const serverName = tool.name.split('__')[0];
if (!serverTools.has(serverName)) {
serverTools.set(serverName, []);
}
serverTools.get(serverName)!.push(tool);
}
for (const [serverName, serverToolList] of serverTools) {
description += `\n**${serverName}**:\n`;
for (const tool of serverToolList) {
const params = tool.inputSchema?.properties
? Object.entries(tool.inputSchema.properties).map(([name, schema]: [string, any]) => {
const type = schema.type || 'any';
return `${name}: value`;
})
: [];
description += ` - \`__mcp_call('${tool.name}', {${params.join(', ')}})\`\n`;
description += ` ${tool.description || 'No description'}\n`;
}
}
description += `\n**Example Usage:**
\`\`\`typescript
const result = await __mcp_call('context7__resolve-library-id', {
libraryName: 'react'
});
console.log('Result:', result);
\`\`\`
`;
return description;
}
// Simulate Scenario A: Direct MCP loading
async function scenarioA() {
console.log('\n📊 SCENARIO A: Direct MCP Server Loading\n');
console.log('Loading all 5 MCP servers...');
const startTime = Date.now();
const orchestrator = new MCPOrchestrator();
// Connect to all servers
for (const serverConfig of MCP_SERVERS) {
try {
await orchestrator.connectServer(serverConfig);
console.log(` ✅ Connected to ${serverConfig.name}`);
} catch (error: any) {
console.log(` ❌ Failed to connect to ${serverConfig.name}: ${error.message}`);
}
}
const allTools = orchestrator.getAllTools();
console.log(`\nTotal tools available: ${allTools.length}`);
// Generate preload description
const preloadDescription = generateDirectMCPDescription(allTools);
const preloadTokens = estimateTokens(preloadDescription);
console.log(`\n📝 Preload Stats:`);
console.log(` - Description length: ${preloadDescription.length} chars`);
console.log(` - Estimated tokens: ${preloadTokens} tokens`);
// Simulate user prompt: "get me documentation on shadcn"
// In Scenario A, Claude would need to:
// 1. Decide which tool to use (context7__resolve-library-id)
// 2. Make tool call with JSON parameters
// 3. Get library ID
// 4. Make another tool call to get-library-docs
// 5. Return result
const userPrompt = "get me documentation on shadcn";
const toolCallsNeeded = [
{
tool: 'context7__resolve-library-id',
input: { libraryName: 'shadcn' },
estimatedSize: 50 // tokens
},
{
tool: 'context7__get-library-docs',
input: { context7CompatibleLibraryID: '/shadcn/ui', topic: 'getting started' },
estimatedSize: 80 // tokens
}
];
const toolUsageTokens = toolCallsNeeded.reduce((sum, call) => sum + call.estimatedSize, 0);
const totalTokens = preloadTokens + estimateTokens(userPrompt) + toolUsageTokens;
const endTime = Date.now();
const duration = endTime - startTime;
console.log(`\n📊 Tool Usage Stats:`);
console.log(` - User prompt tokens: ${estimateTokens(userPrompt)}`);
console.log(` - Tool calls needed: ${toolCallsNeeded.length}`);
console.log(` - Tool call tokens: ${toolUsageTokens}`);
console.log(`\n📈 Total Stats:`);
console.log(` - Total tokens: ${totalTokens}`);
console.log(` - Duration: ${duration}ms`);
await orchestrator.disconnect();
return {
preloadTokens,
toolUsageTokens,
totalTokens,
duration,
toolCallsNeeded: toolCallsNeeded.length,
description: preloadDescription
};
}
// Simulate Scenario B: code2mcp
async function scenarioB() {
console.log('\n📊 SCENARIO B: Code2MCP Approach\n');
console.log('Loading code2mcp orchestrator...');
const startTime = Date.now();
const orchestrator = new MCPOrchestrator();
// Connect to all servers (code2mcp does this internally)
for (const serverConfig of MCP_SERVERS) {
try {
await orchestrator.connectServer(serverConfig);
console.log(` ✅ Connected to ${serverConfig.name}`);
} catch (error: any) {
console.log(` ❌ Failed to connect to ${serverConfig.name}: ${error.message}`);
}
}
const allTools = orchestrator.getAllTools();
console.log(`\nTotal tools available: ${allTools.length}`);
// Generate code2mcp preload description
const preloadDescription = generateCode2MCPDescription(allTools);
const preloadTokens = estimateTokens(preloadDescription);
console.log(`\n📝 Preload Stats:`);
console.log(` - Description length: ${preloadDescription.length} chars`);
console.log(` - Estimated tokens: ${preloadTokens} tokens`);
// Simulate user prompt: "get me documentation on shadcn"
// In Scenario B, Claude would:
// 1. Write TypeScript code that calls __mcp_call twice
// 2. Execute the code once
// 3. Return result
const userPrompt = "get me documentation on shadcn";
// The TypeScript code Claude would write:
const generatedCode = `
// Resolve shadcn library ID
const libResult = await __mcp_call('context7__resolve-library-id', {
libraryName: 'shadcn'
});
const libraryId = libResult.libraries[0].id;
// Get documentation
const docs = await __mcp_call('context7__get-library-docs', {
context7CompatibleLibraryID: libraryId,
topic: 'getting started'
});
console.log('Documentation:', docs);
`;
const codeTokens = estimateTokens(generatedCode);
const totalTokens = preloadTokens + estimateTokens(userPrompt) + codeTokens;
const endTime = Date.now();
const duration = endTime - startTime;
console.log(`\n📊 Tool Usage Stats:`);
console.log(` - User prompt tokens: ${estimateTokens(userPrompt)}`);
console.log(` - Generated code tokens: ${codeTokens}`);
console.log(` - Tool calls made: 1 (execute_code with embedded calls)`);
console.log(`\n📈 Total Stats:`);
console.log(` - Total tokens: ${totalTokens}`);
console.log(` - Duration: ${duration}ms`);
await orchestrator.disconnect();
return {
preloadTokens,
toolUsageTokens: codeTokens,
totalTokens,
duration,
toolCallsNeeded: 1,
description: preloadDescription
};
}
// Run comparison
async function runComparison() {
console.log('🔬 TOKEN COMPARISON TEST');
console.log('='.repeat(60));
console.log('Testing: "get me documentation on shadcn"\n');
try {
const resultA = await scenarioA();
const resultB = await scenarioB();
console.log('\n' + '='.repeat(60));
console.log('📊 COMPARISON RESULTS');
console.log('='.repeat(60));
console.log('\n┌─────────────────────────────┬──────────────┬──────────────┐');
console.log('│ Metric │ Scenario A │ Scenario B │');
console.log('├─────────────────────────────┼──────────────┼──────────────┤');
console.log(`│ Preload Tokens │ ${String(resultA.preloadTokens).padStart(12)} │ ${String(resultB.preloadTokens).padStart(12)} │`);
console.log(`│ Tool Usage Tokens │ ${String(resultA.toolUsageTokens).padStart(12)} │ ${String(resultB.toolUsageTokens).padStart(12)} │`);
console.log(`│ Total Tokens │ ${String(resultA.totalTokens).padStart(12)} │ ${String(resultB.totalTokens).padStart(12)} │`);
console.log(`│ Tool Calls Needed │ ${String(resultA.toolCallsNeeded).padStart(12)} │ ${String(resultB.toolCallsNeeded).padStart(12)} │`);
console.log(`│ Duration (ms) │ ${String(resultA.duration).padStart(12)} │ ${String(resultB.duration).padStart(12)} │`);
console.log('└─────────────────────────────┴──────────────┴──────────────┘');
// Calculate savings
const tokenSavings = resultA.totalTokens - resultB.totalTokens;
const tokenSavingsPercent = ((tokenSavings / resultA.totalTokens) * 100).toFixed(1);
const speedDiff = resultA.duration - resultB.duration;
console.log('\n📈 ANALYSIS:');
console.log(` • Token Savings: ${tokenSavings > 0 ? '+' : ''}${tokenSavings} tokens (${tokenSavingsPercent}%)`);
console.log(` • Speed Difference: ${speedDiff > 0 ? 'Scenario B faster by' : 'Scenario A faster by'} ${Math.abs(speedDiff)}ms`);
if (resultB.totalTokens < resultA.totalTokens) {
console.log('\n✅ RECOMMENDATION: Code2MCP (Scenario B) is more efficient');
console.log(' - Uses fewer tokens overall');
console.log(' - Fewer tool calls needed');
console.log(' - Better for complex multi-step workflows');
} else {
console.log('\n⚠️ RECOMMENDATION: Direct MCP loading (Scenario A) might be better');
console.log(' - Uses fewer preload tokens');
console.log(' - Simpler architecture');
}
// Save results to file
const results = {
timestamp: new Date().toISOString(),
scenarioA: resultA,
scenarioB: resultB,
analysis: {
tokenSavings,
tokenSavingsPercent: parseFloat(tokenSavingsPercent),
speedDiff
}
};
await import('fs/promises').then(fs =>
fs.writeFile(
'/Users/blaser/Documents/Projects/code2mcp/test/comparison-results.json',
JSON.stringify(results, null, 2)
)
);
console.log('\n💾 Results saved to test/comparison-results.json');
} catch (error: any) {
console.error('\n❌ Test failed:', error.message);
console.error(error.stack);
process.exit(1);
}
}
runComparison();
Latest Blog Posts
- MCP Moves to the Linux Foundation: Neutral Stewardship for Agentic InfrastructureBy Om-Shree-0709 on .mcpanthropicLinux Foundation
MCP directory API
We provide all the information about MCP servers via our MCP API.
curl -X GET 'https://glama.ai/api/mcp/v1/servers/blas0/code2mcp'
If you have feedback or need assistance with the MCP directory API, please join our Discord server