OpenRouter MCP Multimodal Server

Instructions

Implementation Reference

• src/tool-handlers/multi-image-analysis.ts:342-546 (handler)

  Core handler function that fetches images from various sources (URLs, files, data), processes/resizes them (using sharp with fallback), constructs OpenAI chat completion request with multiple images, calls OpenRouter vision models with automatic fallback to free models, and returns formatted analysis.
  export async function handleMultiImageAnalysis( request: { params: { arguments: MultiImageAnalysisToolRequest } }, openai: OpenAI, defaultModel?: string ) { const args = request.params.arguments; try { // Validate inputs if (!args.images || !Array.isArray(args.images) || args.images.length === 0) { throw new McpError(ErrorCode.InvalidParams, 'At least one image is required'); } if (!args.prompt) { throw new McpError(ErrorCode.InvalidParams, 'A prompt for analyzing the images is required'); } console.error(`Processing ${args.images.length} images`); // Process each image and convert to base64 if needed const processedImages = await Promise.all( args.images.map(async (image, index) => { try { // Skip processing if already a data URL if (image.url.startsWith('data:')) { console.error(`Image ${index + 1} is already in base64 format`); return image; } console.error(`Processing image ${index + 1}: ${image.url.substring(0, 100)}${image.url.length > 100 ? '...' : ''}`); // Get MIME type const mimeType = getMimeType(image.url); // Fetch and process the image const buffer = await fetchImageAsBuffer(image.url); const base64 = await processImage(buffer, mimeType); return { url: `data:${mimeType === 'application/octet-stream' ? 'image/jpeg' : mimeType};base64,${base64}`, alt: image.alt }; } catch (error: any) { console.error(`Error processing image ${index + 1}:`, error); throw new Error(`Failed to process image ${index + 1}: ${image.url}. Error: ${error.message}`); } }) ); // Select model with priority: // 1. User-specified model // 2. Default model from environment let model = args.model || defaultModel || DEFAULT_FREE_MODEL; console.error(`[Multi-Image Tool] Using IMAGE model: ${model}`); // Build content array for the API call const content: Array<{ type: string; text?: string; image_url?: { url: string } }> = [ { type: 'text', text: args.prompt } ]; // Add each processed image to the content array processedImages.forEach(image => { content.push({ type: 'image_url', image_url: { url: image.url } }); }); // Try primary model first let responseText: string; let usedModel: string; let usage: any; try { const completion = await openai.chat.completions.create({ model, messages: [{ role: 'user', content }] as any }); responseText = completion.choices[0].message.content || ''; usedModel = completion.model; usage = completion.usage; } catch (primaryError: any) { // If primary model fails and backup exists, try backup const backupModel = process.env.OPENROUTER_DEFAULT_MODEL_IMG_BACKUP; if (backupModel && backupModel !== model) { try { console.error(`Primary model failed, trying backup: ${backupModel}`); const completion = await openai.chat.completions.create({ model: backupModel, messages: [{ role: 'user', content }] as any }); responseText = completion.choices[0].message.content || ''; usedModel = completion.model; usage = completion.usage; } catch (backupError: any) { console.error(`Backup model failed, searching for free models...`); // Try to find a free model const freeModel = await findSuitableFreeModel(openai); if (freeModel && freeModel !== model && freeModel !== backupModel) { console.error(`Trying free model: ${freeModel}`); const completion = await openai.chat.completions.create({ model: freeModel, messages: [{ role: 'user', content }] as any }); responseText = completion.choices[0].message.content || ''; usedModel = completion.model; usage = completion.usage; } else { throw backupError; } } } else { // No backup, try free model directly console.error(`Primary model failed, searching for free models...`); const freeModel = await findSuitableFreeModel(openai); if (freeModel && freeModel !== model) { console.error(`Trying free model: ${freeModel}`); const completion = await openai.chat.completions.create({ model: freeModel, messages: [{ role: 'user', content }] as any }); responseText = completion.choices[0].message.content || ''; usedModel = completion.model; usage = completion.usage; } else { throw primaryError; } } } // Format as markdown if requested if (args.markdown_response) { // Simple formatting enhancements responseText = responseText // Add horizontal rule after sections .replace(/^(#{1,3}.*)/gm, '$1\n\n---') // Ensure proper spacing for lists .replace(/^(\s*[-*•]\s.+)$/gm, '\n$1') // Convert plain URLs to markdown links .replace(/(https?:\/\/[^\s]+)/g, '[$1]($1)'); } // Return the analysis result return { content: [ { type: 'text', text: responseText, }, ], metadata: { model: usedModel, usage: usage } }; } catch (error: any) { console.error('Error in multi-image analysis:', error); if (error instanceof McpError) { throw error; } return { content: [ { type: 'text', text: `Error analyzing images: ${error.message}`, }, ], isError: true, metadata: { error_type: error.constructor.name, error_message: error.message } }; } }
• src/tool-handlers.ts:160-200 (schema)

  Input schema definition for the tool, specifying parameters: images array (with urls), prompt, optional markdown_response and model.
  { name: 'mcp_openrouter_multi_image_analysis', description: 'Analyze multiple images at once with a single prompt and receive detailed responses', inputSchema: { type: 'object', properties: { images: { type: 'array', description: 'Array of image objects to analyze', items: { type: 'object', properties: { url: { type: 'string', description: 'URL or data URL of the image (use http(s):// for web images, absolute file paths for local files, or data:image/xxx;base64,... for base64 encoded images)', }, alt: { type: 'string', description: 'Optional alt text or description of the image', }, }, required: ['url'], }, }, prompt: { type: 'string', description: 'Prompt for analyzing the images', }, markdown_response: { type: 'boolean', description: 'Whether to format the response in Markdown (default: true)', default: true, }, model: { type: 'string', description: 'OpenRouter model to use. If not specified, the system will use a free model with vision capabilities or the default model.', }, }, required: ['images', 'prompt'], }, },
• src/tool-handlers.ts:335-340 (registration)

  Registration in the CallToolRequestHandler switch statement, dispatching to the handleMultiImageAnalysis function.
  case 'mcp_openrouter_multi_image_analysis': return handleMultiImageAnalysis({ params: { arguments: request.params.arguments as unknown as MultiImageAnalysisToolRequest } }, this.openai, this.defaultModel);
• src/tool-handlers.ts:55-317 (registration)

  Tool registration in the ListToolsRequestHandler, including the tool in the available tools list with schema.
  this.server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: [ // Chat Completion Tool { name: 'mcp_openrouter_chat_completion', description: 'Send a message to OpenRouter.ai and get a response', inputSchema: { type: 'object', properties: { model: { type: 'string', description: 'The model to use (e.g., "google/gemini-2.5-pro-exp-03-25:free", "undi95/toppy-m-7b:free"). If not provided, uses the default model if set.', }, messages: { type: 'array', description: 'An array of conversation messages with roles and content', minItems: 1, maxItems: 100, items: { type: 'object', properties: { role: { type: 'string', enum: ['system', 'user', 'assistant'], description: 'The role of the message sender', }, content: { oneOf: [ { type: 'string', description: 'The text content of the message', }, { type: 'array', description: 'Array of content parts for multimodal messages (text and images)', items: { type: 'object', properties: { type: { type: 'string', enum: ['text', 'image_url'], description: 'The type of content (text or image)', }, text: { type: 'string', description: 'The text content (for text type)', }, image_url: { type: 'object', description: 'The image URL object (for image_url type)', properties: { url: { type: 'string', description: 'URL of the image (can be a data URL with base64)', }, }, required: ['url'], }, }, required: ['type'], }, }, ], }, }, required: ['role', 'content'], }, }, temperature: { type: 'number', description: 'Sampling temperature (0-2)', minimum: 0, maximum: 2, }, }, required: ['messages'], }, maxContextTokens: 200000 }, // Single Image Analysis Tool { name: 'mcp_openrouter_analyze_image', description: 'Analyze an image using OpenRouter vision models', inputSchema: { type: 'object', properties: { image_path: { type: 'string', description: 'Path to the image file to analyze (can be an absolute file path, URL, or base64 data URL starting with "data:")', }, question: { type: 'string', description: 'Question to ask about the image', }, model: { type: 'string', description: 'OpenRouter model to use (e.g., "anthropic/claude-3.5-sonnet")', }, }, required: ['image_path'], }, }, // Multi-Image Analysis Tool { name: 'mcp_openrouter_multi_image_analysis', description: 'Analyze multiple images at once with a single prompt and receive detailed responses', inputSchema: { type: 'object', properties: { images: { type: 'array', description: 'Array of image objects to analyze', items: { type: 'object', properties: { url: { type: 'string', description: 'URL or data URL of the image (use http(s):// for web images, absolute file paths for local files, or data:image/xxx;base64,... for base64 encoded images)', }, alt: { type: 'string', description: 'Optional alt text or description of the image', }, }, required: ['url'], }, }, prompt: { type: 'string', description: 'Prompt for analyzing the images', }, markdown_response: { type: 'boolean', description: 'Whether to format the response in Markdown (default: true)', default: true, }, model: { type: 'string', description: 'OpenRouter model to use. If not specified, the system will use a free model with vision capabilities or the default model.', }, }, required: ['images', 'prompt'], }, }, // Audio Analysis Tool { name: 'mcp_openrouter_analyze_audio', description: 'Transcribe audio files and provide raw content. Supports wav/mp3 files from CDN URLs or local paths.', inputSchema: { type: 'object', properties: { audio_url: { type: 'string', description: 'Path or URL to the audio file (supports CDN URLs, local file paths, wav/mp3 formats)', }, model: { type: 'string', description: 'OpenRouter model to use (e.g., "mistralai/voxtral-small-24b-2507", "openai/gpt-4o-audio-preview")', }, }, required: ['audio_url'], }, }, // Search Models Tool { name: 'search_models', description: 'Search and filter OpenRouter.ai models based on various criteria', inputSchema: { type: 'object', properties: { query: { type: 'string', description: 'Optional search query to filter by name, description, or provider', }, provider: { type: 'string', description: 'Filter by specific provider (e.g., "anthropic", "openai", "cohere")', }, minContextLength: { type: 'number', description: 'Minimum context length in tokens', }, maxContextLength: { type: 'number', description: 'Maximum context length in tokens', }, maxPromptPrice: { type: 'number', description: 'Maximum price per 1K tokens for prompts', }, maxCompletionPrice: { type: 'number', description: 'Maximum price per 1K tokens for completions', }, capabilities: { type: 'object', description: 'Filter by model capabilities', properties: { functions: { type: 'boolean', description: 'Requires function calling capability', }, tools: { type: 'boolean', description: 'Requires tools capability', }, vision: { type: 'boolean', description: 'Requires vision capability', }, json_mode: { type: 'boolean', description: 'Requires JSON mode capability', } } }, limit: { type: 'number', description: 'Maximum number of results to return (default: 10)', minimum: 1, maximum: 50 } } }, }, // Get Model Info Tool { name: 'get_model_info', description: 'Get detailed information about a specific model', inputSchema: { type: 'object', properties: { model: { type: 'string', description: 'The model ID to get information for', }, }, required: ['model'], }, }, // Validate Model Tool { name: 'validate_model', description: 'Check if a model ID is valid', inputSchema: { type: 'object', properties: { model: { type: 'string', description: 'The model ID to validate', }, }, required: ['model'], }, }, ], }));
• src/tool-handlers/multi-image-analysis.ts:55-63 (schema)

  TypeScript interface matching the input schema for type safety.
  export interface MultiImageAnalysisToolRequest { images: Array<{ url: string; alt?: string; }>; prompt: string; markdown_response?: boolean; model?: string; }