OpenRouter MCP Multimodal Server

Instructions

Implementation Reference

• src/tool-handlers/multi-image-analysis.ts:342-556 (handler)

  Main handler function: validates input, fetches and processes multiple images into base64 data URLs, constructs multimodal message, calls OpenRouter API (with model fallbacks), formats Markdown response if requested, and returns structured result with metadata.

  export async function handleMultiImageAnalysis(
    request: { params: { arguments: MultiImageAnalysisToolRequest } },
    openai: OpenAI,
    defaultModel?: string
  ) {
    const args = request.params.arguments;
    
    try {
      // Validate inputs
      if (!args.images || !Array.isArray(args.images) || args.images.length === 0) {
        throw new McpError(ErrorCode.InvalidParams, 'At least one image is required');
      }
      
      if (!args.prompt) {
        throw new McpError(ErrorCode.InvalidParams, 'A prompt for analyzing the images is required');
      }
      
      console.error(`Processing ${args.images.length} images`);
      
      // Process each image and convert to base64 if needed
      const processedImages = await Promise.all(
        args.images.map(async (image, index) => {
          try {
            // Skip processing if already a data URL
            if (image.url.startsWith('data:')) {
              console.error(`Image ${index + 1} is already in base64 format`);
              return image;
            }
            
            console.error(`Processing image ${index + 1}: ${image.url.substring(0, 100)}${image.url.length > 100 ? '...' : ''}`);
            
            // Get MIME type
            const mimeType = getMimeType(image.url);
            
            // Fetch and process the image
            const buffer = await fetchImageAsBuffer(image.url);
            const base64 = await processImage(buffer, mimeType);
            
            return {
              url: `data:${mimeType === 'application/octet-stream' ? 'image/jpeg' : mimeType};base64,${base64}`,
              alt: image.alt
            };
          } catch (error: any) {
            console.error(`Error processing image ${index + 1}:`, error);
            throw new Error(`Failed to process image ${index + 1}: ${image.url}. Error: ${error.message}`);
          }
        })
      );
      
      // Select model with priority:
      // 1. User-specified model
      // 2. Default model from environment
      let model = args.model || defaultModel || DEFAULT_FREE_MODEL;
      console.error(`[Multi-Image Tool] Using IMAGE model: ${model}`);
      
      // Build content array for the API call
      const content: Array<{ 
        type: string; 
        text?: string; 
        image_url?: { 
          url: string 
        } 
      }> = [
        {
          type: 'text',
          text: args.prompt
        }
      ];
      
      // Add each processed image to the content array
      processedImages.forEach(image => {
        content.push({
          type: 'image_url',
          image_url: {
            url: image.url
          }
        });
      });
      
      // Try primary model first
      let responseText: string;
      let responseId: string;
      let usedModel: string;
      let usage: any;
      
      try {
        const completion = await openai.chat.completions.create({
          model,
          messages: [{
            role: 'user',
            content
          }] as any
        });
        
        const response = completion as any;
        responseText = completion.choices[0].message.content || '';
        responseId = response.id;
        usedModel = response.model;
        usage = response.usage;
      } catch (primaryError: any) {
        // If primary model fails and backup exists, try backup
        const backupModel = process.env.OPENROUTER_DEFAULT_MODEL_IMG_BACKUP;
        if (backupModel && backupModel !== model) {
          try {
            console.error(`Primary model failed, trying backup: ${backupModel}`);
            const completion = await openai.chat.completions.create({
              model: backupModel,
              messages: [{
                role: 'user',
                content
              }] as any
            });
            
            const resp = completion as any;
            responseText = completion.choices[0].message.content || '';
            responseId = resp.id;
            usedModel = resp.model;
            usage = resp.usage;
          } catch (backupError: any) {
            console.error(`Backup model failed, searching for free models...`);
            
            // Try to find a free model
            const freeModel = await findSuitableFreeModel(openai);
            if (freeModel && freeModel !== model && freeModel !== backupModel) {
              console.error(`Trying free model: ${freeModel}`);
              const completion = await openai.chat.completions.create({
                model: freeModel,
                messages: [{
                  role: 'user',
                  content
                }] as any
              });
              
              const resp = completion as any;
              responseText = completion.choices[0].message.content || '';
              responseId = resp.id;
              usedModel = resp.model;
              usage = resp.usage;
            } else {
              throw backupError;
            }
          }
        } else {
          // No backup, try free model directly
          console.error(`Primary model failed, searching for free models...`);
          const freeModel = await findSuitableFreeModel(openai);
          if (freeModel && freeModel !== model) {
            console.error(`Trying free model: ${freeModel}`);
            const completion = await openai.chat.completions.create({
              model: freeModel,
              messages: [{
                role: 'user',
                content
              }] as any
            });
            
            const resp = completion as any;
            responseText = completion.choices[0].message.content || '';
            responseId = resp.id;
            usedModel = resp.model;
            usage = resp.usage;
          } else {
            throw primaryError;
          }
        }
      }
      
      // Format as markdown if requested
      if (args.markdown_response) {
        // Simple formatting enhancements
        responseText = responseText
          // Add horizontal rule after sections
          .replace(/^(#{1,3}.*)/gm, '$1\n\n---')
          // Ensure proper spacing for lists
          .replace(/^(\s*[-*•]\s.+)$/gm, '\n$1')
          // Convert plain URLs to markdown links
          .replace(/(https?:\/\/[^\s]+)/g, '[$1]($1)');
      }
      
      // Return the analysis result
      return {
        content: [
          {
            type: 'text',
            text: responseText,
          },
        ],
        metadata: {
          id: responseId,
          model: usedModel,
          usage: usage
        }
      };
    } catch (error: any) {
      console.error('Error in multi-image analysis:', error);
      
      if (error instanceof McpError) {
        throw error;
      }
      
      return {
        content: [
          {
            type: 'text',
            text: `Error analyzing images: ${error.message}`,
          },
        ],
        isError: true,
        metadata: {
          error_type: error.constructor.name,
          error_message: error.message
        }
      };
    }
  }

• src/tool-handlers.ts:160-200 (schema)

  Tool schema definition including input schema for images array, prompt, markdown_response flag, and optional model.

  {
    name: 'mcp_openrouter_multi_image_analysis',
    description: 'Analyze multiple images at once with a single prompt and receive detailed responses',
    inputSchema: {
      type: 'object',
      properties: {
        images: {
          type: 'array',
          description: 'Array of image objects to analyze',
          items: {
            type: 'object',
            properties: {
              url: {
                type: 'string',
                description: 'URL or data URL of the image (use http(s):// for web images, absolute file paths for local files, or data:image/xxx;base64,... for base64 encoded images)',
              },
              alt: {
                type: 'string',
                description: 'Optional alt text or description of the image',
              },
            },
            required: ['url'],
          },
        },
        prompt: {
          type: 'string',
          description: 'Prompt for analyzing the images',
        },
        markdown_response: {
          type: 'boolean',
          description: 'Whether to format the response in Markdown (default: true)',
          default: true,
        },
        model: {
          type: 'string',
          description: 'OpenRouter model to use. If not specified, the system will use a free model with vision capabilities or the default model.',
        },
      },
      required: ['images', 'prompt'],
    },
  },

• src/tool-handlers.ts:335-340 (registration)

  Registration in the CallToolRequestSchema switch statement that dispatches to the handleMultiImageAnalysis handler function.

  case 'mcp_openrouter_multi_image_analysis':
    return handleMultiImageAnalysis({
      params: {
        arguments: request.params.arguments as unknown as MultiImageAnalysisToolRequest
      }
    }, this.openai, this.defaultModel);

• src/tool-handlers/multi-image-analysis.ts:55-63 (schema)

  TypeScript interface defining the tool request parameters, used for type safety in the handler.

  export interface MultiImageAnalysisToolRequest {
    images: Array<{
      url: string;
      alt?: string;
    }>;
    prompt: string;
    markdown_response?: boolean;
    model?: string;
  }