generate_with_ip_adapter

Instructions

Implementation Reference

• src/tools/conditioning.ts:125-155 (handler)

  Tool handler for 'generate_with_ip_adapter' – registers an MCP tool that fetches a reference image, builds an IP-Adapter workflow, runs it via ComfyUI client, and returns generated image URLs.

  server.tool(
    "generate_with_ip_adapter",
    "Generate an image using a reference image as an IP-Adapter visual/style/subject guide. Requires the ComfyUI-IPAdapter-plus custom node pack and the preset's matching models (IPAdapter weights + CLIP vision). Weight tunes how strongly the reference guides generation.",
    ipAdapterSchema,
    async (args) => {
      const upload = await client.fetchAndUploadImage(args.reference_image_url);
  
      const workflow = ipAdapter({
        prompt: args.prompt,
        negativePrompt: args.negative_prompt ?? "",
        referenceImage: upload.name,
        preset: args.preset,
        weight: args.weight,
        startAt: args.start_at,
        endAt: args.end_at,
        width: args.width,
        height: args.height,
        steps: args.steps,
        cfg: args.cfg,
        seed: args.seed ?? Math.floor(Math.random() * 2 ** 32),
        checkpoint: args.checkpoint ?? DEFAULT_CHECKPOINT,
      });
  
      const result = await client.runWorkflow(workflow);
      const lines = [
        `Generated ${result.images.length} image(s) with IP-Adapter (preset: ${args.preset}, weight: ${args.weight}, prompt_id: ${result.promptId}):`,
        ...result.images.map((u, i) => `  ${i + 1}. ${u}`),
      ];
      return { content: [{ type: "text" as const, text: lines.join("\n") }] };
    },
  );

• src/tools/conditioning.ts:58-87 (schema)

  Zod schema (ipAdapterSchema) defining the input parameters: prompt, negative_prompt, reference_image_url, preset, weight, start_at, end_at, width, height, steps, cfg, seed, checkpoint.

  const ipAdapterSchema = {
    prompt: z.string().min(1).describe("Text prompt for the generated image."),
    negative_prompt: z.string().optional(),
    reference_image_url: z
      .string()
      .url()
      .describe(
        "URL of the reference image that IP-Adapter uses as a visual/style/subject guide.",
      ),
    preset: z
      .string()
      .default("STANDARD (medium strength)")
      .describe(
        `IP-Adapter preset (picks the matching IPAdapter + CLIP Vision models). Common values: ${IP_ADAPTER_PRESETS.join(" | ")}`,
      ),
    weight: z
      .number()
      .min(0)
      .max(3)
      .default(1.0)
      .describe("How strongly the reference guides the output."),
    start_at: z.number().min(0).max(1).default(0),
    end_at: z.number().min(0).max(1).default(1),
    width: z.number().int().min(64).max(2048).default(1024),
    height: z.number().int().min(64).max(2048).default(1024),
    steps: z.number().int().min(1).max(150).default(25),
    cfg: z.number().min(1).max(30).default(7),
    seed: z.number().int().optional(),
    checkpoint: z.string().optional(),
  };

• src/tools/conditioning.ts:89-92 (registration)

  Registration function `registerConditioningTools` exports the tool registration. Called from server.ts line 47 where it's imported and invoked.

  export function registerConditioningTools(
    server: McpServer,
    client: ComfyUIClient,
  ): void {

• src/comfyui/workflows.ts:223-308 (helper)

  IPAdapterParams interface and `ipAdapter()` workflow builder function – constructs the ComfyUI workflow JSON using IPAdapterUnifiedLoader and IPAdapterAdvanced nodes.

  export interface IPAdapterParams {
    prompt: string;
    negativePrompt: string;
    referenceImage: string;
    preset: string;
    weight: number;
    startAt: number;
    endAt: number;
    width: number;
    height: number;
    steps: number;
    cfg: number;
    seed: number;
    checkpoint: string;
  }
  
  /**
   * IP-Adapter workflow. Relies on the ComfyUI-IPAdapter-plus custom node pack
   * (nodes `IPAdapterUnifiedLoader`, `IPAdapterAdvanced`). If the pack or the
   * selected preset's models aren't installed, ComfyUI returns a node error.
   */
  export function ipAdapter(params: IPAdapterParams): Workflow {
    return {
      "4": {
        class_type: "CheckpointLoaderSimple",
        inputs: { ckpt_name: params.checkpoint },
      },
      "5": {
        class_type: "EmptyLatentImage",
        inputs: { width: params.width, height: params.height, batch_size: 1 },
      },
      "6": {
        class_type: "CLIPTextEncode",
        inputs: { text: params.prompt, clip: ["4", 1] },
      },
      "7": {
        class_type: "CLIPTextEncode",
        inputs: { text: params.negativePrompt, clip: ["4", 1] },
      },
      "10": {
        class_type: "LoadImage",
        inputs: { image: params.referenceImage },
      },
      "20": {
        class_type: "IPAdapterUnifiedLoader",
        inputs: { model: ["4", 0], preset: params.preset },
      },
      "21": {
        class_type: "IPAdapterAdvanced",
        inputs: {
          model: ["20", 0],
          ipadapter: ["20", 1],
          image: ["10", 0],
          weight: params.weight,
          weight_type: "linear",
          combine_embeds: "concat",
          start_at: params.startAt,
          end_at: params.endAt,
          embeds_scaling: "V only",
        },
      },
      "3": {
        class_type: "KSampler",
        inputs: {
          seed: params.seed,
          steps: params.steps,
          cfg: params.cfg,
          sampler_name: "euler",
          scheduler: "normal",
          denoise: 1,
          model: ["21", 0],
          positive: ["6", 0],
          negative: ["7", 0],
          latent_image: ["5", 0],
        },
      },
      "8": {
        class_type: "VAEDecode",
        inputs: { samples: ["3", 0], vae: ["4", 2] },
      },
      "9": {
        class_type: "SaveImage",
        inputs: { filename_prefix: "comfyui-mcp-ipa", images: ["8", 0] },
      },
    };
  }