generate_subtitles

Instructions

Implementation Reference

• src/index.ts:969-1008 (registration)

  Tool registration for 'generate_subtitles' — defines name, description, and inputSchema with properties: file_path, language, translate_to_english, background, threads, temperature, prompt, beam_size, best_of, diarize, vad_model.

  {
    name: "generate_subtitles",
    description:
      "Generate subtitle files for an audio or video file using whisper.cpp. " +
      "Set language='auto' to detect the spoken language automatically. " +
      "Set translate_to_english=true to also generate an English translation subtitle file. " +
      "When both are requested, two .srt files are saved: one in the original language (e.g. film.ja.srt) " +
      "and one English translation (film.en.srt). " +
      "Load in VLC via Subtitle → Add Subtitle File. " +
      "Supports all standard formats plus .3gp and .ts.",
    inputSchema: {
      type: "object",
      properties: {
        file_path: { type: "string", description: "Absolute Windows path to the file." },
        language: {
          type: "string",
          description: "Language code (e.g. ja, es, fr, de) or 'auto' to detect automatically. Defaults to en.",
          default: "en",
        },
        translate_to_english: {
          type: "boolean",
          description: "Also generate an English translation .srt alongside the native language .srt. Only applies when language is not 'en'. Not available in background mode.",
          default: false,
        },
        background: {
          type: "boolean",
          description: "Run as a detached background job — recommended for files over 10 minutes. Returns a job ID to use with check_progress. translate_to_english is not available in background mode.",
          default: false,
        },
        threads: { type: "number", description: `CPU threads. Defaults to ${WHISPER_THREADS} of ${SYSTEM_THREADS}.` },
        temperature: { type: "number", description: "Sampling temperature 0.0–1.0. Default 0.0." },
        prompt: { type: "string", description: "Prior context string for domain-specific vocabulary or speaker names." },
        beam_size: { type: "number", description: "Beam search width. Higher = more accurate, slower. Default 5." },
        best_of: { type: "number", description: "Candidate sequences evaluated. Default 5." },
        diarize: { type: "boolean", description: "Stereo speaker diarization. Requires stereo audio with speakers on separate channels.", default: false },
        vad_model: { type: "string", description: "Path to Silero VAD model .bin. Strips silence before transcription. Download via download_model." },
      },
      required: ["file_path"],
    },
  },

• src/index.ts:1734-1843 (handler)

  Handler function for 'generate_subtitles' — takes file_path, language, translate_to_english, background, threads, and extra opts. In background mode, uses spawnDetached with 'srt' format. In blocking mode, converts to WAV if needed, auto-detects language if 'auto', then runs two SRT passes: native language (via runSrtPass) and optionally English translation (via runSrtPass with translate=true). Returns paths to generated .srt files.

  // generate_subtitles
  // -------------------------------------------------------------------------
  if (name === "generate_subtitles") {
    const filePath = args?.file_path as string;
    const language = (args?.language as string) || "en";
    const translateToEnglish = (args?.translate_to_english as boolean) || false;
    const background = (args?.background as boolean) || false;
    const threads = Math.min(SYSTEM_THREADS, Math.max(1, Math.round((args?.threads as number) || WHISPER_THREADS)));
  
    // v2.2.0 quality params
    const extraOpts: Partial<WhisperOptions> = {};
    if (args?.temperature !== undefined) extraOpts.temperature = Number(args.temperature);
    if (args?.prompt) extraOpts.prompt = String(args.prompt);
    if (args?.beam_size !== undefined) extraOpts.beamSize = Number(args.beam_size);
    if (args?.best_of !== undefined) extraOpts.bestOf = Number(args.best_of);
    if (args?.diarize) extraOpts.diarize = true;
    if (args?.vad_model) extraOpts.vadModel = String(args.vad_model);
  
    if (!filePath) return { content: [{ type: "text", text: "file_path is required." }], isError: true };
    const pathError = validateInputPath(filePath);
    if (pathError) return { content: [{ type: "text", text: pathError }], isError: true };
    if (!existsSync(filePath)) return { content: [{ type: "text", text: `File not found: ${filePath}` }], isError: true };
    const configError = validatePaths();
    if (configError) return { content: [{ type: "text", text: configError }], isError: true };
    if (await isWhisperRunning()) {
      return { content: [{ type: "text", text: "Transcription already in progress. Wait for it to finish first." }], isError: true };
    }
  
    // Background mode — detached SRT job
    if (background) {
      try {
        const { jobId, pid } = await spawnDetached(filePath, WHISPER_MODEL, language, threads, "srt", extraOpts);
        return {
          content: [{
            type: "text",
            text:
              `⏳ Background subtitle generation started.\n\n` +
              `Source: ${basename(filePath)}\n` +
              `Job ID: ${jobId}\n` +
              `PID: ${pid}\n` +
              `Language: ${language}\n\n` +
              `Call check_progress with job_id="${jobId}" to monitor.\n` +
              `Note: translate_to_english is not available in background mode. ` +
              `Run generate_subtitles again after completion to create the English translation.`,
          }],
        };
      } catch (err: any) {
        return { content: [{ type: "text", text: `Failed to start background subtitle job:\n\n${err?.message || String(err)}` }], isError: true };
      }
    }
  
    try {
      // Convert to WAV if needed
      let transcribeFrom = filePath;
      let tmpFile: string | null = null;
      if (needsConversion(filePath)) {
        tmpFile = await convertToWav(filePath);
        transcribeFrom = tmpFile;
      }
  
      const baseNoExt = filePath.replace(/\.[^.]+$/, "");
  
      // Auto-detect language if requested
      let detectedLang = language;
      if (language === "auto") {
        const detected = await detectLanguage(transcribeFrom, WHISPER_MODEL, threads);
        detectedLang = detected ?? "en";
      }
  
      const results: string[] = [];
  
      // Pass 1 — native language SRT
      const nativeSrt = language === "en" || detectedLang === "en"
        ? `${baseNoExt}.srt`
        : `${baseNoExt}.${detectedLang}.srt`;
  
      await runSrtPass(transcribeFrom, nativeSrt, WHISPER_MODEL, detectedLang, threads, false, extraOpts);
      results.push(`✅ Native (${detectedLang}): ${nativeSrt}`);
  
      // Pass 2 — English translation SRT (only if language isn't already English)
      if (translateToEnglish && detectedLang !== "en") {
        const englishSrt = `${baseNoExt}.en.srt`;
        await runSrtPass(transcribeFrom, englishSrt, WHISPER_MODEL, detectedLang, threads, true, extraOpts);
        results.push(`✅ English translation: ${englishSrt}`);
      }
  
      // Clean up temp WAV
      if (tmpFile && existsSync(tmpFile)) try { unlinkSync(tmpFile); } catch { }
  
      const langNote = language === "auto"
        ? `Auto-detected language: ${detectedLang}\n\n`
        : "";
  
      return {
        content: [{
          type: "text",
          text:
            `✅ Subtitle file(s) generated!\n\n` +
            langNote +
            results.join("\n") + "\n\n" +
            `To use in VLC: Subtitle → Add Subtitle File → select the .srt file.\n` +
            `Works in any video player that supports external subtitles.\n\n` +
            `Note: whisper's built-in translation only translates to English. ` +
            `For other target languages, translate the .srt file contents separately.`,
        }],
      };
    } catch (err: any) {
      return { content: [{ type: "text", text: `Subtitle generation failed:\n\n${err?.stderr || err?.message || String(err)}` }], isError: true };
    }
  }

• src/index.ts:785-805 (helper)

  Helper function runSrtPass — runs a single whisper-cli SRT pass by building args with buildArgs, executing the process, then moving the temp .srt output to the destination path.

  async function runSrtPass(
    transcribeFrom: string, destSrt: string,
    model: string, language: string, threads: number,
    translate = false, extraOpts: Partial<WhisperOptions> = {}
  ): Promise<string> {
    const opts: WhisperOptions = {
      language, outputFormat: "srt", threads, translate,
      ...extraOpts,
    };
    const args = buildArgs(transcribeFrom, model, opts);
    await execFileAsync(WHISPER_CLI_PATH, args, {
      maxBuffer: 100 * 1024 * 1024,
      windowsHide: true,
    });
    const tmpSrt = transcribeFrom.replace(/\.[^.]+$/, ".srt");
    if (existsSync(tmpSrt)) {
      writeFileSync(destSrt, readFileSync(tmpSrt, "utf8"));
      try { unlinkSync(tmpSrt); } catch { }
    }
    return destSrt;
  }

• src/index.ts:763-779 (helper)

  Helper function detectLanguage — probes the first 30 seconds of audio to auto-detect the spoken language using whisper's auto-detection.

  async function detectLanguage(wavPath: string, model: string, threads: number): Promise<string | null> {
    try {
      const { stdout, stderr } = await execFileAsync(WHISPER_CLI_PATH, [
        "-m", model, "-f", wavPath,
        "-l", "auto",
        "-t", String(threads),
        "--no-timestamps",
        "--duration", "30000",
      ], { maxBuffer: 10 * 1024 * 1024, windowsHide: true });
      const output = stdout + stderr;
      // whisper outputs: "auto-detected language: ja (p = 0.98)"
      const m = output.match(/auto-detected language:\s*([a-z]{2,3})/i);
      return m ? m[1].toLowerCase() : null;
    } catch {
      return null;
    }
  }

• src/index.ts:705-756 (helper)

  Helper function buildArgs — constructs the command-line arguments for whisper-cli.exe, handling output format (srt uses -osrt flag, text uses --no-timestamps), hallucination prevention, and all optional parameters.

  function buildArgs(filePath: string, model: string, opts: WhisperOptions): string[] {
    const lang = opts.language === "auto" ? "auto" : opts.language;
    const args = ["-m", model, "-f", filePath, "-l", lang, "-t", String(opts.threads)];
  
    // Hallucination prevention — set max context tokens to 0 to prevent whisper
    // from conditioning each segment on its own prior output, which causes
    // repetitive hallucination loops on noisy or silent audio.
    // Flag: --max-context 0 (user can re-enable by setting conditionOnPrevText=true)
    if (!opts.conditionOnPrevText) args.push("--max-context", "0");
  
    // Treat segments below this confidence threshold as silence rather than
    // hallucinating content. Confirmed valid flag in whisper-cli (-nth).
    args.push("--no-speech-thold", String(opts.noSpeechThold ?? 0.6));
  
    if (opts.translate) args.push("--translate");
  
    if (opts.temperature !== undefined) args.push("--temperature", String(opts.temperature));
    if (opts.prompt) args.push("--prompt", opts.prompt);
    if (opts.beamSize !== undefined) args.push("--beam-size", String(opts.beamSize));
    if (opts.bestOf !== undefined) args.push("--best-of", String(opts.bestOf));
    if (opts.gpuDevice !== undefined) args.push("-g", String(opts.gpuDevice));
    if (opts.processors !== undefined && opts.processors > 1) args.push("-p", String(opts.processors));
    if (opts.offsetT !== undefined) args.push("--offset-t", String(opts.offsetT));
    if (opts.duration !== undefined) args.push("--duration", String(opts.duration));
    if (opts.diarize) args.push("--diarize");
  
    // word_timestamps: sets max-len=1 + split-on-word for per-word output
    // without requiring JSON parsing — simpler than -oj approach.
    if (opts.wordTimestamps) {
      args.push("--max-len", "1", "--split-on-word");
    } else {
      if (opts.maxLen !== undefined) args.push("--max-len", String(opts.maxLen));
      if (opts.splitOnWord) args.push("--split-on-word");
    }
  
    // VAD: voice activity detection — strips silence before whisper sees the audio
    if (opts.vadModel && existsSync(opts.vadModel)) {
      args.push("--vad", "--vad-model", opts.vadModel);
    }
  
    // Output format
    if (opts.outputFormat === "srt") {
      args.push("-osrt", "-of", filePath.replace(/\.[^.]+$/, ""));
    } else if (opts.outputFormat === "json") {
      args.push("-oj");
    } else if (opts.outputFormat === "text") {
      args.push("--no-timestamps");
    }
    // "timestamps" format: no flag — whisper default stdout includes timestamps
  
    return args;
  }