feat(cli,core): standalone transcribe command, transcript normalization, caption lint rules (#151)

* feat(cli,core): add standalone transcribe command, transcript normalization, and caption lint rules - Add `hyperframes transcribe` command for transcribing audio/video and importing existing transcripts (SRT, VTT, OpenAI Whisper API JSON, whisper.cpp JSON) - Add transcript format normalizer (normalize.ts) with auto-detection and conversion to standard [{text, start, end}] word arrays - Upgrade default whisper model from base.en to small.en for better accuracy - Add --model and --language flags to both `transcribe` and `init` commands - Extract shared patchCaptionHtml() to eliminate duplication between init.ts and transcribe.ts (init.ts reduced by ~55 lines) - Add 3 caption lint rules: caption_exit_missing_hard_kill, caption_text_overflow_risk, caption_container_relative_position - Update captions skill with model guide, format docs, music guidance, text overflow prevention, caption exit guarantee pattern - Expand captions skill trigger to cover lyrics, karaoke, lyric videos Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs(cli): add transcribe command and --model/--language flags to CLI docs Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(cli): fix blank template lint issues - blank/index.html: remove data-start from video (was nested in timed parent), add class="clip" for initial hidden state - blank/captions.html: add max-width + overflow:hidden to prevent text clipping, add tl.set hard kill after exit tween to prevent stuck captions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: add lint-after-edit rule to repo and project CLAUDE.md Agents must run `npx hyperframes lint` after editing compositions. Also expand captions skill description in project template. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * style: format _shared/CLAUDE.md Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-09-07 18:26:17 +00:00 · 2026-03-30 18:58:06 -07:00
parent 7b18c0352e
commit 2f99e33bbe
15 changed files with 1176 additions and 105 deletions
@@ -246,60 +246,11 @@ function patchVideoSrc(
  }
 }

-function patchTranscript(dir: string, transcriptPath: string): void {
-  // Read the whisper transcript and normalize to [{text, start, end}]
-  const raw = JSON.parse(readFileSync(transcriptPath, "utf-8"));
-  const words: { text: string; start: number; end: number }[] = [];
-  for (const seg of raw.transcription ?? []) {
-    for (const token of seg.tokens ?? []) {
-      const text = (token.text ?? "").trim();
-      if (!text || text.startsWith("[_") || text.startsWith("[BLANK")) continue;
-
-      // Merge punctuation with the previous word
-      const isPunctuation = /^[.,!?;:'")\]}>…–—-]+$/.test(text);
-      const lastWord = words[words.length - 1];
-      if (isPunctuation && lastWord) {
-        lastWord.text += text;
-        lastWord.end = Math.round(((token.offsets?.to ?? 0) / 1000) * 1000) / 1000;
-        continue;
-      }
-
-      words.push({
-        text,
-        start: Math.round(((token.offsets?.from ?? 0) / 1000) * 1000) / 1000,
-        end: Math.round(((token.offsets?.to ?? 0) / 1000) * 1000) / 1000,
-      });
-    }
-  }
-
+async function patchTranscript(dir: string, transcriptPath: string): Promise<void> {
+  const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js");
+  const { words } = loadTranscript(transcriptPath);
  if (words.length === 0) return;
-
-  const wordsJson = JSON.stringify(words, null, 10)
-    .replace(/^\[/, "[")
-    .replace(/\n {10}/g, "\n          ");
-
-  // Find captions HTML files and replace the hardcoded script array
-  const htmlFiles = readdirSync(dir, { withFileTypes: true, recursive: true })
-    .filter((e) => e.isFile() && e.name.endsWith(".html"))
-    .map((e) => join(e.parentPath ?? e.path, e.name));
-
-  for (const file of htmlFiles) {
-    let content = readFileSync(file, "utf-8");
-    // Match within <script> blocks only to avoid crossing block boundaries
-    const scriptBlocks = content.match(/<script>[\s\S]*?<\/script>/g) ?? [];
-    let scriptMatch: RegExpMatchArray | null = null;
-    let transcriptMatch: RegExpMatchArray | null = null;
-    for (const block of scriptBlocks) {
-      scriptMatch = scriptMatch ?? block.match(/const script = \[[\s\S]*?\];/);
-      transcriptMatch = transcriptMatch ?? block.match(/const TRANSCRIPT = \[[\s\S]*?\];/);
-    }
-    const match = scriptMatch ?? transcriptMatch;
-    if (match) {
-      const varName = scriptMatch ? "script" : "TRANSCRIPT";
-      content = content.replace(match[0], `const ${varName} = ${wordsJson};`);
-      writeFileSync(file, content, "utf-8");
-    }
-  }
+  patchCaptionHtml(dir, words);
 }

 // ---------------------------------------------------------------------------
@@ -543,6 +494,16 @@ Examples:
      type: "boolean",
      description: "Skip whisper transcription",
    },
+    model: {
+      type: "string",
+      description:
+        "Whisper model for transcription (e.g. tiny.en, base.en, small.en, medium.en, large)",
+    },
+    language: {
+      type: "string",
+      description:
+        "Language code for transcription (e.g. en, es, ja). Filters out non-target speech.",
+    },
    "non-interactive": {
      type: "boolean",
      description: "Disable interactive prompts (for CI/agents)",
@@ -555,6 +516,8 @@ Examples:
    const skipSkills = args["skip-skills"] === true;
    const skipTranscribe = args["skip-transcribe"] === true;
    const nonInteractive = args["non-interactive"] === true;
+    const modelFlag = args.model;
+    const languageFlag = args.language;
    const interactive = !nonInteractive && process.stdout.isTTY === true;

    // -----------------------------------------------------------------------
@@ -615,10 +578,13 @@ Examples:
        try {
          const { ensureWhisper, ensureModel } = await import("../whisper/manager.js");
          await ensureWhisper();
-          await ensureModel();
+          await ensureModel(modelFlag);
          console.log("Transcribing...");
          const { transcribe: runTranscribe } = await import("../whisper/transcribe.js");
-          const result = await runTranscribe(sourceFilePath, destDir);
+          const result = await runTranscribe(sourceFilePath, destDir, {
+            model: modelFlag,
+            language: languageFlag,
+          });
          console.log(
            `Transcribed: ${result.wordCount} words (${result.durationSeconds.toFixed(1)}s)`,
          );
@@ -633,7 +599,7 @@ Examples:
      trackInitTemplate(templateId);
      const transcriptFile = resolve(destDir, "transcript.json");
      if (existsSync(transcriptFile)) {
-        patchTranscript(destDir, transcriptFile);
+        await patchTranscript(destDir, transcriptFile);
      }

      // Skills
@@ -796,13 +762,15 @@ Examples:
          await ensureWhisper({
            onProgress: (msg) => spin.message(msg),
          });
-          await ensureModel(undefined, {
+          await ensureModel(modelFlag, {
            onProgress: (msg) => spin.message(msg),
          });

          spin.message("Transcribing audio...");
          const { transcribe: runTranscribe } = await import("../whisper/transcribe.js");
          const transcribeResult = await runTranscribe(sourceFilePath, destDir, {
+            model: modelFlag,
+            language: languageFlag,
            onProgress: (msg) => spin.message(msg),
          });
          spin.stop(
@@ -0,0 +1,149 @@
+import { defineCommand } from "citty";
+import { existsSync, writeFileSync } from "node:fs";
+import { resolve, join, extname } from "node:path";
+import * as clack from "@clack/prompts";
+import { c } from "../ui/colors.js";
+import { DEFAULT_MODEL } from "../whisper/manager.js";
+
+export default defineCommand({
+  meta: {
+    name: "transcribe",
+    description:
+      "Transcribe audio/video to word-level timestamps, or import an existing transcript",
+  },
+  args: {
+    input: {
+      type: "positional",
+      description:
+        "Audio/video file to transcribe, or transcript file to import (.json, .srt, .vtt)",
+      required: true,
+    },
+    dir: {
+      type: "string",
+      description: "Project directory (default: current directory)",
+      alias: "d",
+    },
+    model: {
+      type: "string",
+      description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny.en, base.en, small.en, medium.en, large-v3`,
+      alias: "m",
+    },
+    language: {
+      type: "string",
+      description: "Language code (e.g. en, es, ja). Filters out non-target language speech.",
+      alias: "l",
+    },
+    json: {
+      type: "boolean",
+      description: "Output result as JSON",
+      default: false,
+    },
+  },
+  async run({ args }) {
+    const inputPath = resolve(args.input);
+    if (!existsSync(inputPath)) {
+      console.error(c.error(`File not found: ${args.input}`));
+      process.exit(1);
+    }
+
+    const dir = resolve(args.dir ?? ".");
+    const ext = extname(inputPath).toLowerCase();
+
+    // ── Import mode: convert existing transcript ──────────────────────────
+    const isImport = ext === ".json" || ext === ".srt" || ext === ".vtt";
+
+    if (isImport) {
+      return importTranscript(inputPath, dir, args.json);
+    }
+
+    // ── Transcribe mode: run whisper ─────────────────────────────────────
+    return transcribeAudio(inputPath, dir, {
+      model: args.model,
+      language: args.language,
+      json: args.json,
+    });
+  },
+});
+
+// ---------------------------------------------------------------------------
+// Import existing transcript
+// ---------------------------------------------------------------------------
+
+async function importTranscript(inputPath: string, dir: string, json: boolean): Promise<void> {
+  const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js");
+  const { words, format } = loadTranscript(inputPath);
+
+  if (words.length === 0) {
+    console.error(c.error("No words found in transcript."));
+    process.exit(1);
+  }
+
+  const outPath = join(dir, "transcript.json");
+  writeFileSync(outPath, JSON.stringify(words, null, 2));
+  patchCaptionHtml(dir, words);
+
+  if (json) {
+    console.log(
+      JSON.stringify({ ok: true, format, wordCount: words.length, transcriptPath: outPath }),
+    );
+  } else {
+    console.log(
+      `${c.success("◇")}  Imported ${c.accent(String(words.length))} words from ${c.accent(format)} format → ${c.accent("transcript.json")}`,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Transcribe audio/video with whisper
+// ---------------------------------------------------------------------------
+
+async function transcribeAudio(
+  inputPath: string,
+  dir: string,
+  opts: { model?: string; language?: string; json?: boolean },
+): Promise<void> {
+  const { transcribe } = await import("../whisper/transcribe.js");
+  const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js");
+
+  const model = opts.model ?? DEFAULT_MODEL;
+  const spin = opts.json ? null : clack.spinner();
+  spin?.start(`Transcribing with ${c.accent(model)}...`);
+
+  try {
+    const result = await transcribe(inputPath, dir, {
+      model,
+      language: opts.language,
+      onProgress: spin ? (msg) => spin.message(msg) : undefined,
+    });
+
+    const { words } = loadTranscript(result.transcriptPath);
+    writeFileSync(result.transcriptPath, JSON.stringify(words, null, 2));
+    patchCaptionHtml(dir, words);
+
+    if (opts.json) {
+      console.log(
+        JSON.stringify({
+          ok: true,
+          model,
+          wordCount: words.length,
+          durationSeconds: result.durationSeconds,
+          transcriptPath: result.transcriptPath,
+        }),
+      );
+    } else {
+      spin!.stop(
+        c.success(
+          `Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s)`,
+        ),
+      );
+    }
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (opts.json) {
+      console.log(JSON.stringify({ ok: false, error: message }));
+    } else {
+      spin!.stop(c.error(`Transcription failed: ${message}`));
+    }
+    process.exit(1);
+  }
+}