mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
* fix(cli): transcribe reads whisper output with nan or inf values * fix(cli): transcribe also reads a Windows whisper's -nan(ind) as no value
957 lines
31 KiB
TypeScript
957 lines
31 KiB
TypeScript
import { describe, it, expect, afterEach } from "vitest";
|
||
import { writeFileSync, readFileSync, mkdtempSync, rmSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { tmpdir } from "node:os";
|
||
import { Script } from "node:vm";
|
||
import { Window } from "happy-dom";
|
||
import {
|
||
loadTranscript,
|
||
detectFormat,
|
||
patchCaptionHtml,
|
||
formatSrt,
|
||
formatVtt,
|
||
wordsToCues,
|
||
} from "./normalize.js";
|
||
import { detectSpeechOnset } from "./transcribe.js";
|
||
|
||
function tmpFile(name: string, content: string): string {
|
||
const dir = mkdtempSync(join(tmpdir(), "hf-normalize-test-"));
|
||
dirs.push(dir);
|
||
const path = join(dir, name);
|
||
writeFileSync(path, content);
|
||
return path;
|
||
}
|
||
|
||
let dirs: string[] = [];
|
||
|
||
afterEach(() => {
|
||
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
||
dirs = [];
|
||
});
|
||
|
||
describe("detectFormat", () => {
|
||
it("detects SRT by extension", () => {
|
||
const path = tmpFile("test.srt", "1\n00:00:01,000 --> 00:00:02,000\nHello\n");
|
||
expect(detectFormat(path)).toBe("srt");
|
||
});
|
||
|
||
it("detects VTT by extension", () => {
|
||
const path = tmpFile("test.vtt", "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nHello\n");
|
||
expect(detectFormat(path)).toBe("vtt");
|
||
});
|
||
|
||
it("detects a JSON transcript saved with a UTF-8 BOM", () => {
|
||
const path = tmpFile(
|
||
"bom.json",
|
||
`\uFEFF${JSON.stringify({ transcription: [{ offsets: { from: 0, to: 1000 }, text: " Hi", tokens: [{ text: " Hi", offsets: { from: 0, to: 1000 }, p: 0.9 }] }] })}`,
|
||
);
|
||
expect(detectFormat(path)).toBe("whisper-cpp");
|
||
});
|
||
|
||
it("detects whisper-cpp JSON", () => {
|
||
const path = tmpFile(
|
||
"transcript.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
offsets: { from: 0, to: 2000 },
|
||
text: " Hello world.",
|
||
tokens: [
|
||
{ text: " Hello", offsets: { from: 0, to: 1000 }, p: 0.98 },
|
||
{ text: " world", offsets: { from: 1000, to: 2000 }, p: 0.95 },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
expect(detectFormat(path)).toBe("whisper-cpp");
|
||
});
|
||
|
||
it("detects OpenAI JSON", () => {
|
||
const path = tmpFile(
|
||
"openai.json",
|
||
JSON.stringify({
|
||
words: [
|
||
{ word: "Hello", start: 0.0, end: 0.5 },
|
||
{ word: "world", start: 0.6, end: 1.2 },
|
||
],
|
||
}),
|
||
);
|
||
expect(detectFormat(path)).toBe("openai");
|
||
});
|
||
|
||
it("detects normalized word array", () => {
|
||
const path = tmpFile(
|
||
"words.json",
|
||
JSON.stringify([
|
||
{ text: "Hello", start: 0.0, end: 0.5 },
|
||
{ text: "world", start: 0.6, end: 1.2 },
|
||
]),
|
||
);
|
||
expect(detectFormat(path)).toBe("words-json");
|
||
});
|
||
});
|
||
|
||
describe("loadTranscript", () => {
|
||
it.each([
|
||
["00:00:01.000", "00:00:03.500", "align:start"],
|
||
["00:01.000", "00:03.500", "align:start"],
|
||
["00:00:01.000", "00:00:03.500", "line:90% position:50%,center size:80% align:center"],
|
||
["00:01.000", "00:03.500", "\tvertical:rl\tline:0"],
|
||
["00:00:01.000", "00:00:03.500", "region:captions"],
|
||
])("retains VTT end time before cue settings: %s --> %s %s", (start, end, settings) => {
|
||
const source = `WEBVTT\n\nfirst\n${start} --> ${end} ${settings}\nFirst phrase\n\nsecond\n00:04.000 --> 00:06.000\nSecond phrase\n`;
|
||
const { words } = loadTranscript(tmpFile("settings.vtt", source));
|
||
expect(words).toEqual([
|
||
{ text: "First phrase", start: 1, end: 3.5, id: "w0" },
|
||
{ text: "Second phrase", start: 4, end: 6, id: "w1" },
|
||
]);
|
||
expect(formatSrt(words, { preGrouped: true })).toBe(
|
||
"1\n00:00:01,000 --> 00:00:03,500\nFirst phrase\n\n2\n00:00:04,000 --> 00:00:06,000\nSecond phrase\n",
|
||
);
|
||
});
|
||
|
||
it("reads an empty word list as a transcript with no words", () => {
|
||
expect(loadTranscript(tmpFile("transcript.json", "[]"))).toEqual({
|
||
words: [],
|
||
format: "words-json",
|
||
});
|
||
});
|
||
|
||
it("still rejects a JSON array that is not a word list", () => {
|
||
expect(() => loadTranscript(tmpFile("transcript.json", '[{"foo":1}]'))).toThrow(
|
||
/Unrecognized JSON transcript format/,
|
||
);
|
||
});
|
||
|
||
it("reads whisper-cli output whose floats are nan, -nan, inf or MSVC's -nan(ind)", () => {
|
||
// Token objects as whisper.cpp's cli.cpp writes them: tab-indented, floats via iostream.
|
||
const token = (text: string, from: number, to: number, p: string) =>
|
||
[
|
||
"\t\t\t{",
|
||
`\t\t\t\t"text": "${text}",`,
|
||
`\t\t\t\t"timestamps": {`,
|
||
`\t\t\t\t\t"from": "00:00:00.000",`,
|
||
`\t\t\t\t\t"to": "00:00:00.500"`,
|
||
`\t\t\t\t},`,
|
||
`\t\t\t\t"offsets": {`,
|
||
`\t\t\t\t\t"from": ${from},`,
|
||
`\t\t\t\t\t"to": ${to}`,
|
||
`\t\t\t\t},`,
|
||
`\t\t\t\t"id": 50364,`,
|
||
`\t\t\t\t"p": ${p},`,
|
||
`\t\t\t\t"t_dtw": -1`,
|
||
"\t\t\t}",
|
||
].join("\n");
|
||
const raw = `{\n\t"transcription": [\n\t\t{\n\t\t\t"tokens": [\n${[
|
||
token(" nan", 0, 500, "nan"),
|
||
token(" world", 600, 1200, "-nan"),
|
||
token(" again", 1300, 1800, "inf"),
|
||
token(" later", 1900, 2400, "-nan(ind)"),
|
||
].join(",\n")}\n\t\t\t]\n\t\t}\n\t]\n}\n`;
|
||
const { words, format } = loadTranscript(tmpFile("transcript.json", raw));
|
||
expect(format).toBe("whisper-cpp");
|
||
expect(words).toEqual([
|
||
{ text: "nan", start: 0, end: 0.5, id: "w0" },
|
||
{ text: "world", start: 0.6, end: 1.2, id: "w1" },
|
||
{ text: "again", start: 1.3, end: 1.8, id: "w2" },
|
||
{ text: "later", start: 1.9, end: 2.4, id: "w3" },
|
||
]);
|
||
});
|
||
|
||
it("parses whisper-cpp JSON with punctuation merging", () => {
|
||
const path = tmpFile(
|
||
"transcript.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: " Hello", offsets: { from: 0, to: 500 } },
|
||
{ text: ",", offsets: { from: 500, to: 550 } },
|
||
{ text: " world", offsets: { from: 600, to: 1200 } },
|
||
{ text: ".", offsets: { from: 1200, to: 1250 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words, format } = loadTranscript(path);
|
||
expect(format).toBe("whisper-cpp");
|
||
expect(words).toEqual([
|
||
{ text: "Hello,", start: 0, end: 0.55, id: "w0" },
|
||
{ text: "world.", start: 0.6, end: 1.25, id: "w1" },
|
||
]);
|
||
});
|
||
|
||
it("filters whisper-cpp non-speech tokens", () => {
|
||
const path = tmpFile(
|
||
"transcript.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: "[_BEG_]", offsets: { from: 0, to: 0 } },
|
||
{ text: " Hello", offsets: { from: 100, to: 500 } },
|
||
{ text: "[BLANK_AUDIO]", offsets: { from: 500, to: 1000 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
expect(words).toHaveLength(1);
|
||
expect(words[0]?.text).toBe("Hello");
|
||
});
|
||
|
||
it("parses OpenAI Whisper API response", () => {
|
||
const path = tmpFile(
|
||
"openai.json",
|
||
JSON.stringify({
|
||
text: "Hello world",
|
||
words: [
|
||
{ word: "Hello", start: 0.0, end: 0.5 },
|
||
{ word: "world", start: 0.6, end: 1.2 },
|
||
],
|
||
}),
|
||
);
|
||
const { words, format } = loadTranscript(path);
|
||
expect(format).toBe("openai");
|
||
expect(words).toEqual([
|
||
{ text: "Hello", start: 0, end: 0.5, id: "w0" },
|
||
{ text: "world", start: 0.6, end: 1.2, id: "w1" },
|
||
]);
|
||
});
|
||
|
||
it("parses SRT files", () => {
|
||
const srt = `1
|
||
00:00:01,000 --> 00:00:03,500
|
||
Hello world
|
||
|
||
2
|
||
00:00:04,000 --> 00:00:06,000
|
||
How are you
|
||
`;
|
||
const path = tmpFile("captions.srt", srt);
|
||
const { words, format } = loadTranscript(path);
|
||
expect(format).toBe("srt");
|
||
expect(words).toEqual([
|
||
{ text: "Hello world", start: 1.0, end: 3.5, id: "w0" },
|
||
{ text: "How are you", start: 4.0, end: 6.0, id: "w1" },
|
||
]);
|
||
});
|
||
|
||
it("parses VTT files", () => {
|
||
const vtt = `WEBVTT
|
||
|
||
00:00:01.000 --> 00:00:03.500
|
||
Hello world
|
||
|
||
00:00:04.000 --> 00:00:06.000
|
||
How are you
|
||
`;
|
||
const path = tmpFile("captions.vtt", vtt);
|
||
const { words, format } = loadTranscript(path);
|
||
expect(format).toBe("vtt");
|
||
expect(words).toEqual([
|
||
{ text: "Hello world", start: 1.0, end: 3.5, id: "w0" },
|
||
{ text: "How are you", start: 4.0, end: 6.0, id: "w1" },
|
||
]);
|
||
});
|
||
|
||
it.each([
|
||
["ALICE: Hello there", "ALICE: Hello there"],
|
||
["ALICE: Hello\nBOB: Welcome", "ALICE: Hello BOB: Welcome"],
|
||
["Hello there\nALICE: Welcome", "Hello there ALICE: Welcome"],
|
||
["ALICE: Hello\nWelcome back", "ALICE: Hello Welcome back"],
|
||
["HOST-NAME: Welcome", "HOST-NAME: Welcome"],
|
||
["Alice: Hello there", "Alice: Hello there"],
|
||
])("preserves the VTT cue payload %j", (payload, text) => {
|
||
const vtt = `WEBVTT\nX-TIMESTAMP-MAP:LOCAL:00:00:00.000,MPEGTS:900000\n\n00:00:01.000 --> 00:00:03.500\n${payload}\n\n00:00:04.000 --> 00:00:06.000\nHow are you\n`;
|
||
const { words } = loadTranscript(tmpFile("speaker.vtt", vtt));
|
||
expect(words).toEqual([
|
||
{ text, start: 1, end: 3.5, id: "w0" },
|
||
{ text: "How are you", start: 4, end: 6, id: "w1" },
|
||
]);
|
||
expect(loadTranscript(tmpFile("speaker-roundtrip.vtt", formatVtt(words))).words).toEqual(words);
|
||
expect(loadTranscript(tmpFile("speaker-roundtrip.srt", formatSrt(words))).words).toEqual(words);
|
||
});
|
||
|
||
it("parses VTT with short timestamps (MM:SS.mmm)", () => {
|
||
const vtt = `WEBVTT
|
||
|
||
01:23.456 --> 02:00.000
|
||
Short format
|
||
`;
|
||
const path = tmpFile("short.vtt", vtt);
|
||
const { words } = loadTranscript(path);
|
||
expect(words[0]?.start).toBeCloseTo(83.456, 2);
|
||
expect(words[0]?.end).toBe(120.0);
|
||
});
|
||
|
||
it("strips HTML tags from SRT/VTT", () => {
|
||
const srt = `1
|
||
00:00:01,000 --> 00:00:03,000
|
||
<b>Bold</b> and <i>italic</i>
|
||
`;
|
||
const path = tmpFile("tags.srt", srt);
|
||
const { words } = loadTranscript(path);
|
||
expect(words[0]?.text).toBe("Bold and italic");
|
||
});
|
||
|
||
it("assigns w{index} ids to normalized word arrays", () => {
|
||
const input = [
|
||
{ text: "Hello", start: 0.0, end: 0.5 },
|
||
{ text: "world", start: 0.6, end: 1.2 },
|
||
];
|
||
const path = tmpFile("normalized.json", JSON.stringify(input));
|
||
const { words, format } = loadTranscript(path);
|
||
expect(format).toBe("words-json");
|
||
expect(words).toEqual([
|
||
{ text: "Hello", start: 0, end: 0.5, id: "w0" },
|
||
{ text: "world", start: 0.6, end: 1.2, id: "w1" },
|
||
]);
|
||
});
|
||
|
||
it("preserves existing ids and repairs empty-string ids from legacy files", () => {
|
||
const input = [
|
||
{ text: "Hello", start: 0.0, end: 0.5, id: "keep-me" },
|
||
{ text: "world", start: 0.6, end: 1.2, id: "" },
|
||
{ text: "again", start: 1.3, end: 1.8 },
|
||
];
|
||
const path = tmpFile("legacy.json", JSON.stringify(input));
|
||
const { words } = loadTranscript(path);
|
||
expect(words.map((w) => w.id)).toEqual(["keep-me", "w1", "w2"]);
|
||
});
|
||
});
|
||
|
||
describe("subtitle line endings", () => {
|
||
it.each([
|
||
["srt", "\r\n"],
|
||
["srt", "\r"],
|
||
["vtt", "\r\n"],
|
||
["vtt", "\r"],
|
||
])("keeps separate %s cues with %j line endings", (ext, newline) => {
|
||
const timestamp = ext === "srt" ? "," : ".";
|
||
const lines = [
|
||
...(ext === "vtt" ? ["WEBVTT", ""] : []),
|
||
"1",
|
||
`00:00:01${timestamp}000 --> 00:00:03${timestamp}500`,
|
||
"<b>Hello</b> world",
|
||
"Again",
|
||
"",
|
||
"2",
|
||
`00:00:04${timestamp}000 --> 00:00:06${timestamp}000`,
|
||
"How are you",
|
||
"",
|
||
];
|
||
const { words, format } = loadTranscript(tmpFile(`captions.${ext}`, lines.join(newline)));
|
||
expect(format).toBe(ext);
|
||
expect(words).toEqual([
|
||
{ text: "Hello world Again", start: 1, end: 3.5, id: "w0" },
|
||
{ text: "How are you", start: 4, end: 6, id: "w1" },
|
||
]);
|
||
expect(formatSrt(words, { preGrouped: true })).toBe(
|
||
"1\n00:00:01,000 --> 00:00:03,500\nHello world Again\n\n2\n00:00:04,000 --> 00:00:06,000\nHow are you\n",
|
||
);
|
||
});
|
||
|
||
it("preserves WebVTT cues when newline forms are mixed", () => {
|
||
const source =
|
||
"WEBVTT\r\n\r\n00:01.000 --> 00:02.000\rFirst phrase\r\r00:03.000 --> 00:04.000\nSecond phrase\n";
|
||
expect(loadTranscript(tmpFile("mixed.vtt", source)).words).toEqual([
|
||
{ text: "First phrase", start: 1, end: 2, id: "w0" },
|
||
{ text: "Second phrase", start: 3, end: 4, id: "w1" },
|
||
]);
|
||
});
|
||
});
|
||
|
||
describe("caption formatting", () => {
|
||
it.each([
|
||
["R&D <config> next", "R&D <config> next"],
|
||
["Literal < & references", "Literal &lt; &amp; references"],
|
||
["3 < 4 and 5 > 2", "3 < 4 and 5 > 2"],
|
||
["R&D --> next", "R&D --> next"],
|
||
])("preserves literal WebVTT text %j on export and reimport", (text, payload) => {
|
||
const words = [{ text, start: 1, end: 2, id: "w0" }];
|
||
const output = formatVtt(words);
|
||
expect(output).toBe(`WEBVTT\n\n00:00:01.000 --> 00:00:02.000\n${payload}\n`);
|
||
expect(loadTranscript(tmpFile("literal.vtt", output)).words).toEqual(words);
|
||
});
|
||
|
||
it.each([
|
||
["R&D <config> next", "R&D <config> next"],
|
||
[""quoted" 'text'", "\"quoted\" 'text'"],
|
||
["& < <", "& < <"],
|
||
["Literal &lt; &amp; references", "Literal < & references"],
|
||
["<b>R&D</b> <config>", "R&D <config>"],
|
||
["A B‎‏", "A\u00a0B\u200e\u200f"],
|
||
["Keep &unknown;", "Keep &unknown;"],
|
||
])("decodes the WebVTT cue payload %j once", (payload, text) => {
|
||
const input = `WEBVTT\n\n00:00:01.000 --> 00:00:02.000\n${payload}\n`;
|
||
expect(loadTranscript(tmpFile("entities.vtt", input)).words).toEqual([
|
||
{ text, start: 1, end: 2, id: "w0" },
|
||
]);
|
||
});
|
||
|
||
it("round-trips SRT cues through normalized words", () => {
|
||
const srt = `1
|
||
00:00:01,000 --> 00:00:03,500
|
||
Write HTML.
|
||
|
||
2
|
||
00:00:03,500 --> 00:00:06,000
|
||
Render video. Built for agents.
|
||
`;
|
||
const path = tmpFile("captions.srt", srt);
|
||
const { words } = loadTranscript(path);
|
||
|
||
const output = formatSrt(words);
|
||
expect(output).toBe(srt);
|
||
|
||
const reparsed = loadTranscript(tmpFile("roundtrip.srt", output));
|
||
expect(reparsed.words).toEqual(words);
|
||
});
|
||
|
||
it("round-trips VTT cues through normalized words", () => {
|
||
const vtt = `WEBVTT
|
||
|
||
00:00:01.000 --> 00:00:03.500
|
||
Write HTML.
|
||
|
||
00:00:03.500 --> 00:00:06.000
|
||
Render video. Built for agents.
|
||
`;
|
||
const path = tmpFile("captions.vtt", vtt);
|
||
const { words } = loadTranscript(path);
|
||
|
||
const output = formatVtt(words);
|
||
expect(output).toBe(vtt);
|
||
|
||
const reparsed = loadTranscript(tmpFile("roundtrip.vtt", output));
|
||
expect(reparsed.words).toEqual(words);
|
||
});
|
||
|
||
it("groups word-level transcript entries into readable cues", () => {
|
||
const cues = wordsToCues(
|
||
[
|
||
{ text: "Write", start: 0, end: 0.2 },
|
||
{ text: "HTML.", start: 0.2, end: 0.5 },
|
||
{ text: "Render", start: 0.7, end: 0.9 },
|
||
{ text: "video", start: 0.9, end: 1.1 },
|
||
{ text: "for", start: 1.1, end: 1.2 },
|
||
{ text: "agents.", start: 1.2, end: 1.6 },
|
||
{ text: "Fresh", start: 2.5, end: 2.8 },
|
||
{ text: "tracks.", start: 3.9, end: 4.1 },
|
||
],
|
||
{ maxChars: 18, maxGap: 0.8 },
|
||
);
|
||
|
||
expect(cues).toEqual([
|
||
{ text: "Write HTML.", start: 0, end: 0.5 },
|
||
{ text: "Render video for", start: 0.7, end: 1.2 },
|
||
{ text: "agents.", start: 1.2, end: 1.6 },
|
||
{ text: "Fresh", start: 2.5, end: 2.8 },
|
||
{ text: "tracks.", start: 3.9, end: 4.1 },
|
||
]);
|
||
});
|
||
|
||
it("joins CJK word-level tokens without inserting spaces", () => {
|
||
const cues = wordsToCues([
|
||
{ text: "你", start: 0, end: 0.3 },
|
||
{ text: "好", start: 0.3, end: 0.6 },
|
||
{ text: "世界", start: 0.6, end: 1.0 },
|
||
]);
|
||
expect(cues).toEqual([{ text: "你好世界", start: 0, end: 1 }]);
|
||
});
|
||
|
||
it("keeps phrase-level CJK entries as separate cues", () => {
|
||
// Chinese has no inter-word spaces, so the whitespace test cannot see
|
||
// that these are phrases; they used to collapse into one cue spanning
|
||
// the whole transcript.
|
||
const cues = wordsToCues([
|
||
{ text: "这是第一个句子", start: 0, end: 2 },
|
||
{ text: "这是第二个句子", start: 2, end: 4 },
|
||
{ text: "这是第三个句子", start: 4, end: 6 },
|
||
]);
|
||
expect(cues).toHaveLength(3);
|
||
expect(cues[0]).toEqual({
|
||
text: "这是第一个句子",
|
||
start: 0,
|
||
end: 2,
|
||
});
|
||
});
|
||
|
||
it("keeps phrase-level Thai entries as separate cues", () => {
|
||
const cues = wordsToCues([
|
||
{ text: "สวัสดีครับ", start: 0, end: 2 },
|
||
{ text: "ยินดีต้อนรับ", start: 2, end: 4 },
|
||
]);
|
||
expect(cues).toHaveLength(2);
|
||
});
|
||
|
||
it("treats entries at the length threshold as phrases", () => {
|
||
// Four characters is the boundary: at or above it the entries are read as
|
||
// phrase-level cues, below it as word-level tokens.
|
||
const cues = wordsToCues([
|
||
{ text: "你好世界", start: 0, end: 2 },
|
||
{ text: "谢谢大家", start: 2, end: 4 },
|
||
]);
|
||
expect(cues).toHaveLength(2);
|
||
});
|
||
|
||
it("still groups word-level CJK tokens into cues", () => {
|
||
// The mirror of the case above: short per-token entries are word-level
|
||
// whisper output and must still be joined.
|
||
const cues = wordsToCues([
|
||
{ text: "你", start: 0, end: 0.3 },
|
||
{ text: "好", start: 0.3, end: 0.6 },
|
||
{ text: "世", start: 0.6, end: 0.9 },
|
||
{ text: "界", start: 0.9, end: 1.2 },
|
||
]);
|
||
expect(cues).toEqual([{ text: "你好世界", start: 0, end: 1.2 }]);
|
||
});
|
||
|
||
it("preserves single-word cue boundaries when preGrouped", () => {
|
||
// Phrase-level cues without internal whitespace (one-word or CJK captions)
|
||
// must not merge — auto-detection can't see them, so the caller forces it.
|
||
const cues = wordsToCues(
|
||
[
|
||
{ text: "Yes", start: 0, end: 1 },
|
||
{ text: "No", start: 1, end: 2 },
|
||
],
|
||
{ preGrouped: true },
|
||
);
|
||
expect(cues).toEqual([
|
||
{ text: "Yes", start: 0, end: 1 },
|
||
{ text: "No", start: 1, end: 2 },
|
||
]);
|
||
});
|
||
});
|
||
|
||
describe("whisper-cpp contraction merging", () => {
|
||
it("merges didn + 't into didn't", () => {
|
||
const path = tmpFile(
|
||
"contractions.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: " I", offsets: { from: 0, to: 200 } },
|
||
{ text: " didn", offsets: { from: 200, to: 500 } },
|
||
{ text: "'t", offsets: { from: 500, to: 700 } },
|
||
{ text: " know", offsets: { from: 700, to: 1000 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
expect(words).toEqual([
|
||
{ text: "I", start: 0, end: 0.2, id: "w0" },
|
||
{ text: "didn't", start: 0.2, end: 0.7, id: "w1" },
|
||
{ text: "know", start: 0.7, end: 1, id: "w2" },
|
||
]);
|
||
});
|
||
|
||
it("merges I + 'm into I'm", () => {
|
||
const path = tmpFile(
|
||
"im.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: " I", offsets: { from: 0, to: 100 } },
|
||
{ text: "'m", offsets: { from: 100, to: 300 } },
|
||
{ text: " done", offsets: { from: 300, to: 600 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
expect(words[0]?.text).toBe("I'm");
|
||
expect(words[0]?.end).toBe(0.3);
|
||
});
|
||
|
||
it("merges could + 've into could've", () => {
|
||
const path = tmpFile(
|
||
"couldve.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: " could", offsets: { from: 0, to: 400 } },
|
||
{ text: "'ve", offsets: { from: 400, to: 600 } },
|
||
{ text: " been", offsets: { from: 600, to: 900 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
expect(words[0]?.text).toBe("could've");
|
||
});
|
||
});
|
||
|
||
describe("whisper-cpp fragment merging", () => {
|
||
it("merges single capital + lowercase: C + aught -> Caught", () => {
|
||
const path = tmpFile(
|
||
"fragments.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: " C", offsets: { from: 0, to: 100 } },
|
||
{ text: "aught", offsets: { from: 100, to: 500 } },
|
||
{ text: " a", offsets: { from: 500, to: 600 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
expect(words[0]?.text).toBe("Caught");
|
||
expect(words[0]?.end).toBe(0.5);
|
||
expect(words).toHaveLength(2);
|
||
});
|
||
|
||
it("merges consonant + in': shin + in' -> shinin'", () => {
|
||
const path = tmpFile(
|
||
"dropg.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: " shin", offsets: { from: 0, to: 300 } },
|
||
{ text: "in'", offsets: { from: 300, to: 500 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
expect(words).toHaveLength(1);
|
||
expect(words[0]?.text).toBe("shinin'");
|
||
});
|
||
});
|
||
|
||
describe("whisper-cpp CJK tokens", () => {
|
||
const token = (text: string, from: number, to: number) => ({ text, offsets: { from, to } });
|
||
|
||
// Shaped like whisper-cli --output-json-full on Mandarin speech: no token
|
||
// carries a leading space, not even the first one of a segment.
|
||
function chineseTranscript(): string {
|
||
return tmpFile(
|
||
"transcript.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
offsets: { from: 0, to: 3990 },
|
||
text: "大家好,今天用AI剪輯。",
|
||
tokens: [
|
||
token("[_BEG_]", 0, 0),
|
||
token("大家好", 10, 730),
|
||
token(",", 1110, 1290),
|
||
token("今天", 1360, 1680),
|
||
token("用", 1680, 2010),
|
||
token("AI", 2010, 2500),
|
||
token("剪", 2970, 3460),
|
||
token("輯", 3460, 3830),
|
||
token("。", 3830, 3990),
|
||
],
|
||
},
|
||
{
|
||
offsets: { from: 4000, to: 5600 },
|
||
text: "謝謝觀看。",
|
||
tokens: [
|
||
token("謝謝", 4000, 4500),
|
||
token("觀", 4500, 4900),
|
||
token("看", 4900, 5300),
|
||
token("。", 5300, 5600),
|
||
token("[_TT_280]", 5600, 5600),
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
}
|
||
|
||
it("keeps each CJK token as its own timed word", () => {
|
||
const { words } = loadTranscript(chineseTranscript());
|
||
expect(words.map((w) => [w.text, w.start, w.end])).toEqual([
|
||
["大家好,", 0.01, 1.29],
|
||
["今天", 1.36, 1.68],
|
||
["用", 1.68, 2.01],
|
||
["AI", 2.01, 2.5],
|
||
["剪", 2.97, 3.46],
|
||
["輯。", 3.46, 3.99],
|
||
["謝謝", 4, 4.5],
|
||
["觀", 4.5, 4.9],
|
||
["看。", 4.9, 5.6],
|
||
]);
|
||
});
|
||
|
||
it("splits Japanese kana tokens and merges full-width punctuation", () => {
|
||
const path = tmpFile(
|
||
"transcript.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
token("こんにち", 0, 400),
|
||
token("は", 400, 600),
|
||
token("、", 600, 700),
|
||
token("世界", 700, 1200),
|
||
token("!", 1200, 1300),
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
expect(words.map((w) => w.text)).toEqual(["こんにち", "は、", "世界!"]);
|
||
});
|
||
|
||
it("ends a cue at a CJK sentence mark", () => {
|
||
const { words } = loadTranscript(chineseTranscript());
|
||
expect(wordsToCues(words).map((c) => c.text)).toEqual(["大家好,今天用AI剪輯。", "謝謝觀看。"]);
|
||
});
|
||
});
|
||
|
||
describe("whisper-cpp zero-duration interpolation", () => {
|
||
it("interpolates a cluster of zero-duration words", () => {
|
||
const path = tmpFile(
|
||
"zerodur.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: " hello", offsets: { from: 0, to: 500 } },
|
||
{ text: " we", offsets: { from: 1000, to: 1000 } },
|
||
{ text: " are", offsets: { from: 1000, to: 1000 } },
|
||
{ text: " here", offsets: { from: 1000, to: 1000 } },
|
||
{ text: " now", offsets: { from: 1500, to: 2000 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
expect(words).toHaveLength(5);
|
||
// The three zero-duration words should be spread between 0.5 and 1.5
|
||
const we = words[1] ?? { start: 0, end: 0, text: "" };
|
||
const are = words[2] ?? { start: 0, end: 0, text: "" };
|
||
const here = words[3] ?? { start: 0, end: 0, text: "" };
|
||
expect(we.start).toBeCloseTo(0.5, 1);
|
||
expect(we.end).toBeCloseTo(0.833, 1);
|
||
expect(are.start).toBeCloseTo(0.833, 1);
|
||
expect(are.end).toBeCloseTo(1.167, 1);
|
||
expect(here.start).toBeCloseTo(1.167, 1);
|
||
expect(here.end).toBeCloseTo(1.5, 1);
|
||
// Each should have positive duration
|
||
expect(we.end).toBeGreaterThan(we.start);
|
||
expect(are.end).toBeGreaterThan(are.start);
|
||
expect(here.end).toBeGreaterThan(here.start);
|
||
});
|
||
|
||
it("handles isolated zero-duration word", () => {
|
||
const path = tmpFile(
|
||
"singlezero.json",
|
||
JSON.stringify({
|
||
transcription: [
|
||
{
|
||
tokens: [
|
||
{ text: " hello", offsets: { from: 0, to: 500 } },
|
||
{ text: " I", offsets: { from: 800, to: 800 } },
|
||
{ text: " know", offsets: { from: 1000, to: 1500 } },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
);
|
||
const { words } = loadTranscript(path);
|
||
const iWord = words[1] ?? { start: 0, end: 0, text: "" };
|
||
expect(iWord.end).toBeGreaterThan(iWord.start);
|
||
expect(iWord.start).toBeCloseTo(0.5, 1);
|
||
expect(iWord.end).toBeCloseTo(1, 1);
|
||
});
|
||
});
|
||
|
||
describe("patchCaptionHtml", () => {
|
||
it.each(["script", "TRANSCRIPT"])(
|
||
"replaces the complete %s array when old caption text contains an array terminator",
|
||
(name) => {
|
||
const dir = mkdtempSync(join(tmpdir(), "hf-patch-test-"));
|
||
dirs.push(dir);
|
||
const file = join(dir, "captions.html");
|
||
const oldWords = [{ text: "items[i];", start: 0, end: 1 }];
|
||
const prefix = `const ${name} = `;
|
||
const suffix = ";\nconst sentinel = true;";
|
||
writeFileSync(file, `<script>${prefix}${JSON.stringify(oldWords)}${suffix}</script>`);
|
||
const words = [{ id: "w0", text: "Replacement caption", start: 0, end: 1 }];
|
||
|
||
patchCaptionHtml(dir, words);
|
||
|
||
const html = readFileSync(file, "utf8");
|
||
const source = html.slice("<script>".length, -"</script>".length);
|
||
expect(() => new Script(source)).not.toThrow();
|
||
expect(source.endsWith(suffix)).toBe(true);
|
||
expect(JSON.parse(source.slice(prefix.length, -suffix.length))).toEqual(words);
|
||
},
|
||
);
|
||
|
||
it.each([
|
||
"</ScRiPt><span data-unexpected>caption</span>",
|
||
"</script\t><span data-unexpected>caption</span>",
|
||
"</script/><span data-unexpected>caption</span>",
|
||
"<!--<script>caption",
|
||
"$&",
|
||
"$`",
|
||
"$'",
|
||
"$$",
|
||
'quotes " and \\ and > & \u2028 \u2029 🎥',
|
||
])("preserves literal caption text in the HTML script: %s", (text) => {
|
||
const dir = mkdtempSync(join(tmpdir(), "hf-patch-test-"));
|
||
dirs.push(dir);
|
||
const file = join(dir, "captions.html");
|
||
const html =
|
||
'<html><body><span id="caption"></span><script>const TRANSCRIPT = [];</script></body></html>';
|
||
writeFileSync(file, html);
|
||
const words = [{ id: `word-${text}`, text, start: 0, end: 1 }];
|
||
|
||
patchCaptionHtml(dir, words);
|
||
|
||
const template = new Window().document.createElement("template");
|
||
template.innerHTML = readFileSync(file, "utf-8");
|
||
expect(template.content.querySelectorAll("script")).toHaveLength(1);
|
||
expect(template.content.querySelector("[data-unexpected]")).toBeNull();
|
||
const source = template.content.querySelector("script")?.textContent ?? "";
|
||
const json = source.slice("const TRANSCRIPT = ".length, -1);
|
||
expect(JSON.parse(json)).toEqual(words);
|
||
});
|
||
|
||
it("replaces const script = [] in HTML files", () => {
|
||
const dir = mkdtempSync(join(tmpdir(), "hf-patch-test-"));
|
||
dirs.push(dir);
|
||
|
||
const html = `<html><body><script>
|
||
const script = [];
|
||
console.log(script);
|
||
</script></body></html>`;
|
||
writeFileSync(join(dir, "captions.html"), html);
|
||
|
||
const words = [
|
||
{ text: "Hello", start: 1.0, end: 1.5 },
|
||
{ text: "world", start: 2.0, end: 2.5 },
|
||
];
|
||
patchCaptionHtml(dir, words);
|
||
|
||
const result = readFileSync(join(dir, "captions.html"), "utf-8");
|
||
expect(result).toContain('"Hello"');
|
||
expect(result).toContain('"world"');
|
||
expect(result).not.toContain("const script = [];");
|
||
});
|
||
|
||
it("replaces const TRANSCRIPT = [] variant", () => {
|
||
const dir = mkdtempSync(join(tmpdir(), "hf-patch-test-"));
|
||
dirs.push(dir);
|
||
|
||
const html = `<script>const TRANSCRIPT = [];</script>`;
|
||
writeFileSync(join(dir, "index.html"), html);
|
||
|
||
patchCaptionHtml(dir, [{ text: "Hi", start: 0, end: 1 }]);
|
||
|
||
const result = readFileSync(join(dir, "index.html"), "utf-8");
|
||
expect(result).toContain("const TRANSCRIPT = ");
|
||
expect(result).toContain('"Hi"');
|
||
});
|
||
|
||
it("does not modify HTML files without matching script patterns", () => {
|
||
const dir = mkdtempSync(join(tmpdir(), "hf-patch-test-"));
|
||
dirs.push(dir);
|
||
|
||
const html = `<html><body><script>console.log("hello");</script></body></html>`;
|
||
writeFileSync(join(dir, "page.html"), html);
|
||
|
||
patchCaptionHtml(dir, [{ text: "Hi", start: 0, end: 1 }]);
|
||
|
||
const result = readFileSync(join(dir, "page.html"), "utf-8");
|
||
expect(result).toBe(html);
|
||
});
|
||
|
||
it("skips empty word arrays", () => {
|
||
const dir = mkdtempSync(join(tmpdir(), "hf-patch-test-"));
|
||
dirs.push(dir);
|
||
|
||
const html = `<script>const script = [];</script>`;
|
||
writeFileSync(join(dir, "captions.html"), html);
|
||
|
||
patchCaptionHtml(dir, []);
|
||
|
||
const result = readFileSync(join(dir, "captions.html"), "utf-8");
|
||
expect(result).toBe(html);
|
||
});
|
||
});
|
||
|
||
describe("detectSpeechOnset", () => {
|
||
function makeSyntheticWav(
|
||
sampleRate: number,
|
||
durationSeconds: number,
|
||
energyFn: (t: number) => number,
|
||
): string {
|
||
const numSamples = Math.floor(sampleRate * durationSeconds);
|
||
const dataSize = numSamples * 2;
|
||
const buf = Buffer.alloc(44 + dataSize);
|
||
// RIFF header
|
||
buf.write("RIFF", 0);
|
||
buf.writeUInt32LE(36 + dataSize, 4);
|
||
buf.write("WAVE", 8);
|
||
buf.write("fmt ", 12);
|
||
buf.writeUInt32LE(16, 16); // chunk size
|
||
buf.writeUInt16LE(1, 20); // PCM
|
||
buf.writeUInt16LE(1, 22); // mono
|
||
buf.writeUInt32LE(sampleRate, 24);
|
||
buf.writeUInt32LE(sampleRate * 2, 28); // byte rate
|
||
buf.writeUInt16LE(2, 32); // block align
|
||
buf.writeUInt16LE(16, 34); // bits per sample
|
||
buf.write("data", 36);
|
||
buf.writeUInt32LE(dataSize, 40);
|
||
for (let i = 0; i < numSamples; i++) {
|
||
const t = i / sampleRate;
|
||
const amplitude = energyFn(t);
|
||
buf.writeInt16LE(Math.round(amplitude * 32767), 44 + i * 2);
|
||
}
|
||
const dir = mkdtempSync(join(tmpdir(), "hf-wav-test-"));
|
||
dirs.push(dir);
|
||
const path = join(dir, "tone.wav");
|
||
writeFileSync(path, buf);
|
||
return path;
|
||
}
|
||
|
||
it("detects onset when silence transitions to loud", () => {
|
||
const wavPath = makeSyntheticWav(16000, 15, (t) => (t < 5 ? 0.01 : 0.8));
|
||
const onset = detectSpeechOnset(wavPath);
|
||
expect(onset).not.toBeNull();
|
||
expect(onset!).toBeGreaterThanOrEqual(4);
|
||
expect(onset!).toBeLessThanOrEqual(7);
|
||
});
|
||
|
||
it("returns null for consistent energy throughout", () => {
|
||
const wavPath = makeSyntheticWav(16000, 10, () => 0.5);
|
||
const onset = detectSpeechOnset(wavPath);
|
||
expect(onset).toBeNull();
|
||
});
|
||
|
||
it("returns null for very short audio", () => {
|
||
const wavPath = makeSyntheticWav(16000, 2, () => 0.5);
|
||
const onset = detectSpeechOnset(wavPath);
|
||
expect(onset).toBeNull();
|
||
});
|
||
|
||
it("returns null when onset is too early (< 3s)", () => {
|
||
const wavPath = makeSyntheticWav(16000, 10, (t) => (t < 1 ? 0.01 : 0.8));
|
||
const onset = detectSpeechOnset(wavPath);
|
||
expect(onset).toBeNull();
|
||
});
|
||
});
|