mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix: render parity for transparent looped videos (#478)
## Summary - preserve alpha for render-injected video frames by detecting alpha streams with ffprobe and extracting alpha video frames as PNG - keep `<video loop>` semantics through static parsing, compiler duration resolution, browser media discovery, and render frame lookup - fail embedded preview startup before opening a broken browser page when the Studio bundle is missing - align snapshot frame injection with looped media timing and VP9 alpha extraction ## Why The Studio preview and rendered MP4 could disagree for timed transparent looped videos. The Comfy funding composition exposed two separate parity bugs: render-injected frames needed alpha-preserving PNG extraction, and the compiler was clamping a looped `data-duration="4"` video down to the 3.125s source duration. After the first source cycle, render lookup treated the video as inactive, hid the native video, and produced the blank polygon/glow the user saw around the rounded `0:03` mark. `hyperframes lint` and `hyperframes validate` did not catch this because they check syntax/load/console/accessibility, not preview-vs-render visual parity. This PR adds regression coverage for the compiler loop-duration path and frame lookup path. ## Verification - `bun run --filter @hyperframes/core test -- src/compiler/timingCompiler.test.ts src/compiler/htmlCompiler.test.ts` - `bun test packages/producer/src/services/htmlCompiler.test.ts` - `bun run --filter @hyperframes/engine test -- videoFrameExtractor ffprobe` - `bun run --filter @hyperframes/core typecheck` - `bun run --filter @hyperframes/engine typecheck` - `bun run --filter @hyperframes/producer typecheck` - `bun run --filter @hyperframes/cli typecheck` - `bun run lint` - `bun run format:check ...` on touched files - Comfy project: `node packages/cli/dist/cli.js validate` -> no console errors, 44 text elements pass WCAG AA - Comfy project patched render from source: `/tmp/comfy-render-compare/fixed6-comfy.mp4`, 1920x1080, 30fps, 21.8s, 654 frames - 3.00s-3.97s render contact sheet: `/tmp/comfy-render-compare/fixed6-window-contact.png` - targeted fixed render capture at 3.733s: `/tmp/comfy-render-compare/probe-capture-fixed/captured/frame_000112.jpg` - agent-browser Studio proof screenshot at 3.7s: `/tmp/comfy-render-compare/agent-browser-studio-3_7-fixed.png` - agent-browser-driven recording of 3s seek pass: `/tmp/comfy-render-compare/agent-browser-wysiwyg-3s-fixed.webm` Note: `bun run --filter @hyperframes/cli dev -- validate` is blocked in source mode by the existing `contrast-audit.browser.js` default-export loader issue; packaged `node packages/cli/dist/cli.js validate` passes for this project.
This commit is contained in:
Miguel Ángel
@@ -318,15 +318,31 @@ async function runEmbeddedMode(
|
||||
projectName?: string,
|
||||
forceNew = false,
|
||||
): Promise<void> {
|
||||
const { createStudioServer } = await import("../server/studioServer.js");
|
||||
const { createStudioServer, resolveStudioBundle } = await import("../server/studioServer.js");
|
||||
|
||||
const pName = projectName ?? basename(dir);
|
||||
const { app } = createStudioServer({ projectDir: dir, projectName: pName });
|
||||
const studioBundle = resolveStudioBundle();
|
||||
|
||||
clack.intro(c.bold("hyperframes preview"));
|
||||
const s = clack.spinner();
|
||||
s.start("Starting studio...");
|
||||
|
||||
if (!studioBundle.available) {
|
||||
s.stop(c.error("Studio build missing"));
|
||||
console.error();
|
||||
console.error(` ${c.dim("Could not find")} ${c.accent("index.html")} ${c.dim("in:")}`);
|
||||
for (const checkedPath of studioBundle.checkedPaths) {
|
||||
console.error(` ${c.dim("-")} ${checkedPath}`);
|
||||
}
|
||||
console.error();
|
||||
console.error(` ${c.dim("Rebuild the CLI package with")} ${c.accent("pnpm run build")}`);
|
||||
console.error();
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const { app } = createStudioServer({ projectDir: dir, projectName: pName });
|
||||
|
||||
let result: FindPortResult;
|
||||
try {
|
||||
result = await findPortAndServe(app.fetch, startPort, dir, forceNew);
|
||||
|
||||
@@ -25,6 +25,7 @@ const FFMPEG_EXTRACT_TIMEOUT_MS = 30_000;
|
||||
async function extractVideoFrameToBuffer(
|
||||
videoPath: string,
|
||||
timeSeconds: number,
|
||||
useVp9AlphaDecoder = false,
|
||||
): Promise<Buffer | null> {
|
||||
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
|
||||
const outPath = join(tmp, "frame.png");
|
||||
@@ -33,10 +34,11 @@ async function extractVideoFrameToBuffer(
|
||||
(resolvePromise) => {
|
||||
// `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
|
||||
// (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
|
||||
const ff = spawn("ffmpeg", [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
const args = ["-hide_banner", "-loglevel", "error"];
|
||||
if (useVp9AlphaDecoder) {
|
||||
args.push("-c:v", "libvpx-vp9");
|
||||
}
|
||||
args.push(
|
||||
"-ss",
|
||||
String(Math.max(0, timeSeconds)),
|
||||
"-i",
|
||||
@@ -47,7 +49,8 @@ async function extractVideoFrameToBuffer(
|
||||
"2",
|
||||
"-y",
|
||||
outPath,
|
||||
]);
|
||||
);
|
||||
const ff = spawn("ffmpeg", args);
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
@@ -252,19 +255,36 @@ async function captureSnapshots(
|
||||
updates: Array<{ videoId: string; dataUri: string }>,
|
||||
) => Promise<void>;
|
||||
type SyncVisibilityFn = (page: unknown, activeVideoIds: string[]) => Promise<void>;
|
||||
type ExtractMediaMetadataFn = (
|
||||
filePath: string,
|
||||
) => Promise<{ videoCodec: string; hasAlpha: boolean }>;
|
||||
let injectVideoFramesBatch: InjectFn | null = null;
|
||||
let syncVideoFrameVisibility: SyncVisibilityFn | null = null;
|
||||
let extractMediaMetadata: ExtractMediaMetadataFn | null = null;
|
||||
try {
|
||||
const engine = (await import("@hyperframes/engine")) as {
|
||||
injectVideoFramesBatch: InjectFn;
|
||||
syncVideoFrameVisibility: SyncVisibilityFn;
|
||||
extractMediaMetadata: ExtractMediaMetadataFn;
|
||||
};
|
||||
injectVideoFramesBatch = engine.injectVideoFramesBatch;
|
||||
syncVideoFrameVisibility = engine.syncVideoFrameVisibility;
|
||||
extractMediaMetadata = engine.extractMediaMetadata;
|
||||
} catch {
|
||||
// Engine unavailable in this install — snapshot will still run, and
|
||||
// compositions without <video data-start> get exactly the old behaviour.
|
||||
}
|
||||
const alphaDecoderCache = new Map<string, Promise<boolean>>();
|
||||
const shouldUseVp9AlphaDecoder = (filePath: string): Promise<boolean> => {
|
||||
if (!extractMediaMetadata) return Promise.resolve(false);
|
||||
const cached = alphaDecoderCache.get(filePath);
|
||||
if (cached) return cached;
|
||||
const pending = extractMediaMetadata(filePath)
|
||||
.then((meta) => meta.hasAlpha && meta.videoCodec === "vp9")
|
||||
.catch(() => false);
|
||||
alphaDecoderCache.set(filePath, pending);
|
||||
return pending;
|
||||
};
|
||||
|
||||
// Seek and capture each frame
|
||||
for (let i = 0; i < positions.length; i++) {
|
||||
@@ -324,7 +344,10 @@ async function captureSnapshots(
|
||||
: srcDur > 0
|
||||
? Math.max(0, (srcDur - mediaStart) / playbackRate)
|
||||
: Number.POSITIVE_INFINITY;
|
||||
const relTime = (t - start) * playbackRate + mediaStart;
|
||||
let relTime = (t - start) * playbackRate + mediaStart;
|
||||
if (v.loop && srcDur > mediaStart && relTime >= srcDur) {
|
||||
relTime = mediaStart + ((relTime - mediaStart) % (srcDur - mediaStart));
|
||||
}
|
||||
const activeNow = t >= start && t < start + duration && relTime >= 0 && !!v.id;
|
||||
return {
|
||||
id: v.id,
|
||||
@@ -356,7 +379,11 @@ async function captureSnapshots(
|
||||
/* unresolvable src (e.g. blob:, data:) — skip */
|
||||
}
|
||||
if (!filePath) continue;
|
||||
const png = await extractVideoFrameToBuffer(filePath, Math.max(0, v.relTime));
|
||||
const png = await extractVideoFrameToBuffer(
|
||||
filePath,
|
||||
Math.max(0, v.relTime),
|
||||
await shouldUseVp9AlphaDecoder(filePath),
|
||||
);
|
||||
if (!png) continue;
|
||||
updates.push({
|
||||
videoId: v.id,
|
||||
|
||||
@@ -23,11 +23,38 @@ import {
|
||||
// ── Path resolution ─────────────────────────────────────────────────────────
|
||||
|
||||
function resolveDistDir(): string {
|
||||
return resolveStudioBundle().dir;
|
||||
}
|
||||
|
||||
export interface StudioBundleResolution {
|
||||
dir: string;
|
||||
indexPath: string;
|
||||
available: boolean;
|
||||
checkedPaths: string[];
|
||||
}
|
||||
|
||||
export function resolveStudioBundle(): StudioBundleResolution {
|
||||
const builtPath = resolve(__dirname, "studio");
|
||||
if (existsSync(resolve(builtPath, "index.html"))) return builtPath;
|
||||
const builtIndex = resolve(builtPath, "index.html");
|
||||
if (existsSync(builtIndex)) {
|
||||
return { dir: builtPath, indexPath: builtIndex, available: true, checkedPaths: [builtIndex] };
|
||||
}
|
||||
const devPath = resolve(__dirname, "..", "..", "..", "studio", "dist");
|
||||
if (existsSync(resolve(devPath, "index.html"))) return devPath;
|
||||
return builtPath;
|
||||
const devIndex = resolve(devPath, "index.html");
|
||||
if (existsSync(devIndex)) {
|
||||
return {
|
||||
dir: devPath,
|
||||
indexPath: devIndex,
|
||||
available: true,
|
||||
checkedPaths: [builtIndex, devIndex],
|
||||
};
|
||||
}
|
||||
return {
|
||||
dir: builtPath,
|
||||
indexPath: builtIndex,
|
||||
available: false,
|
||||
checkedPaths: [builtIndex, devIndex],
|
||||
};
|
||||
}
|
||||
|
||||
function resolveRuntimePath(): string {
|
||||
@@ -348,7 +375,60 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
app.get("*", (c) => {
|
||||
const indexPath = resolve(studioDir, "index.html");
|
||||
if (!existsSync(indexPath)) {
|
||||
return c.text("Studio not found. Rebuild with: pnpm run build", 500);
|
||||
return c.html(
|
||||
`<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>HyperFrames Studio unavailable</title>
|
||||
<style>
|
||||
body {
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
background: #0d0f14;
|
||||
color: #eef2f7;
|
||||
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
}
|
||||
main {
|
||||
width: min(560px, calc(100vw - 48px));
|
||||
border: 1px solid rgba(255, 255, 255, 0.14);
|
||||
border-radius: 8px;
|
||||
padding: 28px;
|
||||
background: #151923;
|
||||
}
|
||||
h1 {
|
||||
margin: 0 0 12px;
|
||||
font-size: 22px;
|
||||
line-height: 1.2;
|
||||
}
|
||||
p {
|
||||
margin: 0 0 18px;
|
||||
color: #aab3c2;
|
||||
line-height: 1.5;
|
||||
}
|
||||
code {
|
||||
display: block;
|
||||
padding: 12px 14px;
|
||||
border-radius: 6px;
|
||||
background: #090b10;
|
||||
color: #8ff0c2;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Studio bundle missing</h1>
|
||||
<p>The preview server started, but this CLI build does not contain the Studio assets.</p>
|
||||
<code>pnpm run build</code>
|
||||
</main>
|
||||
</body>
|
||||
</html>`,
|
||||
500,
|
||||
);
|
||||
}
|
||||
return c.html(readFileSync(indexPath, "utf-8"));
|
||||
});
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { compileHtml } from "./htmlCompiler.js";
|
||||
|
||||
describe("compileHtml", () => {
|
||||
it("preserves explicit looped media durations that exceed source duration", async () => {
|
||||
const html =
|
||||
'<video id="hero" src="hero.webm" data-start="0" data-duration="4" data-end="4" loop>';
|
||||
|
||||
const compiled = await compileHtml(html, "/project", async () => 3.125);
|
||||
|
||||
expect(compiled).toContain('data-duration="4"');
|
||||
expect(compiled).toContain('data-end="4"');
|
||||
});
|
||||
|
||||
it("still clamps non-looping media durations to source duration", async () => {
|
||||
const html = '<video id="hero" src="hero.webm" data-start="0" data-duration="4" data-end="4">';
|
||||
|
||||
const compiled = await compileHtml(html, "/project", async () => 3.125);
|
||||
|
||||
expect(compiled).toContain('data-duration="3.125"');
|
||||
expect(compiled).toContain('data-end="3.125"');
|
||||
});
|
||||
});
|
||||
@@ -70,6 +70,7 @@ export async function compileHtml(
|
||||
|
||||
for (const el of preResolved) {
|
||||
if (!el.src) continue;
|
||||
if (el.loop) continue;
|
||||
const src = resolveMediaSrc(el.src, projectDir);
|
||||
const fileDuration = await probeMediaDuration(src);
|
||||
if (fileDuration <= 0) continue;
|
||||
|
||||
@@ -115,11 +115,26 @@ describe("extractResolvedMedia", () => {
|
||||
expect(resolved[0].tagName).toBe("video");
|
||||
expect(resolved[0].duration).toBe(5);
|
||||
expect(resolved[0].start).toBe(1);
|
||||
expect(resolved[0].loop).toBe(false);
|
||||
expect(resolved[1].id).toBe("a1");
|
||||
expect(resolved[1].tagName).toBe("audio");
|
||||
expect(resolved[1].duration).toBe(10);
|
||||
});
|
||||
|
||||
it("marks looped media so render compilation can preserve display duration", () => {
|
||||
const html = '<video id="v1" src="vid.webm" data-start="0" data-duration="4" loop>';
|
||||
|
||||
const resolved = extractResolvedMedia(html);
|
||||
|
||||
expect(resolved).toHaveLength(1);
|
||||
expect(resolved[0]).toMatchObject({
|
||||
id: "v1",
|
||||
tagName: "video",
|
||||
duration: 4,
|
||||
loop: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("skips elements with invalid durations", () => {
|
||||
const html = '<video id="v1" src="a.mp4" data-start="0" data-duration="NaN">';
|
||||
const resolved = extractResolvedMedia(html);
|
||||
|
||||
@@ -40,6 +40,7 @@ export interface ResolvedMediaElement {
|
||||
start: number;
|
||||
duration: number;
|
||||
mediaStart: number;
|
||||
loop: boolean;
|
||||
}
|
||||
|
||||
export interface CompilationResult {
|
||||
@@ -55,7 +56,7 @@ function getAttr(tag: string, attr: string): string | null {
|
||||
}
|
||||
|
||||
function hasAttr(tag: string, attr: string): boolean {
|
||||
return new RegExp(`${attr}=["']`).test(tag);
|
||||
return new RegExp(`\\s${attr}(?:\\s|=|>|/)`).test(tag);
|
||||
}
|
||||
|
||||
function injectAttr(tag: string, attr: string, value: string): string {
|
||||
@@ -229,6 +230,7 @@ export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
|
||||
start: startStr !== null ? parseFloat(startStr) : 0,
|
||||
duration,
|
||||
mediaStart: mediaStartStr ? parseFloat(mediaStartStr) : 0,
|
||||
loop: hasAttr(tag, "loop"),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,9 @@ import {
|
||||
parseVideoElements,
|
||||
parseImageElements,
|
||||
extractAllVideoFrames,
|
||||
createFrameLookupTable,
|
||||
type VideoElement,
|
||||
type ExtractedFrames,
|
||||
} from "./videoFrameExtractor.js";
|
||||
import { extractVideoMetadata } from "../utils/ffprobe.js";
|
||||
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
||||
@@ -32,6 +34,7 @@ describe("parseVideoElements", () => {
|
||||
start: 0,
|
||||
end: Infinity,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
});
|
||||
});
|
||||
@@ -48,9 +51,97 @@ describe("parseVideoElements", () => {
|
||||
start: 2,
|
||||
end: 7,
|
||||
mediaStart: 1.5,
|
||||
loop: false,
|
||||
hasAudio: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves looped timed video semantics for render frame lookup", () => {
|
||||
const videos = parseVideoElements(
|
||||
'<video id="hero" src="clip.webm" data-start="2" data-duration="5" loop></video>',
|
||||
);
|
||||
|
||||
expect(videos[0]).toMatchObject({
|
||||
id: "hero",
|
||||
start: 2,
|
||||
end: 7,
|
||||
loop: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("FrameLookupTable", () => {
|
||||
function fakeExtracted(totalFrames: number, fps: number): ExtractedFrames {
|
||||
const framePaths = new Map<number, string>();
|
||||
for (let i = 0; i < totalFrames; i += 1) {
|
||||
framePaths.set(i, `frame-${i}.jpg`);
|
||||
}
|
||||
return {
|
||||
videoId: "hero",
|
||||
srcPath: "clip.webm",
|
||||
outputDir: "/tmp/frames",
|
||||
framePattern: "frame-%05d.jpg",
|
||||
fps,
|
||||
totalFrames,
|
||||
metadata: {
|
||||
durationSeconds: totalFrames / fps,
|
||||
width: 320,
|
||||
height: 180,
|
||||
fps,
|
||||
hasAudio: false,
|
||||
videoCodec: "vp9",
|
||||
colorSpace: {
|
||||
colorTransfer: "bt709",
|
||||
colorPrimaries: "bt709",
|
||||
colorSpace: "bt709",
|
||||
},
|
||||
isVFR: false,
|
||||
hasAlpha: false,
|
||||
},
|
||||
framePaths,
|
||||
};
|
||||
}
|
||||
|
||||
it("wraps active frame payloads for looped clips whose display window exceeds source frames", () => {
|
||||
const table = createFrameLookupTable(
|
||||
[
|
||||
{
|
||||
id: "hero",
|
||||
src: "clip.webm",
|
||||
start: 0,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
loop: true,
|
||||
hasAudio: false,
|
||||
},
|
||||
],
|
||||
[fakeExtracted(30, 30)],
|
||||
);
|
||||
|
||||
expect(table.getActiveFramePayloads(0.5).get("hero")?.frameIndex).toBe(15);
|
||||
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(15);
|
||||
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
|
||||
});
|
||||
|
||||
it("does not hold stale frames for non-looping clips after extracted frames end", () => {
|
||||
const table = createFrameLookupTable(
|
||||
[
|
||||
{
|
||||
id: "hero",
|
||||
src: "clip.webm",
|
||||
start: 0,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
},
|
||||
],
|
||||
[fakeExtracted(30, 30)],
|
||||
);
|
||||
|
||||
expect(table.getActiveFramePayloads(0.5).has("hero")).toBe(true);
|
||||
expect(table.getActiveFramePayloads(1.5).has("hero")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseImageElements", () => {
|
||||
@@ -175,6 +266,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
||||
start: 0,
|
||||
end: 4,
|
||||
mediaStart: 3,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
};
|
||||
|
||||
@@ -229,6 +321,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
||||
start: 0,
|
||||
end: 2,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
};
|
||||
|
||||
@@ -298,6 +391,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
||||
start: 0,
|
||||
end: 1,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
};
|
||||
|
||||
@@ -388,8 +482,16 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const videos: VideoElement[] = [
|
||||
{ id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, hasAudio: false },
|
||||
{ id: "hdr", src: HDR_SHORT, start: 2, end: 4, mediaStart: 0, hasAudio: false },
|
||||
{ id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, loop: false, hasAudio: false },
|
||||
{
|
||||
id: "hdr",
|
||||
src: HDR_SHORT,
|
||||
start: 2,
|
||||
end: 4,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
},
|
||||
];
|
||||
|
||||
const result = await extractAllVideoFrames(videos, FIXTURE_DIR, {
|
||||
@@ -424,6 +526,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
||||
start: 0,
|
||||
end: 10,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
};
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ export interface VideoElement {
|
||||
start: number;
|
||||
end: number;
|
||||
mediaStart: number;
|
||||
loop: boolean;
|
||||
hasAudio: boolean;
|
||||
}
|
||||
|
||||
@@ -141,6 +142,7 @@ export function parseVideoElements(html: string): VideoElement[] {
|
||||
start,
|
||||
end,
|
||||
mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
|
||||
loop: el.hasAttribute("loop"),
|
||||
hasAudio: hasAudioAttr === "true",
|
||||
});
|
||||
}
|
||||
@@ -207,12 +209,13 @@ export async function extractVideoFramesRange(
|
||||
outputDirOverride?: string,
|
||||
): Promise<ExtractedFrames> {
|
||||
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
|
||||
const { fps, outputDir, quality = 95, format = "jpg" } = options;
|
||||
const { fps, outputDir, quality = 95 } = options;
|
||||
|
||||
const videoOutputDir = outputDirOverride ?? join(outputDir, videoId);
|
||||
if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true });
|
||||
|
||||
const metadata = await extractMediaMetadata(videoPath);
|
||||
const format = resolveFrameFormat(metadata, options.format);
|
||||
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`;
|
||||
const outputPattern = join(videoOutputDir, framePattern);
|
||||
|
||||
@@ -227,6 +230,9 @@ export async function extractVideoFramesRange(
|
||||
if (isHdr && isMacOS) {
|
||||
args.push("-hwaccel", "videotoolbox");
|
||||
}
|
||||
if (metadata.hasAlpha && metadata.videoCodec === "vp9") {
|
||||
args.push("-c:v", "libvpx-vp9");
|
||||
}
|
||||
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
|
||||
|
||||
const vfFilters: string[] = [];
|
||||
@@ -392,6 +398,11 @@ function resolveSegmentDuration(
|
||||
return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
|
||||
}
|
||||
|
||||
function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat {
|
||||
if (requested) return requested;
|
||||
return metadata.hasAlpha ? "png" : "jpg";
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-encode a VFR (variable frame rate) video segment to CFR so the downstream
|
||||
* fps filter can extract frames reliably. Screen recordings, phone videos, and
|
||||
@@ -682,7 +693,6 @@ export async function extractAllVideoFrames(
|
||||
|
||||
const phase3Start = Date.now();
|
||||
const cacheRootDir = config?.extractCacheDir;
|
||||
const cacheFormat: CacheFrameFormat = options.format ?? "jpg";
|
||||
|
||||
async function tryCachedExtract(
|
||||
video: VideoElement,
|
||||
@@ -694,6 +704,7 @@ export async function extractAllVideoFrames(
|
||||
const keyInput = cacheKeyInputs[i];
|
||||
const probedMeta = videoMetadata[i];
|
||||
if (!keyInput || !probedMeta) return null;
|
||||
const cacheFormat = resolveFrameFormat(probedMeta, options.format);
|
||||
|
||||
const keyDuration = resolveSegmentDuration(
|
||||
keyInput.end - keyInput.start,
|
||||
@@ -729,7 +740,7 @@ export async function extractAllVideoFrames(
|
||||
video.id,
|
||||
video.mediaStart,
|
||||
videoDuration,
|
||||
options,
|
||||
{ ...options, format: cacheFormat },
|
||||
signal,
|
||||
config,
|
||||
lookup.entry.dir,
|
||||
@@ -764,7 +775,7 @@ export async function extractAllVideoFrames(
|
||||
video.id,
|
||||
video.mediaStart,
|
||||
videoDuration,
|
||||
options,
|
||||
{ ...options, format: resolveFrameFormat(probedMeta, options.format) },
|
||||
signal,
|
||||
config,
|
||||
);
|
||||
@@ -807,10 +818,19 @@ export function getFrameAtTime(
|
||||
extracted: ExtractedFrames,
|
||||
globalTime: number,
|
||||
videoStart: number,
|
||||
loop = false,
|
||||
mediaStart = 0,
|
||||
): string | null {
|
||||
const localTime = globalTime - videoStart;
|
||||
let localTime = globalTime - videoStart;
|
||||
if (localTime < 0) return null;
|
||||
const loopDuration = Math.max(0, extracted.metadata.durationSeconds - mediaStart);
|
||||
if (loop && loopDuration > 0 && localTime >= loopDuration) {
|
||||
localTime %= loopDuration;
|
||||
}
|
||||
const frameIndex = Math.floor(localTime * extracted.fps);
|
||||
if (loop && frameIndex >= extracted.totalFrames && extracted.totalFrames > 0) {
|
||||
return extracted.framePaths.get(extracted.totalFrames - 1) || null;
|
||||
}
|
||||
if (frameIndex < 0 || frameIndex >= extracted.totalFrames) return null;
|
||||
return extracted.framePaths.get(frameIndex) || null;
|
||||
}
|
||||
@@ -823,6 +843,7 @@ export class FrameLookupTable {
|
||||
start: number;
|
||||
end: number;
|
||||
mediaStart: number;
|
||||
loop: boolean;
|
||||
}
|
||||
> = new Map();
|
||||
private orderedVideos: Array<{
|
||||
@@ -831,13 +852,20 @@ export class FrameLookupTable {
|
||||
start: number;
|
||||
end: number;
|
||||
mediaStart: number;
|
||||
loop: boolean;
|
||||
}> = [];
|
||||
private activeVideoIds: Set<string> = new Set();
|
||||
private startCursor = 0;
|
||||
private lastTime: number | null = null;
|
||||
|
||||
addVideo(extracted: ExtractedFrames, start: number, end: number, mediaStart: number): void {
|
||||
this.videos.set(extracted.videoId, { extracted, start, end, mediaStart });
|
||||
addVideo(
|
||||
extracted: ExtractedFrames,
|
||||
start: number,
|
||||
end: number,
|
||||
mediaStart: number,
|
||||
loop = false,
|
||||
): void {
|
||||
this.videos.set(extracted.videoId, { extracted, start, end, mediaStart, loop });
|
||||
this.orderedVideos = Array.from(this.videos.entries())
|
||||
.map(([videoId, video]) => ({ videoId, ...video }))
|
||||
.sort((a, b) => a.start - b.start);
|
||||
@@ -848,7 +876,7 @@ export class FrameLookupTable {
|
||||
const video = this.videos.get(videoId);
|
||||
if (!video) return null;
|
||||
if (globalTime < video.start || globalTime >= video.end) return null;
|
||||
return getFrameAtTime(video.extracted, globalTime, video.start);
|
||||
return getFrameAtTime(video.extracted, globalTime, video.start, video.loop, video.mediaStart);
|
||||
}
|
||||
|
||||
private resetActiveState(): void {
|
||||
@@ -904,8 +932,19 @@ export class FrameLookupTable {
|
||||
for (const videoId of this.activeVideoIds) {
|
||||
const video = this.videos.get(videoId);
|
||||
if (!video) continue;
|
||||
const localTime = globalTime - video.start;
|
||||
let localTime = globalTime - video.start;
|
||||
const loopDuration = Math.max(0, video.extracted.metadata.durationSeconds - video.mediaStart);
|
||||
if (video.loop && loopDuration > 0 && localTime >= loopDuration) {
|
||||
localTime %= loopDuration;
|
||||
}
|
||||
const frameIndex = Math.floor(localTime * video.extracted.fps);
|
||||
if (video.loop && frameIndex >= video.extracted.totalFrames) {
|
||||
const framePath = video.extracted.framePaths.get(video.extracted.totalFrames - 1);
|
||||
if (framePath) {
|
||||
frames.set(videoId, { framePath, frameIndex: video.extracted.totalFrames - 1 });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (frameIndex < 0 || frameIndex >= video.extracted.totalFrames) continue;
|
||||
const framePath = video.extracted.framePaths.get(frameIndex);
|
||||
if (!framePath) continue;
|
||||
@@ -949,7 +988,7 @@ export function createFrameLookupTable(
|
||||
|
||||
for (const video of videos) {
|
||||
const ext = extractedMap.get(video.id);
|
||||
if (ext) table.addVideo(ext, video.start, video.end, video.mediaStart);
|
||||
if (ext) table.addVideo(ext, video.start, video.end, video.mediaStart, video.loop);
|
||||
}
|
||||
|
||||
return table;
|
||||
|
||||
@@ -188,10 +188,43 @@ describe("ffprobe missing-binary fallback", () => {
|
||||
expect(meta.fps).toBe(0);
|
||||
expect(meta.hasAudio).toBe(false);
|
||||
expect(meta.isVFR).toBe(false);
|
||||
expect(meta.hasAlpha).toBe(false);
|
||||
expect(meta.colorSpace?.colorTransfer).toBe("smpte2084");
|
||||
expect(meta.colorSpace?.colorPrimaries).toBe("bt2020");
|
||||
});
|
||||
|
||||
it("extractMediaMetadata detects VP9 alpha_mode streams", async () => {
|
||||
const { spawn } = createSpawnSpy([
|
||||
{
|
||||
kind: "exit",
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [
|
||||
{
|
||||
codec_type: "video",
|
||||
codec_name: "vp9",
|
||||
width: 320,
|
||||
height: 180,
|
||||
r_frame_rate: "30/1",
|
||||
avg_frame_rate: "30/1",
|
||||
pix_fmt: "yuv420p",
|
||||
tags: { alpha_mode: "1" },
|
||||
},
|
||||
],
|
||||
format: { duration: "1.5" },
|
||||
}),
|
||||
},
|
||||
]);
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
|
||||
const { extractMediaMetadata: extractMediaMetadataMocked } = await import("./ffprobe.js");
|
||||
const meta = await extractMediaMetadataMocked("/tmp/alpha.webm");
|
||||
|
||||
expect(meta.videoCodec).toBe("vp9");
|
||||
expect(meta.hasAlpha).toBe(true);
|
||||
});
|
||||
|
||||
it("extractMediaMetadata rethrows ffprobe-missing error for non-image files without fallback", async () => {
|
||||
const { spawn } = createSpawnSpy([{ kind: "missing" }]);
|
||||
vi.resetModules();
|
||||
|
||||
@@ -62,6 +62,8 @@ export interface VideoMetadata {
|
||||
hasAudio: boolean;
|
||||
/** True when r_frame_rate and avg_frame_rate differ significantly (>10%), indicating variable frame rate. */
|
||||
isVFR: boolean;
|
||||
/** True when the stream carries an alpha channel. */
|
||||
hasAlpha: boolean;
|
||||
/** Color space info from the video stream. Null if ffprobe didn't report it. */
|
||||
colorSpace: VideoColorSpace | null;
|
||||
}
|
||||
@@ -79,6 +81,7 @@ interface FFProbeStream {
|
||||
codec_name?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
pix_fmt?: string;
|
||||
r_frame_rate?: string;
|
||||
avg_frame_rate?: string;
|
||||
sample_rate?: string;
|
||||
@@ -86,6 +89,7 @@ interface FFProbeStream {
|
||||
color_transfer?: string;
|
||||
color_primaries?: string;
|
||||
color_space?: string;
|
||||
tags?: Record<string, string>;
|
||||
}
|
||||
|
||||
interface FFProbeFormat {
|
||||
@@ -251,6 +255,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
|
||||
videoCodec: "png",
|
||||
hasAudio: false,
|
||||
isVFR: false,
|
||||
hasAlpha: false,
|
||||
colorSpace: stillImageMeta.colorSpace,
|
||||
};
|
||||
}
|
||||
@@ -271,6 +276,10 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
|
||||
? { colorTransfer, colorPrimaries, colorSpace: colorSpaceVal }
|
||||
: null;
|
||||
const colorSpace = ffprobeColorSpace ?? stillImageMeta?.colorSpace ?? null;
|
||||
const pixelFormat = videoStream.pix_fmt || "";
|
||||
const alphaMode = videoStream.tags?.alpha_mode || "";
|
||||
const hasAlpha =
|
||||
/(^|[^a-z])yuva|rgba|argb|bgra|gbrap|gray[a-z0-9]*a/i.test(pixelFormat) || alphaMode === "1";
|
||||
|
||||
return {
|
||||
durationSeconds: output?.format.duration ? parseFloat(output.format.duration) : 0,
|
||||
@@ -280,6 +289,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
|
||||
videoCodec: videoStream.codec_name || "unknown",
|
||||
hasAudio: output?.streams.some((s) => s.codec_type === "audio") ?? false,
|
||||
isVFR,
|
||||
hasAlpha,
|
||||
colorSpace,
|
||||
};
|
||||
})();
|
||||
|
||||
@@ -196,7 +196,7 @@ async function compileHtmlFile(
|
||||
const preResolved = extractResolvedMedia(compiledHtml);
|
||||
const clampResults = await Promise.all(
|
||||
preResolved
|
||||
.filter((el) => !!el.src)
|
||||
.filter((el) => !!el.src && !el.loop)
|
||||
.map(async (el) => {
|
||||
const { duration: maxDuration } = await resolveMediaDuration(
|
||||
el.src!,
|
||||
@@ -1097,6 +1097,7 @@ export interface BrowserMediaElement {
|
||||
end: number;
|
||||
duration: number;
|
||||
mediaStart: number;
|
||||
loop: boolean;
|
||||
hasAudio: boolean;
|
||||
volume: number;
|
||||
}
|
||||
@@ -1111,6 +1112,7 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
|
||||
end: number;
|
||||
duration: number;
|
||||
mediaStart: number;
|
||||
loop: boolean;
|
||||
hasAudio: boolean;
|
||||
volume: number;
|
||||
}[] = [];
|
||||
@@ -1126,6 +1128,7 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
|
||||
const end = parseFloat(htmlEl.getAttribute("data-end") || "0");
|
||||
const duration = parseFloat(htmlEl.getAttribute("data-duration") || "0");
|
||||
const mediaStart = parseFloat(htmlEl.getAttribute("data-media-start") || "0");
|
||||
const loop = htmlEl.hasAttribute("loop");
|
||||
const hasAudio = htmlEl.getAttribute("data-has-audio") === "true";
|
||||
const volume = parseFloat(htmlEl.getAttribute("data-volume") || "1");
|
||||
|
||||
@@ -1137,6 +1140,7 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
|
||||
end,
|
||||
duration,
|
||||
mediaStart,
|
||||
loop,
|
||||
hasAudio,
|
||||
volume,
|
||||
});
|
||||
|
||||
@@ -1231,6 +1231,9 @@ export async function executeRenderJob(
|
||||
if (el.hasAudio && !existing.hasAudio) {
|
||||
existing.hasAudio = true;
|
||||
}
|
||||
if (el.loop && !existing.loop) {
|
||||
existing.loop = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// New video discovered from browser
|
||||
@@ -1240,6 +1243,7 @@ export async function executeRenderJob(
|
||||
start: el.start,
|
||||
end: el.end,
|
||||
mediaStart: el.mediaStart,
|
||||
loop: el.loop,
|
||||
hasAudio: el.hasAudio,
|
||||
});
|
||||
existingVideoIds.add(el.id);
|
||||
|
||||
Reference in New Issue
Block a user