mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix: render parity for transparent looped videos (#478)
## Summary - preserve alpha for render-injected video frames by detecting alpha streams with ffprobe and extracting alpha video frames as PNG - keep `<video loop>` semantics through static parsing, compiler duration resolution, browser media discovery, and render frame lookup - fail embedded preview startup before opening a broken browser page when the Studio bundle is missing - align snapshot frame injection with looped media timing and VP9 alpha extraction ## Why The Studio preview and rendered MP4 could disagree for timed transparent looped videos. The Comfy funding composition exposed two separate parity bugs: render-injected frames needed alpha-preserving PNG extraction, and the compiler was clamping a looped `data-duration="4"` video down to the 3.125s source duration. After the first source cycle, render lookup treated the video as inactive, hid the native video, and produced the blank polygon/glow the user saw around the rounded `0:03` mark. `hyperframes lint` and `hyperframes validate` did not catch this because they check syntax/load/console/accessibility, not preview-vs-render visual parity. This PR adds regression coverage for the compiler loop-duration path and frame lookup path. ## Verification - `bun run --filter @hyperframes/core test -- src/compiler/timingCompiler.test.ts src/compiler/htmlCompiler.test.ts` - `bun test packages/producer/src/services/htmlCompiler.test.ts` - `bun run --filter @hyperframes/engine test -- videoFrameExtractor ffprobe` - `bun run --filter @hyperframes/core typecheck` - `bun run --filter @hyperframes/engine typecheck` - `bun run --filter @hyperframes/producer typecheck` - `bun run --filter @hyperframes/cli typecheck` - `bun run lint` - `bun run format:check ...` on touched files - Comfy project: `node packages/cli/dist/cli.js validate` -> no console errors, 44 text elements pass WCAG AA - Comfy project patched render from source: `/tmp/comfy-render-compare/fixed6-comfy.mp4`, 1920x1080, 30fps, 21.8s, 654 frames - 3.00s-3.97s render contact sheet: `/tmp/comfy-render-compare/fixed6-window-contact.png` - targeted fixed render capture at 3.733s: `/tmp/comfy-render-compare/probe-capture-fixed/captured/frame_000112.jpg` - agent-browser Studio proof screenshot at 3.7s: `/tmp/comfy-render-compare/agent-browser-studio-3_7-fixed.png` - agent-browser-driven recording of 3s seek pass: `/tmp/comfy-render-compare/agent-browser-wysiwyg-3s-fixed.webm` Note: `bun run --filter @hyperframes/cli dev -- validate` is blocked in source mode by the existing `contrast-audit.browser.js` default-export loader issue; packaged `node packages/cli/dist/cli.js validate` passes for this project.
This commit is contained in:
@@ -318,15 +318,31 @@ async function runEmbeddedMode(
|
|||||||
projectName?: string,
|
projectName?: string,
|
||||||
forceNew = false,
|
forceNew = false,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const { createStudioServer } = await import("../server/studioServer.js");
|
const { createStudioServer, resolveStudioBundle } = await import("../server/studioServer.js");
|
||||||
|
|
||||||
const pName = projectName ?? basename(dir);
|
const pName = projectName ?? basename(dir);
|
||||||
const { app } = createStudioServer({ projectDir: dir, projectName: pName });
|
const studioBundle = resolveStudioBundle();
|
||||||
|
|
||||||
clack.intro(c.bold("hyperframes preview"));
|
clack.intro(c.bold("hyperframes preview"));
|
||||||
const s = clack.spinner();
|
const s = clack.spinner();
|
||||||
s.start("Starting studio...");
|
s.start("Starting studio...");
|
||||||
|
|
||||||
|
if (!studioBundle.available) {
|
||||||
|
s.stop(c.error("Studio build missing"));
|
||||||
|
console.error();
|
||||||
|
console.error(` ${c.dim("Could not find")} ${c.accent("index.html")} ${c.dim("in:")}`);
|
||||||
|
for (const checkedPath of studioBundle.checkedPaths) {
|
||||||
|
console.error(` ${c.dim("-")} ${checkedPath}`);
|
||||||
|
}
|
||||||
|
console.error();
|
||||||
|
console.error(` ${c.dim("Rebuild the CLI package with")} ${c.accent("pnpm run build")}`);
|
||||||
|
console.error();
|
||||||
|
process.exitCode = 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { app } = createStudioServer({ projectDir: dir, projectName: pName });
|
||||||
|
|
||||||
let result: FindPortResult;
|
let result: FindPortResult;
|
||||||
try {
|
try {
|
||||||
result = await findPortAndServe(app.fetch, startPort, dir, forceNew);
|
result = await findPortAndServe(app.fetch, startPort, dir, forceNew);
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ const FFMPEG_EXTRACT_TIMEOUT_MS = 30_000;
|
|||||||
async function extractVideoFrameToBuffer(
|
async function extractVideoFrameToBuffer(
|
||||||
videoPath: string,
|
videoPath: string,
|
||||||
timeSeconds: number,
|
timeSeconds: number,
|
||||||
|
useVp9AlphaDecoder = false,
|
||||||
): Promise<Buffer | null> {
|
): Promise<Buffer | null> {
|
||||||
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
|
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
|
||||||
const outPath = join(tmp, "frame.png");
|
const outPath = join(tmp, "frame.png");
|
||||||
@@ -33,10 +34,11 @@ async function extractVideoFrameToBuffer(
|
|||||||
(resolvePromise) => {
|
(resolvePromise) => {
|
||||||
// `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
|
// `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
|
||||||
// (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
|
// (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
|
||||||
const ff = spawn("ffmpeg", [
|
const args = ["-hide_banner", "-loglevel", "error"];
|
||||||
"-hide_banner",
|
if (useVp9AlphaDecoder) {
|
||||||
"-loglevel",
|
args.push("-c:v", "libvpx-vp9");
|
||||||
"error",
|
}
|
||||||
|
args.push(
|
||||||
"-ss",
|
"-ss",
|
||||||
String(Math.max(0, timeSeconds)),
|
String(Math.max(0, timeSeconds)),
|
||||||
"-i",
|
"-i",
|
||||||
@@ -47,7 +49,8 @@ async function extractVideoFrameToBuffer(
|
|||||||
"2",
|
"2",
|
||||||
"-y",
|
"-y",
|
||||||
outPath,
|
outPath,
|
||||||
]);
|
);
|
||||||
|
const ff = spawn("ffmpeg", args);
|
||||||
let stderr = "";
|
let stderr = "";
|
||||||
let timedOut = false;
|
let timedOut = false;
|
||||||
const timer = setTimeout(() => {
|
const timer = setTimeout(() => {
|
||||||
@@ -252,19 +255,36 @@ async function captureSnapshots(
|
|||||||
updates: Array<{ videoId: string; dataUri: string }>,
|
updates: Array<{ videoId: string; dataUri: string }>,
|
||||||
) => Promise<void>;
|
) => Promise<void>;
|
||||||
type SyncVisibilityFn = (page: unknown, activeVideoIds: string[]) => Promise<void>;
|
type SyncVisibilityFn = (page: unknown, activeVideoIds: string[]) => Promise<void>;
|
||||||
|
type ExtractMediaMetadataFn = (
|
||||||
|
filePath: string,
|
||||||
|
) => Promise<{ videoCodec: string; hasAlpha: boolean }>;
|
||||||
let injectVideoFramesBatch: InjectFn | null = null;
|
let injectVideoFramesBatch: InjectFn | null = null;
|
||||||
let syncVideoFrameVisibility: SyncVisibilityFn | null = null;
|
let syncVideoFrameVisibility: SyncVisibilityFn | null = null;
|
||||||
|
let extractMediaMetadata: ExtractMediaMetadataFn | null = null;
|
||||||
try {
|
try {
|
||||||
const engine = (await import("@hyperframes/engine")) as {
|
const engine = (await import("@hyperframes/engine")) as {
|
||||||
injectVideoFramesBatch: InjectFn;
|
injectVideoFramesBatch: InjectFn;
|
||||||
syncVideoFrameVisibility: SyncVisibilityFn;
|
syncVideoFrameVisibility: SyncVisibilityFn;
|
||||||
|
extractMediaMetadata: ExtractMediaMetadataFn;
|
||||||
};
|
};
|
||||||
injectVideoFramesBatch = engine.injectVideoFramesBatch;
|
injectVideoFramesBatch = engine.injectVideoFramesBatch;
|
||||||
syncVideoFrameVisibility = engine.syncVideoFrameVisibility;
|
syncVideoFrameVisibility = engine.syncVideoFrameVisibility;
|
||||||
|
extractMediaMetadata = engine.extractMediaMetadata;
|
||||||
} catch {
|
} catch {
|
||||||
// Engine unavailable in this install — snapshot will still run, and
|
// Engine unavailable in this install — snapshot will still run, and
|
||||||
// compositions without <video data-start> get exactly the old behaviour.
|
// compositions without <video data-start> get exactly the old behaviour.
|
||||||
}
|
}
|
||||||
|
const alphaDecoderCache = new Map<string, Promise<boolean>>();
|
||||||
|
const shouldUseVp9AlphaDecoder = (filePath: string): Promise<boolean> => {
|
||||||
|
if (!extractMediaMetadata) return Promise.resolve(false);
|
||||||
|
const cached = alphaDecoderCache.get(filePath);
|
||||||
|
if (cached) return cached;
|
||||||
|
const pending = extractMediaMetadata(filePath)
|
||||||
|
.then((meta) => meta.hasAlpha && meta.videoCodec === "vp9")
|
||||||
|
.catch(() => false);
|
||||||
|
alphaDecoderCache.set(filePath, pending);
|
||||||
|
return pending;
|
||||||
|
};
|
||||||
|
|
||||||
// Seek and capture each frame
|
// Seek and capture each frame
|
||||||
for (let i = 0; i < positions.length; i++) {
|
for (let i = 0; i < positions.length; i++) {
|
||||||
@@ -324,7 +344,10 @@ async function captureSnapshots(
|
|||||||
: srcDur > 0
|
: srcDur > 0
|
||||||
? Math.max(0, (srcDur - mediaStart) / playbackRate)
|
? Math.max(0, (srcDur - mediaStart) / playbackRate)
|
||||||
: Number.POSITIVE_INFINITY;
|
: Number.POSITIVE_INFINITY;
|
||||||
const relTime = (t - start) * playbackRate + mediaStart;
|
let relTime = (t - start) * playbackRate + mediaStart;
|
||||||
|
if (v.loop && srcDur > mediaStart && relTime >= srcDur) {
|
||||||
|
relTime = mediaStart + ((relTime - mediaStart) % (srcDur - mediaStart));
|
||||||
|
}
|
||||||
const activeNow = t >= start && t < start + duration && relTime >= 0 && !!v.id;
|
const activeNow = t >= start && t < start + duration && relTime >= 0 && !!v.id;
|
||||||
return {
|
return {
|
||||||
id: v.id,
|
id: v.id,
|
||||||
@@ -356,7 +379,11 @@ async function captureSnapshots(
|
|||||||
/* unresolvable src (e.g. blob:, data:) — skip */
|
/* unresolvable src (e.g. blob:, data:) — skip */
|
||||||
}
|
}
|
||||||
if (!filePath) continue;
|
if (!filePath) continue;
|
||||||
const png = await extractVideoFrameToBuffer(filePath, Math.max(0, v.relTime));
|
const png = await extractVideoFrameToBuffer(
|
||||||
|
filePath,
|
||||||
|
Math.max(0, v.relTime),
|
||||||
|
await shouldUseVp9AlphaDecoder(filePath),
|
||||||
|
);
|
||||||
if (!png) continue;
|
if (!png) continue;
|
||||||
updates.push({
|
updates.push({
|
||||||
videoId: v.id,
|
videoId: v.id,
|
||||||
|
|||||||
@@ -23,11 +23,38 @@ import {
|
|||||||
// ── Path resolution ─────────────────────────────────────────────────────────
|
// ── Path resolution ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
function resolveDistDir(): string {
|
function resolveDistDir(): string {
|
||||||
|
return resolveStudioBundle().dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StudioBundleResolution {
|
||||||
|
dir: string;
|
||||||
|
indexPath: string;
|
||||||
|
available: boolean;
|
||||||
|
checkedPaths: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveStudioBundle(): StudioBundleResolution {
|
||||||
const builtPath = resolve(__dirname, "studio");
|
const builtPath = resolve(__dirname, "studio");
|
||||||
if (existsSync(resolve(builtPath, "index.html"))) return builtPath;
|
const builtIndex = resolve(builtPath, "index.html");
|
||||||
|
if (existsSync(builtIndex)) {
|
||||||
|
return { dir: builtPath, indexPath: builtIndex, available: true, checkedPaths: [builtIndex] };
|
||||||
|
}
|
||||||
const devPath = resolve(__dirname, "..", "..", "..", "studio", "dist");
|
const devPath = resolve(__dirname, "..", "..", "..", "studio", "dist");
|
||||||
if (existsSync(resolve(devPath, "index.html"))) return devPath;
|
const devIndex = resolve(devPath, "index.html");
|
||||||
return builtPath;
|
if (existsSync(devIndex)) {
|
||||||
|
return {
|
||||||
|
dir: devPath,
|
||||||
|
indexPath: devIndex,
|
||||||
|
available: true,
|
||||||
|
checkedPaths: [builtIndex, devIndex],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
dir: builtPath,
|
||||||
|
indexPath: builtIndex,
|
||||||
|
available: false,
|
||||||
|
checkedPaths: [builtIndex, devIndex],
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function resolveRuntimePath(): string {
|
function resolveRuntimePath(): string {
|
||||||
@@ -348,7 +375,60 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
|||||||
app.get("*", (c) => {
|
app.get("*", (c) => {
|
||||||
const indexPath = resolve(studioDir, "index.html");
|
const indexPath = resolve(studioDir, "index.html");
|
||||||
if (!existsSync(indexPath)) {
|
if (!existsSync(indexPath)) {
|
||||||
return c.text("Studio not found. Rebuild with: pnpm run build", 500);
|
return c.html(
|
||||||
|
`<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>HyperFrames Studio unavailable</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
min-height: 100vh;
|
||||||
|
display: grid;
|
||||||
|
place-items: center;
|
||||||
|
background: #0d0f14;
|
||||||
|
color: #eef2f7;
|
||||||
|
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||||
|
}
|
||||||
|
main {
|
||||||
|
width: min(560px, calc(100vw - 48px));
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.14);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 28px;
|
||||||
|
background: #151923;
|
||||||
|
}
|
||||||
|
h1 {
|
||||||
|
margin: 0 0 12px;
|
||||||
|
font-size: 22px;
|
||||||
|
line-height: 1.2;
|
||||||
|
}
|
||||||
|
p {
|
||||||
|
margin: 0 0 18px;
|
||||||
|
color: #aab3c2;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
code {
|
||||||
|
display: block;
|
||||||
|
padding: 12px 14px;
|
||||||
|
border-radius: 6px;
|
||||||
|
background: #090b10;
|
||||||
|
color: #8ff0c2;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<h1>Studio bundle missing</h1>
|
||||||
|
<p>The preview server started, but this CLI build does not contain the Studio assets.</p>
|
||||||
|
<code>pnpm run build</code>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>`,
|
||||||
|
500,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
return c.html(readFileSync(indexPath, "utf-8"));
|
return c.html(readFileSync(indexPath, "utf-8"));
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { compileHtml } from "./htmlCompiler.js";
|
||||||
|
|
||||||
|
describe("compileHtml", () => {
|
||||||
|
it("preserves explicit looped media durations that exceed source duration", async () => {
|
||||||
|
const html =
|
||||||
|
'<video id="hero" src="hero.webm" data-start="0" data-duration="4" data-end="4" loop>';
|
||||||
|
|
||||||
|
const compiled = await compileHtml(html, "/project", async () => 3.125);
|
||||||
|
|
||||||
|
expect(compiled).toContain('data-duration="4"');
|
||||||
|
expect(compiled).toContain('data-end="4"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still clamps non-looping media durations to source duration", async () => {
|
||||||
|
const html = '<video id="hero" src="hero.webm" data-start="0" data-duration="4" data-end="4">';
|
||||||
|
|
||||||
|
const compiled = await compileHtml(html, "/project", async () => 3.125);
|
||||||
|
|
||||||
|
expect(compiled).toContain('data-duration="3.125"');
|
||||||
|
expect(compiled).toContain('data-end="3.125"');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -70,6 +70,7 @@ export async function compileHtml(
|
|||||||
|
|
||||||
for (const el of preResolved) {
|
for (const el of preResolved) {
|
||||||
if (!el.src) continue;
|
if (!el.src) continue;
|
||||||
|
if (el.loop) continue;
|
||||||
const src = resolveMediaSrc(el.src, projectDir);
|
const src = resolveMediaSrc(el.src, projectDir);
|
||||||
const fileDuration = await probeMediaDuration(src);
|
const fileDuration = await probeMediaDuration(src);
|
||||||
if (fileDuration <= 0) continue;
|
if (fileDuration <= 0) continue;
|
||||||
|
|||||||
@@ -115,11 +115,26 @@ describe("extractResolvedMedia", () => {
|
|||||||
expect(resolved[0].tagName).toBe("video");
|
expect(resolved[0].tagName).toBe("video");
|
||||||
expect(resolved[0].duration).toBe(5);
|
expect(resolved[0].duration).toBe(5);
|
||||||
expect(resolved[0].start).toBe(1);
|
expect(resolved[0].start).toBe(1);
|
||||||
|
expect(resolved[0].loop).toBe(false);
|
||||||
expect(resolved[1].id).toBe("a1");
|
expect(resolved[1].id).toBe("a1");
|
||||||
expect(resolved[1].tagName).toBe("audio");
|
expect(resolved[1].tagName).toBe("audio");
|
||||||
expect(resolved[1].duration).toBe(10);
|
expect(resolved[1].duration).toBe(10);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("marks looped media so render compilation can preserve display duration", () => {
|
||||||
|
const html = '<video id="v1" src="vid.webm" data-start="0" data-duration="4" loop>';
|
||||||
|
|
||||||
|
const resolved = extractResolvedMedia(html);
|
||||||
|
|
||||||
|
expect(resolved).toHaveLength(1);
|
||||||
|
expect(resolved[0]).toMatchObject({
|
||||||
|
id: "v1",
|
||||||
|
tagName: "video",
|
||||||
|
duration: 4,
|
||||||
|
loop: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("skips elements with invalid durations", () => {
|
it("skips elements with invalid durations", () => {
|
||||||
const html = '<video id="v1" src="a.mp4" data-start="0" data-duration="NaN">';
|
const html = '<video id="v1" src="a.mp4" data-start="0" data-duration="NaN">';
|
||||||
const resolved = extractResolvedMedia(html);
|
const resolved = extractResolvedMedia(html);
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ export interface ResolvedMediaElement {
|
|||||||
start: number;
|
start: number;
|
||||||
duration: number;
|
duration: number;
|
||||||
mediaStart: number;
|
mediaStart: number;
|
||||||
|
loop: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CompilationResult {
|
export interface CompilationResult {
|
||||||
@@ -55,7 +56,7 @@ function getAttr(tag: string, attr: string): string | null {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function hasAttr(tag: string, attr: string): boolean {
|
function hasAttr(tag: string, attr: string): boolean {
|
||||||
return new RegExp(`${attr}=["']`).test(tag);
|
return new RegExp(`\\s${attr}(?:\\s|=|>|/)`).test(tag);
|
||||||
}
|
}
|
||||||
|
|
||||||
function injectAttr(tag: string, attr: string, value: string): string {
|
function injectAttr(tag: string, attr: string, value: string): string {
|
||||||
@@ -229,6 +230,7 @@ export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
|
|||||||
start: startStr !== null ? parseFloat(startStr) : 0,
|
start: startStr !== null ? parseFloat(startStr) : 0,
|
||||||
duration,
|
duration,
|
||||||
mediaStart: mediaStartStr ? parseFloat(mediaStartStr) : 0,
|
mediaStart: mediaStartStr ? parseFloat(mediaStartStr) : 0,
|
||||||
|
loop: hasAttr(tag, "loop"),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ import {
|
|||||||
parseVideoElements,
|
parseVideoElements,
|
||||||
parseImageElements,
|
parseImageElements,
|
||||||
extractAllVideoFrames,
|
extractAllVideoFrames,
|
||||||
|
createFrameLookupTable,
|
||||||
type VideoElement,
|
type VideoElement,
|
||||||
|
type ExtractedFrames,
|
||||||
} from "./videoFrameExtractor.js";
|
} from "./videoFrameExtractor.js";
|
||||||
import { extractVideoMetadata } from "../utils/ffprobe.js";
|
import { extractVideoMetadata } from "../utils/ffprobe.js";
|
||||||
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
||||||
@@ -32,6 +34,7 @@ describe("parseVideoElements", () => {
|
|||||||
start: 0,
|
start: 0,
|
||||||
end: Infinity,
|
end: Infinity,
|
||||||
mediaStart: 0,
|
mediaStart: 0,
|
||||||
|
loop: false,
|
||||||
hasAudio: false,
|
hasAudio: false,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -48,9 +51,97 @@ describe("parseVideoElements", () => {
|
|||||||
start: 2,
|
start: 2,
|
||||||
end: 7,
|
end: 7,
|
||||||
mediaStart: 1.5,
|
mediaStart: 1.5,
|
||||||
|
loop: false,
|
||||||
hasAudio: true,
|
hasAudio: true,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("preserves looped timed video semantics for render frame lookup", () => {
|
||||||
|
const videos = parseVideoElements(
|
||||||
|
'<video id="hero" src="clip.webm" data-start="2" data-duration="5" loop></video>',
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(videos[0]).toMatchObject({
|
||||||
|
id: "hero",
|
||||||
|
start: 2,
|
||||||
|
end: 7,
|
||||||
|
loop: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("FrameLookupTable", () => {
|
||||||
|
function fakeExtracted(totalFrames: number, fps: number): ExtractedFrames {
|
||||||
|
const framePaths = new Map<number, string>();
|
||||||
|
for (let i = 0; i < totalFrames; i += 1) {
|
||||||
|
framePaths.set(i, `frame-${i}.jpg`);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
videoId: "hero",
|
||||||
|
srcPath: "clip.webm",
|
||||||
|
outputDir: "/tmp/frames",
|
||||||
|
framePattern: "frame-%05d.jpg",
|
||||||
|
fps,
|
||||||
|
totalFrames,
|
||||||
|
metadata: {
|
||||||
|
durationSeconds: totalFrames / fps,
|
||||||
|
width: 320,
|
||||||
|
height: 180,
|
||||||
|
fps,
|
||||||
|
hasAudio: false,
|
||||||
|
videoCodec: "vp9",
|
||||||
|
colorSpace: {
|
||||||
|
colorTransfer: "bt709",
|
||||||
|
colorPrimaries: "bt709",
|
||||||
|
colorSpace: "bt709",
|
||||||
|
},
|
||||||
|
isVFR: false,
|
||||||
|
hasAlpha: false,
|
||||||
|
},
|
||||||
|
framePaths,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it("wraps active frame payloads for looped clips whose display window exceeds source frames", () => {
|
||||||
|
const table = createFrameLookupTable(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
id: "hero",
|
||||||
|
src: "clip.webm",
|
||||||
|
start: 0,
|
||||||
|
end: 5,
|
||||||
|
mediaStart: 0,
|
||||||
|
loop: true,
|
||||||
|
hasAudio: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[fakeExtracted(30, 30)],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(table.getActiveFramePayloads(0.5).get("hero")?.frameIndex).toBe(15);
|
||||||
|
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(15);
|
||||||
|
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not hold stale frames for non-looping clips after extracted frames end", () => {
|
||||||
|
const table = createFrameLookupTable(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
id: "hero",
|
||||||
|
src: "clip.webm",
|
||||||
|
start: 0,
|
||||||
|
end: 5,
|
||||||
|
mediaStart: 0,
|
||||||
|
loop: false,
|
||||||
|
hasAudio: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[fakeExtracted(30, 30)],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(table.getActiveFramePayloads(0.5).has("hero")).toBe(true);
|
||||||
|
expect(table.getActiveFramePayloads(1.5).has("hero")).toBe(false);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("parseImageElements", () => {
|
describe("parseImageElements", () => {
|
||||||
@@ -175,6 +266,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
|||||||
start: 0,
|
start: 0,
|
||||||
end: 4,
|
end: 4,
|
||||||
mediaStart: 3,
|
mediaStart: 3,
|
||||||
|
loop: false,
|
||||||
hasAudio: false,
|
hasAudio: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -229,6 +321,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
|||||||
start: 0,
|
start: 0,
|
||||||
end: 2,
|
end: 2,
|
||||||
mediaStart: 0,
|
mediaStart: 0,
|
||||||
|
loop: false,
|
||||||
hasAudio: false,
|
hasAudio: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -298,6 +391,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
|||||||
start: 0,
|
start: 0,
|
||||||
end: 1,
|
end: 1,
|
||||||
mediaStart: 0,
|
mediaStart: 0,
|
||||||
|
loop: false,
|
||||||
hasAudio: false,
|
hasAudio: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -388,8 +482,16 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
|||||||
mkdirSync(outputDir, { recursive: true });
|
mkdirSync(outputDir, { recursive: true });
|
||||||
|
|
||||||
const videos: VideoElement[] = [
|
const videos: VideoElement[] = [
|
||||||
{ id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, hasAudio: false },
|
{ id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, loop: false, hasAudio: false },
|
||||||
{ id: "hdr", src: HDR_SHORT, start: 2, end: 4, mediaStart: 0, hasAudio: false },
|
{
|
||||||
|
id: "hdr",
|
||||||
|
src: HDR_SHORT,
|
||||||
|
start: 2,
|
||||||
|
end: 4,
|
||||||
|
mediaStart: 0,
|
||||||
|
loop: false,
|
||||||
|
hasAudio: false,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const result = await extractAllVideoFrames(videos, FIXTURE_DIR, {
|
const result = await extractAllVideoFrames(videos, FIXTURE_DIR, {
|
||||||
@@ -424,6 +526,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
|||||||
start: 0,
|
start: 0,
|
||||||
end: 10,
|
end: 10,
|
||||||
mediaStart: 0,
|
mediaStart: 0,
|
||||||
|
loop: false,
|
||||||
hasAudio: false,
|
hasAudio: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ export interface VideoElement {
|
|||||||
start: number;
|
start: number;
|
||||||
end: number;
|
end: number;
|
||||||
mediaStart: number;
|
mediaStart: number;
|
||||||
|
loop: boolean;
|
||||||
hasAudio: boolean;
|
hasAudio: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -141,6 +142,7 @@ export function parseVideoElements(html: string): VideoElement[] {
|
|||||||
start,
|
start,
|
||||||
end,
|
end,
|
||||||
mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
|
mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
|
||||||
|
loop: el.hasAttribute("loop"),
|
||||||
hasAudio: hasAudioAttr === "true",
|
hasAudio: hasAudioAttr === "true",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -207,12 +209,13 @@ export async function extractVideoFramesRange(
|
|||||||
outputDirOverride?: string,
|
outputDirOverride?: string,
|
||||||
): Promise<ExtractedFrames> {
|
): Promise<ExtractedFrames> {
|
||||||
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
|
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
|
||||||
const { fps, outputDir, quality = 95, format = "jpg" } = options;
|
const { fps, outputDir, quality = 95 } = options;
|
||||||
|
|
||||||
const videoOutputDir = outputDirOverride ?? join(outputDir, videoId);
|
const videoOutputDir = outputDirOverride ?? join(outputDir, videoId);
|
||||||
if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true });
|
if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true });
|
||||||
|
|
||||||
const metadata = await extractMediaMetadata(videoPath);
|
const metadata = await extractMediaMetadata(videoPath);
|
||||||
|
const format = resolveFrameFormat(metadata, options.format);
|
||||||
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`;
|
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`;
|
||||||
const outputPattern = join(videoOutputDir, framePattern);
|
const outputPattern = join(videoOutputDir, framePattern);
|
||||||
|
|
||||||
@@ -227,6 +230,9 @@ export async function extractVideoFramesRange(
|
|||||||
if (isHdr && isMacOS) {
|
if (isHdr && isMacOS) {
|
||||||
args.push("-hwaccel", "videotoolbox");
|
args.push("-hwaccel", "videotoolbox");
|
||||||
}
|
}
|
||||||
|
if (metadata.hasAlpha && metadata.videoCodec === "vp9") {
|
||||||
|
args.push("-c:v", "libvpx-vp9");
|
||||||
|
}
|
||||||
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
|
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
|
||||||
|
|
||||||
const vfFilters: string[] = [];
|
const vfFilters: string[] = [];
|
||||||
@@ -392,6 +398,11 @@ function resolveSegmentDuration(
|
|||||||
return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
|
return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat {
|
||||||
|
if (requested) return requested;
|
||||||
|
return metadata.hasAlpha ? "png" : "jpg";
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Re-encode a VFR (variable frame rate) video segment to CFR so the downstream
|
* Re-encode a VFR (variable frame rate) video segment to CFR so the downstream
|
||||||
* fps filter can extract frames reliably. Screen recordings, phone videos, and
|
* fps filter can extract frames reliably. Screen recordings, phone videos, and
|
||||||
@@ -682,7 +693,6 @@ export async function extractAllVideoFrames(
|
|||||||
|
|
||||||
const phase3Start = Date.now();
|
const phase3Start = Date.now();
|
||||||
const cacheRootDir = config?.extractCacheDir;
|
const cacheRootDir = config?.extractCacheDir;
|
||||||
const cacheFormat: CacheFrameFormat = options.format ?? "jpg";
|
|
||||||
|
|
||||||
async function tryCachedExtract(
|
async function tryCachedExtract(
|
||||||
video: VideoElement,
|
video: VideoElement,
|
||||||
@@ -694,6 +704,7 @@ export async function extractAllVideoFrames(
|
|||||||
const keyInput = cacheKeyInputs[i];
|
const keyInput = cacheKeyInputs[i];
|
||||||
const probedMeta = videoMetadata[i];
|
const probedMeta = videoMetadata[i];
|
||||||
if (!keyInput || !probedMeta) return null;
|
if (!keyInput || !probedMeta) return null;
|
||||||
|
const cacheFormat = resolveFrameFormat(probedMeta, options.format);
|
||||||
|
|
||||||
const keyDuration = resolveSegmentDuration(
|
const keyDuration = resolveSegmentDuration(
|
||||||
keyInput.end - keyInput.start,
|
keyInput.end - keyInput.start,
|
||||||
@@ -729,7 +740,7 @@ export async function extractAllVideoFrames(
|
|||||||
video.id,
|
video.id,
|
||||||
video.mediaStart,
|
video.mediaStart,
|
||||||
videoDuration,
|
videoDuration,
|
||||||
options,
|
{ ...options, format: cacheFormat },
|
||||||
signal,
|
signal,
|
||||||
config,
|
config,
|
||||||
lookup.entry.dir,
|
lookup.entry.dir,
|
||||||
@@ -764,7 +775,7 @@ export async function extractAllVideoFrames(
|
|||||||
video.id,
|
video.id,
|
||||||
video.mediaStart,
|
video.mediaStart,
|
||||||
videoDuration,
|
videoDuration,
|
||||||
options,
|
{ ...options, format: resolveFrameFormat(probedMeta, options.format) },
|
||||||
signal,
|
signal,
|
||||||
config,
|
config,
|
||||||
);
|
);
|
||||||
@@ -807,10 +818,19 @@ export function getFrameAtTime(
|
|||||||
extracted: ExtractedFrames,
|
extracted: ExtractedFrames,
|
||||||
globalTime: number,
|
globalTime: number,
|
||||||
videoStart: number,
|
videoStart: number,
|
||||||
|
loop = false,
|
||||||
|
mediaStart = 0,
|
||||||
): string | null {
|
): string | null {
|
||||||
const localTime = globalTime - videoStart;
|
let localTime = globalTime - videoStart;
|
||||||
if (localTime < 0) return null;
|
if (localTime < 0) return null;
|
||||||
|
const loopDuration = Math.max(0, extracted.metadata.durationSeconds - mediaStart);
|
||||||
|
if (loop && loopDuration > 0 && localTime >= loopDuration) {
|
||||||
|
localTime %= loopDuration;
|
||||||
|
}
|
||||||
const frameIndex = Math.floor(localTime * extracted.fps);
|
const frameIndex = Math.floor(localTime * extracted.fps);
|
||||||
|
if (loop && frameIndex >= extracted.totalFrames && extracted.totalFrames > 0) {
|
||||||
|
return extracted.framePaths.get(extracted.totalFrames - 1) || null;
|
||||||
|
}
|
||||||
if (frameIndex < 0 || frameIndex >= extracted.totalFrames) return null;
|
if (frameIndex < 0 || frameIndex >= extracted.totalFrames) return null;
|
||||||
return extracted.framePaths.get(frameIndex) || null;
|
return extracted.framePaths.get(frameIndex) || null;
|
||||||
}
|
}
|
||||||
@@ -823,6 +843,7 @@ export class FrameLookupTable {
|
|||||||
start: number;
|
start: number;
|
||||||
end: number;
|
end: number;
|
||||||
mediaStart: number;
|
mediaStart: number;
|
||||||
|
loop: boolean;
|
||||||
}
|
}
|
||||||
> = new Map();
|
> = new Map();
|
||||||
private orderedVideos: Array<{
|
private orderedVideos: Array<{
|
||||||
@@ -831,13 +852,20 @@ export class FrameLookupTable {
|
|||||||
start: number;
|
start: number;
|
||||||
end: number;
|
end: number;
|
||||||
mediaStart: number;
|
mediaStart: number;
|
||||||
|
loop: boolean;
|
||||||
}> = [];
|
}> = [];
|
||||||
private activeVideoIds: Set<string> = new Set();
|
private activeVideoIds: Set<string> = new Set();
|
||||||
private startCursor = 0;
|
private startCursor = 0;
|
||||||
private lastTime: number | null = null;
|
private lastTime: number | null = null;
|
||||||
|
|
||||||
addVideo(extracted: ExtractedFrames, start: number, end: number, mediaStart: number): void {
|
addVideo(
|
||||||
this.videos.set(extracted.videoId, { extracted, start, end, mediaStart });
|
extracted: ExtractedFrames,
|
||||||
|
start: number,
|
||||||
|
end: number,
|
||||||
|
mediaStart: number,
|
||||||
|
loop = false,
|
||||||
|
): void {
|
||||||
|
this.videos.set(extracted.videoId, { extracted, start, end, mediaStart, loop });
|
||||||
this.orderedVideos = Array.from(this.videos.entries())
|
this.orderedVideos = Array.from(this.videos.entries())
|
||||||
.map(([videoId, video]) => ({ videoId, ...video }))
|
.map(([videoId, video]) => ({ videoId, ...video }))
|
||||||
.sort((a, b) => a.start - b.start);
|
.sort((a, b) => a.start - b.start);
|
||||||
@@ -848,7 +876,7 @@ export class FrameLookupTable {
|
|||||||
const video = this.videos.get(videoId);
|
const video = this.videos.get(videoId);
|
||||||
if (!video) return null;
|
if (!video) return null;
|
||||||
if (globalTime < video.start || globalTime >= video.end) return null;
|
if (globalTime < video.start || globalTime >= video.end) return null;
|
||||||
return getFrameAtTime(video.extracted, globalTime, video.start);
|
return getFrameAtTime(video.extracted, globalTime, video.start, video.loop, video.mediaStart);
|
||||||
}
|
}
|
||||||
|
|
||||||
private resetActiveState(): void {
|
private resetActiveState(): void {
|
||||||
@@ -904,8 +932,19 @@ export class FrameLookupTable {
|
|||||||
for (const videoId of this.activeVideoIds) {
|
for (const videoId of this.activeVideoIds) {
|
||||||
const video = this.videos.get(videoId);
|
const video = this.videos.get(videoId);
|
||||||
if (!video) continue;
|
if (!video) continue;
|
||||||
const localTime = globalTime - video.start;
|
let localTime = globalTime - video.start;
|
||||||
|
const loopDuration = Math.max(0, video.extracted.metadata.durationSeconds - video.mediaStart);
|
||||||
|
if (video.loop && loopDuration > 0 && localTime >= loopDuration) {
|
||||||
|
localTime %= loopDuration;
|
||||||
|
}
|
||||||
const frameIndex = Math.floor(localTime * video.extracted.fps);
|
const frameIndex = Math.floor(localTime * video.extracted.fps);
|
||||||
|
if (video.loop && frameIndex >= video.extracted.totalFrames) {
|
||||||
|
const framePath = video.extracted.framePaths.get(video.extracted.totalFrames - 1);
|
||||||
|
if (framePath) {
|
||||||
|
frames.set(videoId, { framePath, frameIndex: video.extracted.totalFrames - 1 });
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (frameIndex < 0 || frameIndex >= video.extracted.totalFrames) continue;
|
if (frameIndex < 0 || frameIndex >= video.extracted.totalFrames) continue;
|
||||||
const framePath = video.extracted.framePaths.get(frameIndex);
|
const framePath = video.extracted.framePaths.get(frameIndex);
|
||||||
if (!framePath) continue;
|
if (!framePath) continue;
|
||||||
@@ -949,7 +988,7 @@ export function createFrameLookupTable(
|
|||||||
|
|
||||||
for (const video of videos) {
|
for (const video of videos) {
|
||||||
const ext = extractedMap.get(video.id);
|
const ext = extractedMap.get(video.id);
|
||||||
if (ext) table.addVideo(ext, video.start, video.end, video.mediaStart);
|
if (ext) table.addVideo(ext, video.start, video.end, video.mediaStart, video.loop);
|
||||||
}
|
}
|
||||||
|
|
||||||
return table;
|
return table;
|
||||||
|
|||||||
@@ -188,10 +188,43 @@ describe("ffprobe missing-binary fallback", () => {
|
|||||||
expect(meta.fps).toBe(0);
|
expect(meta.fps).toBe(0);
|
||||||
expect(meta.hasAudio).toBe(false);
|
expect(meta.hasAudio).toBe(false);
|
||||||
expect(meta.isVFR).toBe(false);
|
expect(meta.isVFR).toBe(false);
|
||||||
|
expect(meta.hasAlpha).toBe(false);
|
||||||
expect(meta.colorSpace?.colorTransfer).toBe("smpte2084");
|
expect(meta.colorSpace?.colorTransfer).toBe("smpte2084");
|
||||||
expect(meta.colorSpace?.colorPrimaries).toBe("bt2020");
|
expect(meta.colorSpace?.colorPrimaries).toBe("bt2020");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("extractMediaMetadata detects VP9 alpha_mode streams", async () => {
|
||||||
|
const { spawn } = createSpawnSpy([
|
||||||
|
{
|
||||||
|
kind: "exit",
|
||||||
|
code: 0,
|
||||||
|
stdout: JSON.stringify({
|
||||||
|
streams: [
|
||||||
|
{
|
||||||
|
codec_type: "video",
|
||||||
|
codec_name: "vp9",
|
||||||
|
width: 320,
|
||||||
|
height: 180,
|
||||||
|
r_frame_rate: "30/1",
|
||||||
|
avg_frame_rate: "30/1",
|
||||||
|
pix_fmt: "yuv420p",
|
||||||
|
tags: { alpha_mode: "1" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
format: { duration: "1.5" },
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock("child_process", () => ({ spawn }));
|
||||||
|
|
||||||
|
const { extractMediaMetadata: extractMediaMetadataMocked } = await import("./ffprobe.js");
|
||||||
|
const meta = await extractMediaMetadataMocked("/tmp/alpha.webm");
|
||||||
|
|
||||||
|
expect(meta.videoCodec).toBe("vp9");
|
||||||
|
expect(meta.hasAlpha).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("extractMediaMetadata rethrows ffprobe-missing error for non-image files without fallback", async () => {
|
it("extractMediaMetadata rethrows ffprobe-missing error for non-image files without fallback", async () => {
|
||||||
const { spawn } = createSpawnSpy([{ kind: "missing" }]);
|
const { spawn } = createSpawnSpy([{ kind: "missing" }]);
|
||||||
vi.resetModules();
|
vi.resetModules();
|
||||||
|
|||||||
@@ -62,6 +62,8 @@ export interface VideoMetadata {
|
|||||||
hasAudio: boolean;
|
hasAudio: boolean;
|
||||||
/** True when r_frame_rate and avg_frame_rate differ significantly (>10%), indicating variable frame rate. */
|
/** True when r_frame_rate and avg_frame_rate differ significantly (>10%), indicating variable frame rate. */
|
||||||
isVFR: boolean;
|
isVFR: boolean;
|
||||||
|
/** True when the stream carries an alpha channel. */
|
||||||
|
hasAlpha: boolean;
|
||||||
/** Color space info from the video stream. Null if ffprobe didn't report it. */
|
/** Color space info from the video stream. Null if ffprobe didn't report it. */
|
||||||
colorSpace: VideoColorSpace | null;
|
colorSpace: VideoColorSpace | null;
|
||||||
}
|
}
|
||||||
@@ -79,6 +81,7 @@ interface FFProbeStream {
|
|||||||
codec_name?: string;
|
codec_name?: string;
|
||||||
width?: number;
|
width?: number;
|
||||||
height?: number;
|
height?: number;
|
||||||
|
pix_fmt?: string;
|
||||||
r_frame_rate?: string;
|
r_frame_rate?: string;
|
||||||
avg_frame_rate?: string;
|
avg_frame_rate?: string;
|
||||||
sample_rate?: string;
|
sample_rate?: string;
|
||||||
@@ -86,6 +89,7 @@ interface FFProbeStream {
|
|||||||
color_transfer?: string;
|
color_transfer?: string;
|
||||||
color_primaries?: string;
|
color_primaries?: string;
|
||||||
color_space?: string;
|
color_space?: string;
|
||||||
|
tags?: Record<string, string>;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface FFProbeFormat {
|
interface FFProbeFormat {
|
||||||
@@ -251,6 +255,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
|
|||||||
videoCodec: "png",
|
videoCodec: "png",
|
||||||
hasAudio: false,
|
hasAudio: false,
|
||||||
isVFR: false,
|
isVFR: false,
|
||||||
|
hasAlpha: false,
|
||||||
colorSpace: stillImageMeta.colorSpace,
|
colorSpace: stillImageMeta.colorSpace,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -271,6 +276,10 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
|
|||||||
? { colorTransfer, colorPrimaries, colorSpace: colorSpaceVal }
|
? { colorTransfer, colorPrimaries, colorSpace: colorSpaceVal }
|
||||||
: null;
|
: null;
|
||||||
const colorSpace = ffprobeColorSpace ?? stillImageMeta?.colorSpace ?? null;
|
const colorSpace = ffprobeColorSpace ?? stillImageMeta?.colorSpace ?? null;
|
||||||
|
const pixelFormat = videoStream.pix_fmt || "";
|
||||||
|
const alphaMode = videoStream.tags?.alpha_mode || "";
|
||||||
|
const hasAlpha =
|
||||||
|
/(^|[^a-z])yuva|rgba|argb|bgra|gbrap|gray[a-z0-9]*a/i.test(pixelFormat) || alphaMode === "1";
|
||||||
|
|
||||||
return {
|
return {
|
||||||
durationSeconds: output?.format.duration ? parseFloat(output.format.duration) : 0,
|
durationSeconds: output?.format.duration ? parseFloat(output.format.duration) : 0,
|
||||||
@@ -280,6 +289,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
|
|||||||
videoCodec: videoStream.codec_name || "unknown",
|
videoCodec: videoStream.codec_name || "unknown",
|
||||||
hasAudio: output?.streams.some((s) => s.codec_type === "audio") ?? false,
|
hasAudio: output?.streams.some((s) => s.codec_type === "audio") ?? false,
|
||||||
isVFR,
|
isVFR,
|
||||||
|
hasAlpha,
|
||||||
colorSpace,
|
colorSpace,
|
||||||
};
|
};
|
||||||
})();
|
})();
|
||||||
|
|||||||
@@ -196,7 +196,7 @@ async function compileHtmlFile(
|
|||||||
const preResolved = extractResolvedMedia(compiledHtml);
|
const preResolved = extractResolvedMedia(compiledHtml);
|
||||||
const clampResults = await Promise.all(
|
const clampResults = await Promise.all(
|
||||||
preResolved
|
preResolved
|
||||||
.filter((el) => !!el.src)
|
.filter((el) => !!el.src && !el.loop)
|
||||||
.map(async (el) => {
|
.map(async (el) => {
|
||||||
const { duration: maxDuration } = await resolveMediaDuration(
|
const { duration: maxDuration } = await resolveMediaDuration(
|
||||||
el.src!,
|
el.src!,
|
||||||
@@ -1097,6 +1097,7 @@ export interface BrowserMediaElement {
|
|||||||
end: number;
|
end: number;
|
||||||
duration: number;
|
duration: number;
|
||||||
mediaStart: number;
|
mediaStart: number;
|
||||||
|
loop: boolean;
|
||||||
hasAudio: boolean;
|
hasAudio: boolean;
|
||||||
volume: number;
|
volume: number;
|
||||||
}
|
}
|
||||||
@@ -1111,6 +1112,7 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
|
|||||||
end: number;
|
end: number;
|
||||||
duration: number;
|
duration: number;
|
||||||
mediaStart: number;
|
mediaStart: number;
|
||||||
|
loop: boolean;
|
||||||
hasAudio: boolean;
|
hasAudio: boolean;
|
||||||
volume: number;
|
volume: number;
|
||||||
}[] = [];
|
}[] = [];
|
||||||
@@ -1126,6 +1128,7 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
|
|||||||
const end = parseFloat(htmlEl.getAttribute("data-end") || "0");
|
const end = parseFloat(htmlEl.getAttribute("data-end") || "0");
|
||||||
const duration = parseFloat(htmlEl.getAttribute("data-duration") || "0");
|
const duration = parseFloat(htmlEl.getAttribute("data-duration") || "0");
|
||||||
const mediaStart = parseFloat(htmlEl.getAttribute("data-media-start") || "0");
|
const mediaStart = parseFloat(htmlEl.getAttribute("data-media-start") || "0");
|
||||||
|
const loop = htmlEl.hasAttribute("loop");
|
||||||
const hasAudio = htmlEl.getAttribute("data-has-audio") === "true";
|
const hasAudio = htmlEl.getAttribute("data-has-audio") === "true";
|
||||||
const volume = parseFloat(htmlEl.getAttribute("data-volume") || "1");
|
const volume = parseFloat(htmlEl.getAttribute("data-volume") || "1");
|
||||||
|
|
||||||
@@ -1137,6 +1140,7 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
|
|||||||
end,
|
end,
|
||||||
duration,
|
duration,
|
||||||
mediaStart,
|
mediaStart,
|
||||||
|
loop,
|
||||||
hasAudio,
|
hasAudio,
|
||||||
volume,
|
volume,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1231,6 +1231,9 @@ export async function executeRenderJob(
|
|||||||
if (el.hasAudio && !existing.hasAudio) {
|
if (el.hasAudio && !existing.hasAudio) {
|
||||||
existing.hasAudio = true;
|
existing.hasAudio = true;
|
||||||
}
|
}
|
||||||
|
if (el.loop && !existing.loop) {
|
||||||
|
existing.loop = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// New video discovered from browser
|
// New video discovered from browser
|
||||||
@@ -1240,6 +1243,7 @@ export async function executeRenderJob(
|
|||||||
start: el.start,
|
start: el.start,
|
||||||
end: el.end,
|
end: el.end,
|
||||||
mediaStart: el.mediaStart,
|
mediaStart: el.mediaStart,
|
||||||
|
loop: el.loop,
|
||||||
hasAudio: el.hasAudio,
|
hasAudio: el.hasAudio,
|
||||||
});
|
});
|
||||||
existingVideoIds.add(el.id);
|
existingVideoIds.add(el.id);
|
||||||
|
|||||||
Reference in New Issue
Block a user