feat: make creator media edits render-safe (#3322)

* feat: make creator media edits render-safe

* fix: align media playback timing

* docs: add creator editing recipes

* docs: expand creator editing guidance

* fix: unify media source offsets

* fix: scale natural media duration

* fix: preserve natural media zero spans

* fix: align compiled natural media timing

* test: classify compiler media test as integration

* fix: drop inactive media windows

* fix: unify literal timing parsing

* fix: keep browser media parsing serializable

* fix: keep page timing readers strict

* fix: close remaining preview timing gaps

* fix(core): preserve Studio voice pitch at playback speed

* chore: keep creator contract source-neutral
This commit is contained in:
Miguel Ángel
2026-08-18 10:17:02 -04:00
committed by GitHub
parent 049f5618d7
commit afafca4b96
42 changed files with 2695 additions and 307 deletions
@@ -0,0 +1,133 @@
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import { spawnSync } from "node:child_process";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { parseHTML } from "linkedom";
import { compileForRender } from "./htmlCompiler.js";
describe("compileForRender natural media duration parity", () => {
const projectDir = mkdtempSync(join(tmpdir(), "hf-compiler-natural-duration-"));
const sourcePath = join(projectDir, "ten-seconds.mp4");
const audioPath = join(projectDir, "ten-seconds.wav");
beforeAll(() => {
const ffmpeg = process.env.HYPERFRAMES_FFMPEG_PATH || "ffmpeg";
const generate = (args: string[]) => {
const generated = spawnSync(ffmpeg, ["-hide_banner", "-loglevel", "error", ...args], {
encoding: "utf8",
});
if (generated.status !== 0) throw new Error(generated.stderr || "failed to create fixture");
};
generate([
"-f",
"lavfi",
"-i",
"color=c=red:s=16x16:r=1:d=10",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
"-y",
sourcePath,
]);
generate([
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=48000:duration=10",
"-c:a",
"pcm_s16le",
"-y",
audioPath,
]);
});
afterAll(() => rmSync(projectDir, { recursive: true, force: true }));
async function compile(media: string) {
const htmlPath = join(projectDir, "index.html");
writeFileSync(
htmlPath,
`<!doctype html><html><body><main data-composition-id="root" data-start="0" data-duration="100" data-width="16" data-height="16">${media}</main></body></html>`,
);
const result = await compileForRender(projectDir, htmlPath, projectDir, {
allowSystemFontCapture: false,
});
return { result, document: parseHTML(result.html).document };
}
it("uses shared playback-rate parsing for duration-less video and audio", async () => {
const cases = [
["video-rate-2", "video", 'data-playback-rate="2"', 5],
["video-rate-2x", "video", 'data-playback-rate="2x"', 5],
["video-rate-0x2", "video", 'data-playback-rate="0x2"', 10],
["audio-rate-2x", "audio", 'data-playback-rate="2x"', 5],
] as const;
const tags = cases
.map(
([id, tag, attrs]) =>
`<${tag} id="${id}" src="${tag === "audio" ? "ten-seconds.wav" : "ten-seconds.mp4"}" data-start="0" ${attrs}></${tag}>`,
)
.join("");
const { document } = await compile(tags);
for (const [id, , , expected] of cases) {
const element = document.getElementById(id)!;
expect(Number(element.getAttribute("data-duration"))).toBeCloseTo(expected, 3);
expect(Number(element.getAttribute("data-end"))).toBeCloseTo(expected, 3);
}
});
it("uses shared playback-start precedence and fallback semantics", async () => {
const cases = [
["precedence", 'data-playback-start="2" data-media-start="7"', 8],
["empty", 'data-playback-start="" data-media-start="7"', 3],
["invalid", 'data-playback-start="later" data-media-start="7"', 3],
["negative-playback", 'data-playback-start="-1" data-media-start="7"', 3],
["negative-media", 'data-media-start="-1"', 10],
] as const;
const tags = cases
.map(
([id, attrs]) => `<video id="${id}" src="ten-seconds.mp4" data-start="0" ${attrs}></video>`,
)
.join("");
const { document } = await compile(tags);
for (const [id, , expected] of cases) {
const element = document.getElementById(id)!;
expect(Number(element.getAttribute("data-duration"))).toBeCloseTo(expected, 3);
expect(Number(element.getAttribute("data-end"))).toBeCloseTo(expected, 3);
}
});
it("injects a known zero span at and past EOF so video and audio are inactive", async () => {
const { result, document } = await compile(`
<video id="video-eof" src="ten-seconds.mp4" data-start="4" data-media-start="10" muted></video>
<video id="video-past" src="ten-seconds.mp4" data-start="5" data-media-start="11" muted></video>
<audio id="audio-eof" src="ten-seconds.wav" data-start="6" data-media-start="10"></audio>
<audio id="audio-past" src="ten-seconds.wav" data-start="7" data-media-start="11"></audio>`);
for (const [id, start] of [
["video-eof", 4],
["video-past", 5],
["audio-eof", 6],
["audio-past", 7],
] as const) {
const element = document.getElementById(id)!;
expect(element.getAttribute("data-duration")).toBe("0");
expect(element.getAttribute("data-end")).toBe(String(start));
}
expect(result.videos).toEqual([]);
expect(result.audios).toEqual([]);
});
it("keeps explicit data-duration authoritative", async () => {
const { document } = await compile(
'<video id="explicit" src="ten-seconds.mp4" data-start="2" data-duration="7" data-playback-rate="2"></video>',
);
const element = document.getElementById("explicit")!;
expect(element.getAttribute("data-duration")).toBe("7");
expect(element.getAttribute("data-end")).toBe("9");
});
});
@@ -3,6 +3,7 @@ import { describe, expect, it, mock, beforeAll } from "bun:test";
import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { runInThisContext } from "node:vm";
import { parseHTML } from "linkedom";
import { interpolateVolumeGain } from "@hyperframes/core/media-volume-envelope";
import { defaultLogger } from "../logger.js";
@@ -22,11 +23,16 @@ import {
localizeRemoteImageSources,
localizeRemoteFontFaces,
recompileWithResolutions,
resolveCompositionDurations,
} from "./htmlCompiler.js";
import { validateNoSystemFonts } from "./render/planValidation.js";
describe("discoverMediaFromBrowser", () => {
async function discover(html: string, currentSrcById: Record<string, string>) {
async function discover(
html: string,
currentSrcById: Record<string, string>,
serializeCallback = false,
) {
const { document } = parseHTML(html);
for (const [id, currentSrc] of Object.entries(currentSrcById)) {
const element = document.getElementById(id);
@@ -35,7 +41,13 @@ describe("discoverMediaFromBrowser", () => {
const previousDocument = Reflect.get(globalThis, "document");
Reflect.set(globalThis, "document", document);
try {
return await discoverMediaFromBrowser({ evaluate: async (collect) => collect() } as never);
return await discoverMediaFromBrowser({
evaluate: async (collect) => {
if (!serializeCallback) return collect();
const isolated = runInThisContext(`(${collect.toString()})`) as typeof collect;
return isolated();
},
} as never);
} finally {
if (previousDocument === undefined) Reflect.deleteProperty(globalThis, "document");
else Reflect.set(globalThis, "document", previousDocument);
@@ -84,6 +96,15 @@ describe("discoverMediaFromBrowser", () => {
src: "https://cdn.example/runtime.avif",
});
});
it("keeps strict timing parsing self-contained after Puppeteer serializes the callback", async () => {
const media = await discover(
'<video id="clip" src="clip.mp4" data-start="0" data-end="2" data-duration="2" data-media-start="1"></video>',
{},
true,
);
expect(media[0]).toMatchObject({ start: 0, end: 2, duration: 2, mediaStart: 1 });
});
});
function validTestMediaResponse(): Response {
@@ -1975,6 +1996,41 @@ h1 { font-size: 2rem; }`;
});
describe("discoverAudioVolumeAutomationFromTimeline", () => {
it("treats trailing-garbage duration as unknown instead of truncating automation sampling", async () => {
class TestAudioElement {
id = "music";
dataset = { start: "0", duration: "5s", volume: "0" };
volume = 0;
}
class TestVideoElement {}
const audio = new TestAudioElement();
const previous = {
window: globalThis.window,
document: globalThis.document,
audio: globalThis.HTMLAudioElement,
video: globalThis.HTMLVideoElement,
};
globalThis.window = {
__timelines: { root: { totalTime: (time: number) => (audio.volume = time < 7 ? 0 : 1) } },
} as any;
globalThis.document = {
querySelector: () => ({ getAttribute: () => "root" }),
getElementById: () => audio,
} as any;
globalThis.HTMLAudioElement = TestAudioElement as any;
globalThis.HTMLVideoElement = TestVideoElement as any;
try {
const page = { evaluate: async (fn: (arg: any) => unknown, arg: unknown) => fn(arg) } as any;
const [automation] = await discoverAudioVolumeAutomationFromTimeline(page, ["music"], 10, 1);
expect(automation?.keyframes).toContainEqual({ time: 7, volume: 1 });
} finally {
globalThis.window = previous.window;
globalThis.document = previous.document;
globalThis.HTMLAudioElement = previous.audio;
globalThis.HTMLVideoElement = previous.video;
}
});
it("emits plateau boundaries around a sampled volume change", async () => {
class TestAudioElement {
id = "music";
@@ -2095,6 +2151,55 @@ describe("discoverAudioVolumeAutomationFromTimeline", () => {
});
});
describe("resolveCompositionDurations strict literal timing", () => {
it("falls through whitespace data-duration to a valid composition duration", async () => {
const previousDocument = globalThis.document;
const previousWindow = globalThis.window;
globalThis.window = { __timelines: {} } as any;
globalThis.document = {
getElementById: () => ({
getAttribute: (name: string) =>
name === "data-duration" ? " " : name === "data-composition-duration" ? "5" : null,
}),
} as any;
try {
const page = {
evaluate: async (fn: (arg: unknown) => unknown, arg: unknown) => fn(arg),
} as any;
const result = await resolveCompositionDurations(page, [
{ id: "scene", tagName: "div", start: 0, mediaStart: 0, playbackRate: 1 },
]);
expect(result).toEqual([{ id: "scene", duration: 5 }]);
} finally {
globalThis.document = previousDocument;
globalThis.window = previousWindow;
}
});
it("does not partially parse trailing-garbage composition duration", async () => {
const previousDocument = globalThis.document;
const previousWindow = globalThis.window;
globalThis.window = { __timelines: {} } as any;
globalThis.document = {
getElementById: () => ({
getAttribute: (name: string) => (name === "data-composition-duration" ? "5s" : null),
}),
} as any;
try {
const page = {
evaluate: async (fn: (arg: unknown) => unknown, arg: unknown) => fn(arg),
} as any;
const result = await resolveCompositionDurations(page, [
{ id: "scene", tagName: "div", start: 0, mediaStart: 0, playbackRate: 1 },
]);
expect(result).toEqual([]);
} finally {
globalThis.document = previousDocument;
globalThis.window = previousWindow;
}
});
});
describe("sub-composition variable injection (render path, #2064)", () => {
function writeSubCompVarProject(hostVars: string): string {
const projectDir = mkdtempSync(join(tmpdir(), "hf-subvar-"));
+108 -51
View File
@@ -21,6 +21,9 @@ import {
shouldClampResolvedMediaDuration,
CSS_URL_RE,
isNonRelativeUrl,
parseStrictFiniteTimingNumber,
readMediaStart,
resolveNaturalMediaTimelineDurationFromValues,
type ResolvedDuration,
type UnresolvedElement,
} from "@hyperframes/core";
@@ -421,12 +424,13 @@ export function detectShaderTransitionUsage(html: string): boolean {
async function resolveMediaDuration(
src: string,
mediaStart: number,
playbackRate: number,
baseDir: string,
downloadDir: string,
tagName: string,
elementIdentity: string,
log?: ProducerLogger,
): Promise<{ duration: number; resolvedPath: string }> {
): Promise<{ duration: number | null; resolvedPath: string }> {
let filePath = src;
if (isHttpUrl(src)) {
@@ -438,14 +442,14 @@ async function resolveMediaDuration(
} catch {
// Download failed (e.g. 404 placeholder URL) — skip gracefully.
// The element will get duration 0 and be excluded from the render.
return { duration: 0, resolvedPath: src };
return { duration: null, resolvedPath: src };
}
} else if (!filePath.startsWith("/")) {
filePath = join(baseDir, filePath);
}
if (!existsSync(filePath)) {
return { duration: 0, resolvedPath: filePath };
return { duration: null, resolvedPath: filePath };
}
return withMediaProbeSlot(async () => {
@@ -473,7 +477,7 @@ async function resolveMediaDuration(
"file — the element is dropped from the render. Point it at a rendered media file.",
);
}
return { duration: 0, resolvedPath: filePath };
return { duration: null, resolvedPath: filePath };
}
throw error;
}
@@ -489,13 +493,16 @@ async function resolveMediaDuration(
// Source file has no audio stream (e.g. a silent video used as an audio src).
// Return duration 0 so the element is excluded from the composition gracefully,
// matching how missing files and failed downloads are already handled above.
return { duration: 0, resolvedPath: filePath };
return { duration: null, resolvedPath: filePath };
}
}
const fileDuration = metadata.durationSeconds;
const effectiveDuration = fileDuration - mediaStart;
const duration = effectiveDuration > 0 ? effectiveDuration : fileDuration;
const duration = resolveNaturalMediaTimelineDurationFromValues(
fileDuration,
mediaStart,
playbackRate,
);
return { duration, resolvedPath: filePath };
});
@@ -525,6 +532,7 @@ async function compileHtmlFile(
resolveMediaDuration(
el.src!,
el.mediaStart,
el.playbackRate,
baseDir,
downloadDir,
el.tagName,
@@ -533,7 +541,9 @@ async function compileHtmlFile(
).then(({ duration }) => ({ id: el.id, duration })),
),
);
const resolutions: ResolvedDuration[] = resolvedResults.filter((r) => r.duration > 0);
const resolutions: ResolvedDuration[] = resolvedResults.filter(
(r): r is ResolvedDuration => r.duration != null && Number.isFinite(r.duration),
);
let compiledHtml =
resolutions.length > 0 ? injectDurations(staticCompiled, resolutions) : staticCompiled;
@@ -548,6 +558,7 @@ async function compileHtmlFile(
const { duration: maxDuration } = await resolveMediaDuration(
el.src!,
el.mediaStart,
el.playbackRate,
baseDir,
downloadDir,
el.tagName,
@@ -560,7 +571,7 @@ async function compileHtmlFile(
const clampList: ResolvedDuration[] = [];
for (const r of clampResults) {
if (
r.maxDuration > 0 &&
r.maxDuration != null &&
shouldClampResolvedMediaDuration(r.tagName, r.duration, r.maxDuration)
) {
clampList.push({ id: r.id, duration: r.maxDuration });
@@ -646,8 +657,7 @@ async function parseSubCompositions(
if (!srcPath) continue;
const elStart = parseFloat(el.getAttribute("data-start") || "0");
const elEndRaw = el.getAttribute("data-end");
const elEnd = elEndRaw ? parseFloat(elEndRaw) : Infinity;
const elEnd = parseStrictFiniteTimingNumber(el.getAttribute("data-end")) ?? Infinity;
const absoluteStart = parentOffset + elStart;
const absoluteEnd = Math.min(parentEnd, isFinite(elEnd) ? parentOffset + elEnd : Infinity);
@@ -2070,11 +2080,9 @@ export async function compileForRender(
// Static duration (may be 0 if set at runtime by GSAP)
const staticDuration = rootEl
? parseFloat(
rootEl.getAttribute("data-duration") ||
rootEl.getAttribute("data-composition-duration") ||
"0",
)
? (parseStrictFiniteTimingNumber(rootEl.getAttribute("data-duration")) ??
parseStrictFiniteTimingNumber(rootEl.getAttribute("data-composition-duration")) ??
0)
: 0;
return {
@@ -2126,12 +2134,13 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
const elements = await page.evaluate(() => {
const results: {
id: string;
tagName: string;
tagName: "video" | "audio" | "image";
src: string;
start: number;
end: number;
duration: number;
mediaStart: number;
endRaw: string | null;
durationRaw: string | null;
playbackStartRaw: string | null;
mediaStartRaw: string | null;
loop: boolean;
hasAudio: boolean;
volume: number;
@@ -2156,15 +2165,21 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
mediaEls.forEach((el) => {
const htmlEl = el as HTMLVideoElement | HTMLAudioElement | HTMLImageElement;
const isImage = htmlEl.tagName.toLowerCase() === "img";
const tagName: "video" | "audio" | "image" = isImage
? "image"
: htmlEl.tagName.toLowerCase() === "video"
? "video"
: "audio";
const id = htmlEl.id || (isImage ? autoImageIds.get(htmlEl) : undefined);
if (!id) return;
// currentSrc is authoritative for <video>/<audio><source> and responsive images.
const src = htmlEl.currentSrc || htmlEl.src || htmlEl.getAttribute("src") || "";
const start = parseFloat(htmlEl.getAttribute("data-start") || "0");
const end = parseFloat(htmlEl.getAttribute("data-end") || "0");
const duration = parseFloat(htmlEl.getAttribute("data-duration") || "0");
const mediaStart = parseFloat(htmlEl.getAttribute("data-media-start") || "0");
const endRaw = htmlEl.getAttribute("data-end");
const durationRaw = htmlEl.getAttribute("data-duration");
const playbackStartRaw = htmlEl.getAttribute("data-playback-start");
const mediaStartRaw = htmlEl.getAttribute("data-media-start");
const loop = htmlEl.hasAttribute("loop");
const hasAudio = htmlEl.getAttribute("data-has-audio") === "true";
const volume = parseFloat(htmlEl.getAttribute("data-volume") || "1");
@@ -2174,12 +2189,13 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
results.push({
id,
tagName: isImage ? "image" : htmlEl.tagName.toLowerCase(),
tagName,
src,
start,
end,
duration,
mediaStart,
endRaw,
durationRaw,
playbackStartRaw,
mediaStartRaw,
loop,
hasAudio,
volume,
@@ -2190,7 +2206,18 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
return results;
});
return elements as BrowserMediaElement[];
return elements.map(({ endRaw, durationRaw, playbackStartRaw, mediaStartRaw, ...element }) => ({
...element,
end: parseStrictFiniteTimingNumber(endRaw) ?? 0,
duration: parseStrictFiniteTimingNumber(durationRaw) ?? 0,
mediaStart: readMediaStart({
getAttribute(name: string) {
if (name === "data-playback-start") return playbackStartRaw;
if (name === "data-media-start") return mediaStartRaw;
return null;
},
}),
}));
}
export async function discoverAudioVolumeAutomationFromTimeline(
@@ -2202,8 +2229,34 @@ export async function discoverAudioVolumeAutomationFromTimeline(
if (audioIds.length === 0 || compositionDuration <= 0) return [];
const sampleStep = 1 / Math.min(60, Math.max(1, sampleFps));
const rawWindows = await page.evaluate((ids: string[]) => {
return ids.flatMap((id) => {
const el = document.getElementById(id) ?? document.getElementById(id.replace(/-audio$/, ""));
if (!(el instanceof HTMLAudioElement) && !(el instanceof HTMLVideoElement)) return [];
return [
{
id,
startRaw: el.dataset.start ?? null,
endRaw: el.dataset.end ?? null,
durationRaw: el.dataset.duration ?? null,
},
];
});
}, audioIds);
const clips = rawWindows.map(({ id, startRaw, endRaw, durationRaw }) => {
const start = parseStrictFiniteTimingNumber(startRaw) ?? 0;
const authoredDuration = parseStrictFiniteTimingNumber(durationRaw);
const authoredEnd = parseStrictFiniteTimingNumber(endRaw);
const end =
authoredDuration != null && authoredDuration > 0
? start + authoredDuration
: authoredEnd != null && authoredEnd > start
? authoredEnd
: compositionDuration;
return { id, start, end };
});
return page.evaluate(
({ ids, duration, step }) => {
({ clips, duration, step }) => {
const results: { id: string; keyframes: { time: number; volume: number }[] }[] = [];
const timelines = (window as unknown as { __timelines?: Record<string, unknown> })
.__timelines;
@@ -2229,20 +2282,11 @@ export async function discoverAudioVolumeAutomationFromTimeline(
}
};
for (const id of ids) {
for (const { id, start, end } of clips) {
const el =
document.getElementById(id) ?? document.getElementById(id.replace(/-audio$/, ""));
if (!(el instanceof HTMLAudioElement) && !(el instanceof HTMLVideoElement)) continue;
const start = Number.parseFloat(el.dataset.start ?? "0") || 0;
const endAttr = Number.parseFloat(el.dataset.end ?? "");
const durationAttr = Number.parseFloat(el.dataset.duration ?? "");
const end =
Number.isFinite(durationAttr) && durationAttr > 0
? start + durationAttr
: Number.isFinite(endAttr) && endAttr > start
? endAttr
: duration;
const sampleStart = Math.max(0, start);
const sampleEnd = Math.min(duration, end);
const initialVolumeAttr = Number.parseFloat(el.dataset.volume ?? "");
@@ -2293,7 +2337,7 @@ export async function discoverAudioVolumeAutomationFromTimeline(
seekTl(0);
return results;
},
{ ids: audioIds, duration: compositionDuration, step: sampleStep },
{ clips, duration: compositionDuration, step: sampleStep },
);
}
@@ -2415,7 +2459,13 @@ export async function resolveCompositionDurations(
const results = await page.evaluate((compIds: string[]) => {
const win = window as unknown as { __timelines?: Record<string, { duration(): number }> };
const timelines = win.__timelines || {};
const resolved: { id: string; duration: number; source: string }[] = [];
const resolved: {
id: string;
duration: number;
source: string;
durationRaw?: string;
compositionDurationRaw?: string;
}[] = [];
for (const id of compIds) {
// Try window.__timelines[id].duration() first (GSAP timeline)
@@ -2431,14 +2481,17 @@ export async function resolveCompositionDurations(
// Fallback: check for authored duration on the element itself
const el = document.getElementById(id);
if (el) {
const compDurAttr =
el.getAttribute("data-duration") || el.getAttribute("data-composition-duration");
if (compDurAttr) {
const dur = parseFloat(compDurAttr);
if (dur > 0) {
resolved.push({ id, duration: dur, source: "data-duration" });
continue;
}
const durationRaw = el.getAttribute("data-duration");
const compositionDurationRaw = el.getAttribute("data-composition-duration");
if (durationRaw != null || compositionDurationRaw != null) {
resolved.push({
id,
duration: 0,
source: "data-duration",
...(durationRaw != null ? { durationRaw } : {}),
...(compositionDurationRaw != null ? { compositionDurationRaw } : {}),
});
continue;
}
}
@@ -2450,8 +2503,12 @@ export async function resolveCompositionDurations(
const resolutions: ResolvedDuration[] = [];
for (const r of results) {
if (r.duration > 0) {
resolutions.push({ id: r.id, duration: r.duration });
const duration =
parseStrictFiniteTimingNumber(r.durationRaw) ??
parseStrictFiniteTimingNumber(r.compositionDurationRaw) ??
r.duration;
if (duration != null && duration > 0) {
resolutions.push({ id: r.id, duration });
}
}