mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 12:54:29 +00:00
Merge pull request #2516 from heygen-com/fix/2514-hold-last-video-frame
fix(video): hold final frame through composition
This commit is contained in:
Miguel Ángel
@@ -11,6 +11,7 @@ import {
|
||||
runAndParseJsonEnvelope,
|
||||
} from "./deprecationTestHarness.js";
|
||||
import {
|
||||
auditClipDurations,
|
||||
extractCompositionErrorsFromLint,
|
||||
navigationTimeoutHint,
|
||||
raceMediaReady,
|
||||
@@ -49,6 +50,58 @@ vi.mock("../utils/producer.js", () => ({
|
||||
vi.mock("../utils/project.js", () => resolveProjectMock());
|
||||
vi.mock("../utils/lintProject.js", () => lintProjectFailureMock());
|
||||
|
||||
describe("auditClipDurations", () => {
|
||||
it("audits audio only because explicit video slots hold their final frame", async () => {
|
||||
let selector = "";
|
||||
const originalDocument = globalThis.document;
|
||||
const audio = {
|
||||
duration: 1,
|
||||
id: "voice",
|
||||
loop: false,
|
||||
tagName: "AUDIO",
|
||||
getAttribute: (name: string) =>
|
||||
name === "data-duration" ? "5" : name === "data-media-start" ? "0" : null,
|
||||
};
|
||||
const page = {
|
||||
evaluate: async (fn: (waitMs: number) => unknown, waitMs: number) => {
|
||||
Object.defineProperty(globalThis, "document", {
|
||||
configurable: true,
|
||||
value: {
|
||||
querySelectorAll: (query: string) => {
|
||||
selector = query;
|
||||
return [audio];
|
||||
},
|
||||
},
|
||||
});
|
||||
return fn(waitMs);
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const warnings = await auditClipDurations(
|
||||
page as never,
|
||||
({ slotSeconds, mediaSeconds }) => ({
|
||||
shortfallSeconds: slotSeconds - mediaSeconds,
|
||||
toleranceSeconds: 0.05,
|
||||
}),
|
||||
10,
|
||||
);
|
||||
expect(selector).toBe("audio[data-duration]");
|
||||
expect(warnings).toHaveLength(1);
|
||||
expect(warnings[0]?.text).toContain('Audio "voice"');
|
||||
} finally {
|
||||
if (originalDocument === undefined) {
|
||||
Reflect.deleteProperty(globalThis, "document");
|
||||
} else {
|
||||
Object.defineProperty(globalThis, "document", {
|
||||
configurable: true,
|
||||
value: originalDocument,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Regression for the validate audio-duration-probe timeout: a slow-loading
|
||||
// media element's duration was snapshotted once, at a fixed point in time,
|
||||
// and any element still mid-load was permanently misreported as unreadable.
|
||||
|
||||
@@ -127,8 +127,9 @@ export function raceMediaReady(
|
||||
}
|
||||
|
||||
/**
|
||||
* Flag `<video>`/`<audio>` clips whose source is meaningfully shorter than their
|
||||
* `data-duration` slot (the slot gets silently shortened in renders). Runs in
|
||||
* Flag `<audio>` clips whose source is meaningfully shorter than their
|
||||
* `data-duration` slot (the slot gets silently shortened in renders). Videos
|
||||
* intentionally hold their final frame through an explicit longer slot. Runs in
|
||||
* the live page to read each element's intrinsic `.duration`, which static lint
|
||||
* can't see.
|
||||
*/
|
||||
@@ -140,7 +141,7 @@ export async function auditClipDurations(
|
||||
// fallow-ignore-next-line complexity
|
||||
const clips = await page.evaluate(async (maxWaitMs: number) => {
|
||||
const nodes = Array.from(
|
||||
document.querySelectorAll("video[data-duration], audio[data-duration]"),
|
||||
document.querySelectorAll("audio[data-duration]"),
|
||||
) as HTMLMediaElement[];
|
||||
|
||||
// The caller's page-settle sleep is a flat, unconditional wait shared with
|
||||
|
||||
@@ -12,11 +12,39 @@ describe("compileHtml", () => {
|
||||
expect(compiled).toContain('data-end="4"');
|
||||
});
|
||||
|
||||
it("still clamps non-looping media durations to source duration", async () => {
|
||||
it("preserves an explicit non-looping video slot past source end", async () => {
|
||||
const html = '<video id="hero" src="hero.webm" data-start="0" data-duration="4" data-end="4">';
|
||||
|
||||
const compiled = await compileHtml(html, "/project", async () => 3.125);
|
||||
|
||||
expect(compiled).toContain('data-duration="4"');
|
||||
expect(compiled).toContain('data-end="4"');
|
||||
});
|
||||
|
||||
it("uses natural duration for a video without an explicit slot inside a composition", async () => {
|
||||
const html =
|
||||
'<div data-composition-id="root" data-start="0" data-duration="5">' +
|
||||
'<video id="hero" src="hero.webm" data-start="0">' +
|
||||
"</div>";
|
||||
|
||||
const compiled = await compileHtml(html, "/project", async () => 1);
|
||||
|
||||
expect(compiled).toContain('data-duration="1"');
|
||||
expect(compiled).toContain('data-end="1"');
|
||||
});
|
||||
|
||||
it("uses natural duration for a standalone video without a composition window", async () => {
|
||||
const html = '<video id="hero" src="hero.webm" data-start="0">';
|
||||
const compiled = await compileHtml(html, "/project", async () => 1);
|
||||
expect(compiled).toContain('data-duration="1"');
|
||||
expect(compiled).toContain('data-end="1"');
|
||||
});
|
||||
|
||||
it("still clamps non-looping audio durations to source duration", async () => {
|
||||
const html = '<audio id="voice" src="voice.wav" data-start="0" data-duration="4" data-end="4">';
|
||||
|
||||
const compiled = await compileHtml(html, "/project", async () => 3.125);
|
||||
|
||||
expect(compiled).toContain('data-duration="3.125"');
|
||||
expect(compiled).toContain('data-end="3.125"');
|
||||
});
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
injectDurations,
|
||||
extractResolvedMedia,
|
||||
clampDurations,
|
||||
shouldClampMediaDuration,
|
||||
shouldClampResolvedMediaDuration,
|
||||
type ResolvedDuration,
|
||||
} from "./timingCompiler";
|
||||
|
||||
@@ -23,7 +23,8 @@ function resolveMediaSrc(src: string, projectDir: string): string {
|
||||
*
|
||||
* 1. Static pass: compileTimingAttrs() adds data-end where data-duration exists
|
||||
* 2. For unresolved video/audio (no data-duration): probe via probeMediaDuration, inject durations
|
||||
* 3. For pre-resolved video/audio: validate data-duration against actual source, clamp if needed
|
||||
* 3. For pre-resolved audio: clamp data-duration to playable source when needed.
|
||||
* Explicit video slots are preserved and hold their final frame.
|
||||
*
|
||||
* @param rawHtml - The raw HTML string
|
||||
* @param projectDir - The project directory for resolving relative paths
|
||||
@@ -54,10 +55,8 @@ export async function compileHtml(
|
||||
if (fileDuration <= 0) continue;
|
||||
|
||||
const effectiveDuration = fileDuration - el.mediaStart;
|
||||
resolutions.push({
|
||||
id: el.id,
|
||||
duration: effectiveDuration > 0 ? effectiveDuration : fileDuration,
|
||||
});
|
||||
const sourceDuration = effectiveDuration > 0 ? effectiveDuration : fileDuration;
|
||||
resolutions.push({ id: el.id, duration: sourceDuration });
|
||||
}
|
||||
|
||||
if (resolutions.length > 0) {
|
||||
@@ -65,7 +64,8 @@ export async function compileHtml(
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: Validate pre-resolved media — clamp data-duration to actual source duration
|
||||
// Phase 2: Bound authored audio to playable source. Explicit video slots may
|
||||
// outlive their source and render by holding the final frame.
|
||||
const preResolved = extractResolvedMedia(html);
|
||||
const clampList: ResolvedDuration[] = [];
|
||||
|
||||
@@ -77,7 +77,7 @@ export async function compileHtml(
|
||||
if (fileDuration <= 0) continue;
|
||||
|
||||
const maxDuration = fileDuration - el.mediaStart;
|
||||
if (maxDuration > 0 && shouldClampMediaDuration(el.duration, maxDuration)) {
|
||||
if (maxDuration > 0 && shouldClampResolvedMediaDuration(el.tagName, el.duration, maxDuration)) {
|
||||
clampList.push({ id: el.id, duration: maxDuration });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ export {
|
||||
extractResolvedMedia,
|
||||
clampDurations,
|
||||
shouldClampMediaDuration,
|
||||
shouldClampResolvedMediaDuration,
|
||||
type UnresolvedElement,
|
||||
type ResolvedDuration,
|
||||
type ResolvedMediaElement,
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
injectDurations,
|
||||
extractResolvedMedia,
|
||||
clampDurations,
|
||||
shouldClampResolvedMediaDuration,
|
||||
} from "./timingCompiler.js";
|
||||
|
||||
// Raw 0x00 bytes in the HFMASK delimiters shipped once and broke every render
|
||||
@@ -264,3 +265,10 @@ describe("clampDurations", () => {
|
||||
expect(result).toContain('data-end="7"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldClampResolvedMediaDuration", () => {
|
||||
it("preserves an explicit video slot but keeps audio source-bounded", () => {
|
||||
expect(shouldClampResolvedMediaDuration("video", 5, 1)).toBe(false);
|
||||
expect(shouldClampResolvedMediaDuration("audio", 5, 1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -48,15 +48,29 @@ export interface CompilationResult {
|
||||
unresolved: UnresolvedElement[];
|
||||
}
|
||||
|
||||
// ffprobe precision can differ slightly across local and CI media stacks. Also
|
||||
// the floor for the engine's hold-last-frame tolerance (a slot left unclamped is
|
||||
// short by at most this), so they must move together.
|
||||
// ffprobe precision can differ slightly across local and CI media stacks, so
|
||||
// avoid shortening authored audio for insignificant probe drift.
|
||||
export const MEDIA_DURATION_CLAMP_EPSILON_SECONDS = 0.05;
|
||||
|
||||
export function shouldClampMediaDuration(declaredDuration: number, maxDuration: number): boolean {
|
||||
return declaredDuration > maxDuration + MEDIA_DURATION_CLAMP_EPSILON_SECONDS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether compilation should shorten an authored media slot to its source.
|
||||
*
|
||||
* Non-looping video intentionally keeps an explicit longer slot: browsers and
|
||||
* the render frame injector hold its final frame until that authored slot ends.
|
||||
* Audio has no frame to hold, so its slot remains bounded by playable source.
|
||||
*/
|
||||
export function shouldClampResolvedMediaDuration(
|
||||
tagName: ResolvedMediaElement["tagName"],
|
||||
declaredDuration: number,
|
||||
maxDuration: number,
|
||||
): boolean {
|
||||
return tagName === "audio" && shouldClampMediaDuration(declaredDuration, maxDuration);
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
function getAttr(tag: string, attr: string): string | null {
|
||||
|
||||
@@ -124,6 +124,7 @@ describe("@hyperframes/core public API exports", () => {
|
||||
expect(typeof core.extractResolvedMedia).toBe("function");
|
||||
expect(typeof core.clampDurations).toBe("function");
|
||||
expect(typeof core.shouldClampMediaDuration).toBe("function");
|
||||
expect(typeof core.shouldClampResolvedMediaDuration).toBe("function");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -138,6 +138,7 @@ export {
|
||||
extractResolvedMedia,
|
||||
clampDurations,
|
||||
shouldClampMediaDuration,
|
||||
shouldClampResolvedMediaDuration,
|
||||
MEDIA_DURATION_CLAMP_EPSILON_SECONDS,
|
||||
} from "./compiler/timingCompiler";
|
||||
|
||||
|
||||
@@ -19,7 +19,11 @@ import {
|
||||
} from "./adapters/video-texture-compat";
|
||||
import { forceDispatchSeekEvent } from "./adapters/seek-dispatch";
|
||||
import { createWaapiAdapter } from "./adapters/waapi";
|
||||
import { refreshRuntimeMediaCache, syncRuntimeMedia } from "./media";
|
||||
import {
|
||||
refreshRuntimeMediaCache,
|
||||
resolveRuntimeMediaClipDuration,
|
||||
syncRuntimeMedia,
|
||||
} from "./media";
|
||||
import { probeAndCacheElementVolume, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
|
||||
import { createPickerModule } from "./picker";
|
||||
import { createRuntimePlayer, type RuntimePlayerTransport } from "./player";
|
||||
@@ -1807,10 +1811,12 @@ export function initSandboxRuntimeModular(): void {
|
||||
const ownDuration = Number.parseFloat(element.dataset.duration ?? "");
|
||||
const explicitDuration =
|
||||
Number.isFinite(ownDuration) && ownDuration > 0 ? ownDuration : null;
|
||||
const candidates = [sourceDuration, hostRemaining, explicitDuration].filter(
|
||||
(value): value is number => value != null,
|
||||
);
|
||||
return candidates.length > 0 ? Math.min(...candidates) : null;
|
||||
return resolveRuntimeMediaClipDuration({
|
||||
isVideo: element.tagName === "VIDEO",
|
||||
sourceDuration,
|
||||
hostRemaining,
|
||||
explicitDuration,
|
||||
});
|
||||
},
|
||||
});
|
||||
// Attach probed volume keyframes to clips so syncRuntimeMedia can use the
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { describe, it, expect, vi, afterEach } from "vitest";
|
||||
import { readElementPlaybackRate, refreshRuntimeMediaCache, syncRuntimeMedia } from "./media";
|
||||
import {
|
||||
readElementPlaybackRate,
|
||||
refreshRuntimeMediaCache,
|
||||
resolveRuntimeMediaClipDuration,
|
||||
syncRuntimeMedia,
|
||||
} from "./media";
|
||||
import type { RuntimeMediaClip } from "./media";
|
||||
|
||||
function createVideo(attrs: Record<string, string>): HTMLVideoElement {
|
||||
@@ -195,6 +200,65 @@ describe("refreshRuntimeMediaCache", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveRuntimeMediaClipDuration", () => {
|
||||
it("preserves an explicit video slot beyond the source", () => {
|
||||
expect(
|
||||
resolveRuntimeMediaClipDuration({
|
||||
isVideo: true,
|
||||
sourceDuration: 1,
|
||||
hostRemaining: 8,
|
||||
explicitDuration: 5,
|
||||
}),
|
||||
).toBe(5);
|
||||
});
|
||||
|
||||
it("honors an explicit slot for a looping video instead of truncating it to one loop", () => {
|
||||
// Loop wrapping happens later in syncRuntimeMedia. Duration resolution must
|
||||
// preserve the authored window that the loop fills.
|
||||
expect(
|
||||
resolveRuntimeMediaClipDuration({
|
||||
isVideo: true,
|
||||
sourceDuration: 1,
|
||||
hostRemaining: null,
|
||||
explicitDuration: 10,
|
||||
}),
|
||||
).toBe(10);
|
||||
});
|
||||
|
||||
it("keeps audio bounded by its playable source", () => {
|
||||
expect(
|
||||
resolveRuntimeMediaClipDuration({
|
||||
isVideo: false,
|
||||
sourceDuration: 1,
|
||||
hostRemaining: 8,
|
||||
explicitDuration: 5,
|
||||
}),
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("uses natural duration for a video without an explicit slot", () => {
|
||||
expect(
|
||||
resolveRuntimeMediaClipDuration({
|
||||
isVideo: true,
|
||||
sourceDuration: 1,
|
||||
hostRemaining: 8,
|
||||
explicitDuration: null,
|
||||
}),
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("uses natural source duration when a video has no slot or host window", () => {
|
||||
expect(
|
||||
resolveRuntimeMediaClipDuration({
|
||||
isVideo: true,
|
||||
sourceDuration: 1,
|
||||
hostRemaining: null,
|
||||
explicitDuration: null,
|
||||
}),
|
||||
).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncRuntimeMedia", () => {
|
||||
function fakePlayedRanges(el: HTMLMediaElement, ranges: Array<[number, number]>): void {
|
||||
Object.defineProperty(el, "played", {
|
||||
@@ -439,6 +503,22 @@ describe("syncRuntimeMedia", () => {
|
||||
expect(clip.el.play).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("seeks a non-looping video to its final frame when entering an authored hold tail", () => {
|
||||
const clip = createMockClip({ start: 0, end: 5, duration: 5, sourceDuration: 0.25 });
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 4, playing: true, playbackRate: 1 });
|
||||
expect(clip.el.currentTime).toBe(0.25);
|
||||
expect(clip.el.play).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("seeks an ended video backward into its playable source", () => {
|
||||
const clip = createMockClip({ start: 0, end: 5, duration: 5, sourceDuration: 1 });
|
||||
Object.defineProperty(clip.el, "currentTime", { value: 1, writable: true, configurable: true });
|
||||
Object.defineProperty(clip.el, "ended", { value: true, writable: true, configurable: true });
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 0.9, playing: true, playbackRate: 1 });
|
||||
expect(clip.el.currentTime).toBe(0.9);
|
||||
expect(clip.el.play).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does restart a loop clip that has naturally ended while still within its active window", () => {
|
||||
const clip = createMockClip({ start: 0, end: 68.6, loop: true, sourceDuration: 60 });
|
||||
Object.defineProperty(clip.el, "paused", { value: true, writable: true });
|
||||
@@ -722,7 +802,7 @@ describe("syncRuntimeMedia", () => {
|
||||
expect(clip.el.currentTime).toBe(7);
|
||||
});
|
||||
|
||||
it("does not loop when loop is false", () => {
|
||||
it("holds the final frame instead of looping a non-looping video", () => {
|
||||
const clip = createMockClip({
|
||||
start: 0,
|
||||
end: 10,
|
||||
@@ -731,9 +811,9 @@ describe("syncRuntimeMedia", () => {
|
||||
sourceDuration: 3,
|
||||
});
|
||||
Object.defineProperty(clip.el, "currentTime", { value: 0, writable: true });
|
||||
// At t=7, relTime = 7 (no wrapping, even though > sourceDuration)
|
||||
// At t=7 the source is exhausted, so the final frame remains visible.
|
||||
syncRuntimeMedia({ clips: [clip], timeSeconds: 7, playing: false, playbackRate: 1 });
|
||||
expect(clip.el.currentTime).toBe(7);
|
||||
expect(clip.el.currentTime).toBe(3);
|
||||
});
|
||||
|
||||
it("asserts muted=true every tick while outputMuted is set", () => {
|
||||
|
||||
@@ -6,6 +6,30 @@ export function readElementPlaybackRate(el: HTMLMediaElement): number {
|
||||
return Number.isFinite(raw) && raw > 0 ? Math.max(0.1, Math.min(5, raw)) : 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a media element's timeline window without conflating a video's
|
||||
* authored display slot with the amount of source left to decode.
|
||||
*
|
||||
* An explicit video slot may outlive its source and holds the final frame.
|
||||
* Audio remains source-bounded because it has no visual hold state.
|
||||
*/
|
||||
export function resolveRuntimeMediaClipDuration(params: {
|
||||
isVideo: boolean;
|
||||
sourceDuration: number | null;
|
||||
hostRemaining: number | null;
|
||||
explicitDuration: number | null;
|
||||
}): number | null {
|
||||
const mediaDuration = params.isVideo
|
||||
? (params.explicitDuration ?? params.sourceDuration)
|
||||
: params.sourceDuration;
|
||||
const candidates = (
|
||||
params.isVideo
|
||||
? [mediaDuration, params.hostRemaining]
|
||||
: [mediaDuration, params.hostRemaining, params.explicitDuration]
|
||||
).filter((value): value is number => value != null && Number.isFinite(value) && value > 0);
|
||||
return candidates.length > 0 ? Math.min(...candidates) : null;
|
||||
}
|
||||
|
||||
export type RuntimeMediaClip = {
|
||||
el: HTMLVideoElement | HTMLAudioElement;
|
||||
start: number;
|
||||
@@ -167,15 +191,30 @@ export function syncRuntimeMedia(params: {
|
||||
const { el } = clip;
|
||||
if (!el.isConnected) continue;
|
||||
let relTime = (params.timeSeconds - clip.start) * clip.playbackRate + clip.mediaStart;
|
||||
// An ended non-loop element has played its file to natural completion.
|
||||
// Don't restart it — if the authored duration extends past the file's
|
||||
// actual length, the element sits silently until the composition ends.
|
||||
// (el.ended resets to false when the user scrubs back, so seeks work.)
|
||||
const isNonLoopVideo = el.tagName === "VIDEO" && !clip.loop;
|
||||
const isHeldVideoTail =
|
||||
isNonLoopVideo &&
|
||||
clip.sourceDuration != null &&
|
||||
relTime >= clip.sourceDuration &&
|
||||
params.timeSeconds >= clip.start &&
|
||||
params.timeSeconds < clip.end;
|
||||
if (isHeldVideoTail && clip.sourceDuration != null) {
|
||||
relTime = clip.sourceDuration;
|
||||
}
|
||||
const canSeekEndedVideoBackward =
|
||||
isNonLoopVideo &&
|
||||
clip.sourceDuration != null &&
|
||||
relTime >= clip.mediaStart &&
|
||||
relTime < clip.sourceDuration;
|
||||
// Audio that ended naturally stays silent. A non-loop video remains an
|
||||
// active visual through its authored window: tail seeks clamp to the final
|
||||
// frame, and backward seeks can re-enter playable source without depending
|
||||
// on the browser having reset `ended` first.
|
||||
const isActive =
|
||||
params.timeSeconds >= clip.start &&
|
||||
params.timeSeconds < clip.end &&
|
||||
relTime >= 0 &&
|
||||
(!el.ended || clip.loop);
|
||||
(!el.ended || clip.loop || isHeldVideoTail || canSeekEndedVideoBackward);
|
||||
if (isActive) {
|
||||
// Loop wrapping: when media reaches end, restart from mediaStart
|
||||
if (clip.loop && clip.sourceDuration != null && clip.sourceDuration > 0) {
|
||||
@@ -258,7 +297,10 @@ export function syncRuntimeMedia(params: {
|
||||
const firstTickOfClip = prevOffset === undefined;
|
||||
const offsetJumped = !firstTickOfClip && Math.abs(offset - prevOffset!) > 0.5;
|
||||
const catastrophicDrift = drift > 3;
|
||||
const hardSync = drift > 0.5 && (firstTickOfClip || offsetJumped || catastrophicDrift);
|
||||
const hardSync =
|
||||
(isHeldVideoTail && drift > 0.001) ||
|
||||
(el.ended && canSeekEndedVideoBackward && drift > 0.001) ||
|
||||
(drift > 0.5 && (firstTickOfClip || offsetJumped || catastrophicDrift));
|
||||
// Playing video elements use the browser's native decoder pipeline for
|
||||
// timing. Seeking a playing video resets the decoder, causing a ~150ms
|
||||
// freeze while it re-buffers — during which the monotonic clock advances,
|
||||
@@ -321,7 +363,9 @@ export function syncRuntimeMedia(params: {
|
||||
}
|
||||
playRequested.delete(el);
|
||||
}
|
||||
if (params.playing && el.paused && !playRequested.has(el) && !isUnplayable(el)) {
|
||||
if (isHeldVideoTail) {
|
||||
if (!el.paused) el.pause();
|
||||
} else if (params.playing && el.paused && !playRequested.has(el) && !isUnplayable(el)) {
|
||||
// `HTMLMediaElement.play()` is spec'd to queue playback and resolve
|
||||
// once enough data is buffered, so we can unconditionally call it —
|
||||
// no need to gate on `readyState` or defer to a `canplay` listener.
|
||||
|
||||
@@ -389,7 +389,7 @@ describe("FrameLookupTable", () => {
|
||||
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
|
||||
});
|
||||
|
||||
it("does not hold stale frames for non-looping clips after extracted frames end", () => {
|
||||
it("holds the last frame for a non-looping clip until its authored slot ends", () => {
|
||||
const table = createFrameLookupTable(
|
||||
[
|
||||
{
|
||||
@@ -406,7 +406,31 @@ describe("FrameLookupTable", () => {
|
||||
);
|
||||
|
||||
expect(table.getActiveFramePayloads(0.5).has("hero")).toBe(true);
|
||||
expect(table.getActiveFramePayloads(1.5).has("hero")).toBe(false);
|
||||
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
|
||||
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(29);
|
||||
expect(table.getFrame("hero", 4.5)).toBeTruthy();
|
||||
expect(table.getActiveFramePayloads(5.1).has("hero")).toBe(false);
|
||||
expect(table.getFrame("hero", 5.1)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not invent a held frame when extraction produced no frames", () => {
|
||||
const table = createFrameLookupTable(
|
||||
[
|
||||
{
|
||||
id: "hero",
|
||||
src: "clip.webm",
|
||||
start: 0,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
},
|
||||
],
|
||||
[fakeExtracted(0, 30)],
|
||||
);
|
||||
|
||||
expect(table.getActiveFramePayloads(4.5).has("hero")).toBe(false);
|
||||
expect(table.getFrame("hero", 4.5)).toBeNull();
|
||||
});
|
||||
|
||||
it("places a relative-reference video in its resolved window end-to-end (was blank)", () => {
|
||||
@@ -448,10 +472,9 @@ describe("FrameLookupTable", () => {
|
||||
expect(table.getActiveFramePayloads(2.5).get("hero")?.frameIndex).toBe(45);
|
||||
});
|
||||
|
||||
it("holds the last frame at the clip end even when the source is shorter than the window", () => {
|
||||
// clip [0,5] with only 1s of source (30 @ 30fps). The mid-clip tail stays
|
||||
// blank (source exhausted), but t === end still holds the last frame to
|
||||
// match the runtime's inclusive visibility.
|
||||
it("holds the last frame across the tail when the source is shorter than the window", () => {
|
||||
// clip [0,5] with only 1s of source (30 @ 30fps). The authored slot is the
|
||||
// visibility contract, so the final source frame fills its remaining tail.
|
||||
const table = createFrameLookupTable(
|
||||
[
|
||||
{
|
||||
@@ -466,7 +489,7 @@ describe("FrameLookupTable", () => {
|
||||
],
|
||||
[fakeExtracted(30, 30)],
|
||||
);
|
||||
expect(table.getActiveFramePayloads(1.5).has("hero")).toBe(false);
|
||||
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
|
||||
expect(table.getActiveFramePayloads(5.0).get("hero")?.frameIndex).toBe(29);
|
||||
});
|
||||
|
||||
@@ -1529,4 +1552,9 @@ describe("getFrameAtTime — IEEE 754 boundary precision", () => {
|
||||
const frame = getFrameAtTime(extracted, 0, 0, false, 1.0);
|
||||
expect(frame).toBe("frame-0.jpg");
|
||||
});
|
||||
|
||||
it("returns null after source exhaustion without an authored slot boundary", () => {
|
||||
const extracted = makeExtracted(25, 25);
|
||||
expect(getFrameAtTime(extracted, 3, 0)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1235,13 +1235,14 @@ export async function extractAllVideoFrames(
|
||||
};
|
||||
}
|
||||
|
||||
export function getFrameAtTime(
|
||||
function getFrameIndexAtTime(
|
||||
extracted: ExtractedFrames,
|
||||
globalTime: number,
|
||||
videoStart: number,
|
||||
loop = false,
|
||||
mediaStart = 0,
|
||||
): string | null {
|
||||
holdLastFrame = false,
|
||||
): number | null {
|
||||
let localTime = globalTime - videoStart;
|
||||
if (localTime < 0) return null;
|
||||
const loopDuration = Math.max(0, extracted.metadata.durationSeconds - mediaStart);
|
||||
@@ -1251,21 +1252,29 @@ export function getFrameAtTime(
|
||||
// Add epsilon before flooring to avoid IEEE 754 boundary errors where
|
||||
// e.g. 0.28 * 25 === 6.999999999999999 instead of 7.
|
||||
const frameIndex = Math.floor(localTime * extracted.fps + 1e-9);
|
||||
if (loop && frameIndex >= extracted.totalFrames && extracted.totalFrames > 0) {
|
||||
return extracted.framePaths.get(extracted.totalFrames - 1) || null;
|
||||
if (frameIndex < 0 || extracted.totalFrames <= 0) return null;
|
||||
if (frameIndex >= extracted.totalFrames) {
|
||||
return loop || holdLastFrame ? extracted.totalFrames - 1 : null;
|
||||
}
|
||||
if (frameIndex < 0 || frameIndex >= extracted.totalFrames) return null;
|
||||
return extracted.framePaths.get(frameIndex) || null;
|
||||
return frameIndex;
|
||||
}
|
||||
|
||||
const HOLD_LAST_FRAME_TOLERANCE_FRAMES = 2;
|
||||
export function getFrameAtTime(
|
||||
extracted: ExtractedFrames,
|
||||
globalTime: number,
|
||||
videoStart: number,
|
||||
loop = false,
|
||||
mediaStart = 0,
|
||||
): string | null {
|
||||
const frameIndex = getFrameIndexAtTime(extracted, globalTime, videoStart, loop, mediaStart);
|
||||
return frameIndex == null ? null : extracted.framePaths.get(frameIndex) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a clip's source is shorter than its `data-duration` slot by more than
|
||||
* the compiler tolerates before clamping the slot to the media
|
||||
* (MEDIA_DURATION_CLAMP_EPSILON_SECONDS) — the case worth warning about. Shared
|
||||
* by the render and `validate` warnings. `null` when the media covers the slot,
|
||||
* the clip loops, or inputs are unusable.
|
||||
* Whether a media source is shorter than its `data-duration` slot by more than
|
||||
* the compiler tolerance. The calculation stays tag-agnostic; current in-repo
|
||||
* warnings call it for audio only because video slots may intentionally outlive
|
||||
* their source and hold the final frame.
|
||||
*/
|
||||
export function analyzeClipMediaFit(params: {
|
||||
/** Timeline slot length in seconds — `end - start` (a.k.a. data-duration). */
|
||||
@@ -1325,7 +1334,15 @@ export class FrameLookupTable {
|
||||
const video = this.videos.get(videoId);
|
||||
if (!video) return null;
|
||||
if (globalTime < video.start || globalTime > video.end) return null;
|
||||
return getFrameAtTime(video.extracted, globalTime, video.start, video.loop, video.mediaStart);
|
||||
const frameIndex = getFrameIndexAtTime(
|
||||
video.extracted,
|
||||
globalTime,
|
||||
video.start,
|
||||
video.loop,
|
||||
video.mediaStart,
|
||||
true,
|
||||
);
|
||||
return frameIndex == null ? null : video.extracted.framePaths.get(frameIndex) || null;
|
||||
}
|
||||
|
||||
private resetActiveState(): void {
|
||||
@@ -1386,37 +1403,15 @@ export class FrameLookupTable {
|
||||
for (const videoId of this.activeVideoIds) {
|
||||
const video = this.videos.get(videoId);
|
||||
if (!video) continue;
|
||||
let localTime = globalTime - video.start;
|
||||
const loopDuration = Math.max(0, video.extracted.metadata.durationSeconds - video.mediaStart);
|
||||
if (video.loop && loopDuration > 0 && localTime >= loopDuration) {
|
||||
localTime %= loopDuration;
|
||||
}
|
||||
const frameIndex = Math.floor(localTime * video.extracted.fps + 1e-9);
|
||||
if (video.loop && frameIndex >= video.extracted.totalFrames) {
|
||||
const framePath = video.extracted.framePaths.get(video.extracted.totalFrames - 1);
|
||||
if (framePath) {
|
||||
frames.set(videoId, { framePath, frameIndex: video.extracted.totalFrames - 1 });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (frameIndex < 0 || frameIndex >= video.extracted.totalFrames) {
|
||||
// Source exhausted. Hold the last frame near the clip end so a media that
|
||||
// falls a hair short of its slot (e.g. `ffmpeg -t 1.45` → 1.433s at 30fps)
|
||||
// doesn't flash the background for one frame. A clip that's substantially
|
||||
// shorter than its slot still blanks for the tail. Tolerance floored at
|
||||
// the clamp epsilon so the seam is covered at any fps (see that const).
|
||||
const fps = video.extracted.fps;
|
||||
const holdTolerance = Math.max(
|
||||
fps > 0 ? HOLD_LAST_FRAME_TOLERANCE_FRAMES / fps : 0,
|
||||
MEDIA_DURATION_CLAMP_EPSILON_SECONDS,
|
||||
);
|
||||
if (globalTime >= video.end - holdTolerance && video.extracted.totalFrames > 0) {
|
||||
const lastIndex = video.extracted.totalFrames - 1;
|
||||
const lastPath = video.extracted.framePaths.get(lastIndex);
|
||||
if (lastPath) frames.set(videoId, { framePath: lastPath, frameIndex: lastIndex });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const frameIndex = getFrameIndexAtTime(
|
||||
video.extracted,
|
||||
globalTime,
|
||||
video.start,
|
||||
video.loop,
|
||||
video.mediaStart,
|
||||
true,
|
||||
);
|
||||
if (frameIndex == null) continue;
|
||||
const framePath = video.extracted.framePaths.get(frameIndex);
|
||||
if (!framePath) continue;
|
||||
frames.set(videoId, { framePath, frameIndex });
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
injectDurations,
|
||||
extractResolvedMedia,
|
||||
clampDurations,
|
||||
shouldClampMediaDuration,
|
||||
shouldClampResolvedMediaDuration,
|
||||
CSS_URL_RE,
|
||||
isNonRelativeUrl,
|
||||
type ResolvedDuration,
|
||||
@@ -478,7 +478,8 @@ async function compileHtmlFile(
|
||||
let compiledHtml =
|
||||
resolutions.length > 0 ? injectDurations(staticCompiled, resolutions) : staticCompiled;
|
||||
|
||||
// Phase 2: Validate pre-resolved media — clamp data-duration to actual source duration (parallel ffprobe)
|
||||
// Phase 2: Bound authored audio to playable source (parallel ffprobe).
|
||||
// Explicit video slots may outlive their source and hold the final frame.
|
||||
const preResolved = extractResolvedMedia(compiledHtml);
|
||||
const clampResults = await Promise.all(
|
||||
preResolved
|
||||
@@ -496,16 +497,17 @@ async function compileHtmlFile(
|
||||
);
|
||||
const clampList: ResolvedDuration[] = [];
|
||||
for (const r of clampResults) {
|
||||
if (r.maxDuration > 0 && shouldClampMediaDuration(r.duration, r.maxDuration)) {
|
||||
if (
|
||||
r.maxDuration > 0 &&
|
||||
shouldClampResolvedMediaDuration(r.tagName, r.duration, r.maxDuration)
|
||||
) {
|
||||
clampList.push({ id: r.id, duration: r.maxDuration });
|
||||
// This clip's `data-duration` is being silently shortened to its source.
|
||||
// Surface it so the author can confirm the longer slot wasn't intended.
|
||||
// ponytail: top-level only — sub-composition clips still get clamped (and
|
||||
// videos still hold the last frame); thread `log` through
|
||||
// parseSubCompositions to warn for them too.
|
||||
const kind = r.tagName === "audio" ? "Audio" : "Video";
|
||||
// ponytail: top-level only — sub-composition audio still gets clamped;
|
||||
// thread `log` through parseSubCompositions to warn for it too.
|
||||
log?.warn(
|
||||
`[compile] ${kind} "${r.id}" (${r.src}) is ${r.maxDuration.toFixed(2)}s but its ` +
|
||||
`[compile] Audio "${r.id}" (${r.src}) is ${r.maxDuration.toFixed(2)}s but its ` +
|
||||
`data-duration is ${r.duration.toFixed(2)}s — the slot is shortened to the media ` +
|
||||
`length. Set data-duration to ~${r.maxDuration.toFixed(2)}s, trim data-media-start, ` +
|
||||
`or use a longer/looping source if that isn't intended.`,
|
||||
|
||||
@@ -92,7 +92,6 @@
|
||||
id="a-roll-video"
|
||||
src="https://gen-os-static.s3.us-east-2.amazonaws.com/astral_assets/uploaded_assets/61ae9527_715ce519da784c0daae4f3aade82ee56.mp4"
|
||||
data-start="0"
|
||||
data-duration="14"
|
||||
data-track-index="2"
|
||||
></video>
|
||||
</div>
|
||||
|
||||
@@ -54,7 +54,6 @@
|
||||
id="aroll-video"
|
||||
src="https://gen-os-static.s3.us-east-2.amazonaws.com/astral_assets/uploaded_assets/f82d6e98_9076b56b449e44ac95a2099375863f91.mp4"
|
||||
data-start="0"
|
||||
data-duration="16"
|
||||
data-track-index="1"
|
||||
></video>
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
id="aroll-video"
|
||||
src="https://gen-os-static.s3.us-east-2.amazonaws.com/astral_assets/uploaded_assets/c10d5d5c_5e0cbe68f361481f8d2c77b028e84504.mp4"
|
||||
data-start="0"
|
||||
data-duration="14"
|
||||
data-track-index="0"
|
||||
crossorigin="anonymous"
|
||||
></video>
|
||||
|
||||
@@ -87,7 +87,6 @@
|
||||
id="a-roll"
|
||||
class="full-screen"
|
||||
data-start="0"
|
||||
data-duration="14"
|
||||
data-track-index="1"
|
||||
src="https://gen-os-static.s3.us-east-2.amazonaws.com/astral_assets/uploaded_assets/1dd17e4d_2c60a8b3482744e29584975d2b84164a.mp4"
|
||||
></video>
|
||||
|
||||
@@ -130,7 +130,6 @@
|
||||
id="aroll"
|
||||
src="https://gen-os-static.s3.us-east-2.amazonaws.com/astral_assets/uploaded_assets/57704cd3_8424a748578a497084fd1278949184fc.mp4"
|
||||
data-start="0"
|
||||
data-duration="17"
|
||||
data-track-index="1"
|
||||
></video>
|
||||
</div>
|
||||
|
||||
@@ -127,7 +127,6 @@
|
||||
id="a-roll"
|
||||
src="https://gen-os-static.s3.us-east-2.amazonaws.com/astral_assets/uploaded_assets/2111e840_0579531a2ffe4c3ea8295b42051fb394.mp4"
|
||||
data-start="0"
|
||||
data-duration="17"
|
||||
data-track-index="0"
|
||||
></video>
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
id="aroll"
|
||||
src="https://gen-os-static.s3.us-east-2.amazonaws.com/astral_assets/uploaded_assets/70648f53_abe84fb9991841ae8ba82ee21ba6d37e.mp4"
|
||||
data-start="0"
|
||||
data-duration="16.7"
|
||||
data-track-index="1"
|
||||
muted
|
||||
crossorigin="anonymous"
|
||||
|
||||
Reference in New Issue
Block a user