ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-04 16:42:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(video): hold final frame through composition

Browse Source
This commit is contained in:
Miguel Ángel
2026-07-16 02:15:01 +00:00
parent 71d7be7921
commit 2e8f871bc8
15 changed files with 352 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/engine/src/services/videoFrameExtractor.test.ts
+35 -7
View File
@@ -389,7 +389,7 @@ describe("FrameLookupTable", () => {
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
});
it("does not hold stale frames for non-looping clips after extracted frames end", () => {
it("holds the last frame for a non-looping clip until its authored slot ends", () => {
const table = createFrameLookupTable(
[
{
@@ -406,7 +406,31 @@ describe("FrameLookupTable", () => {
);
expect(table.getActiveFramePayloads(0.5).has("hero")).toBe(true);
expect(table.getActiveFramePayloads(1.5).has("hero")).toBe(false);
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(29);
expect(table.getFrame("hero", 4.5)).toBeTruthy();
expect(table.getActiveFramePayloads(5.1).has("hero")).toBe(false);
expect(table.getFrame("hero", 5.1)).toBeNull();
});
it("does not invent a held frame when extraction produced no frames", () => {
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.webm",
start: 0,
end: 5,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
[fakeExtracted(0, 30)],
);
expect(table.getActiveFramePayloads(4.5).has("hero")).toBe(false);
expect(table.getFrame("hero", 4.5)).toBeNull();
});
it("places a relative-reference video in its resolved window end-to-end (was blank)", () => {
@@ -448,10 +472,9 @@ describe("FrameLookupTable", () => {
expect(table.getActiveFramePayloads(2.5).get("hero")?.frameIndex).toBe(45);
});
it("holds the last frame at the clip end even when the source is shorter than the window", () => {
// clip [0,5] with only 1s of source (30 @ 30fps). The mid-clip tail stays
// blank (source exhausted), but t === end still holds the last frame to
// match the runtime's inclusive visibility.
it("holds the last frame across the tail when the source is shorter than the window", () => {
// clip [0,5] with only 1s of source (30 @ 30fps). The authored slot is the
// visibility contract, so the final source frame fills its remaining tail.
const table = createFrameLookupTable(
[
{
@@ -466,7 +489,7 @@ describe("FrameLookupTable", () => {
],
[fakeExtracted(30, 30)],
);
expect(table.getActiveFramePayloads(1.5).has("hero")).toBe(false);
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
expect(table.getActiveFramePayloads(5.0).get("hero")?.frameIndex).toBe(29);
});
@@ -1529,4 +1552,9 @@ describe("getFrameAtTime — IEEE 754 boundary precision", () => {
const frame = getFrameAtTime(extracted, 0, 0, false, 1.0);
expect(frame).toBe("frame-0.jpg");
});
it("returns null after source exhaustion without an authored slot boundary", () => {
const extracted = makeExtracted(25, 25);
expect(getFrameAtTime(extracted, 3, 0)).toBeNull();
});
});
packages/engine/src/services/videoFrameExtractor.ts
+39 -44
View File
@@ -1235,13 +1235,14 @@ export async function extractAllVideoFrames(
};
}
export function getFrameAtTime(
function getFrameIndexAtTime(
extracted: ExtractedFrames,
globalTime: number,
videoStart: number,
loop = false,
mediaStart = 0,
): string | null {
holdLastFrame = false,
): number | null {
let localTime = globalTime - videoStart;
if (localTime < 0) return null;
const loopDuration = Math.max(0, extracted.metadata.durationSeconds - mediaStart);
@@ -1251,21 +1252,29 @@ export function getFrameAtTime(
// Add epsilon before flooring to avoid IEEE 754 boundary errors where
// e.g. 0.28 * 25 === 6.999999999999999 instead of 7.
const frameIndex = Math.floor(localTime * extracted.fps + 1e-9);
if (loop && frameIndex >= extracted.totalFrames && extracted.totalFrames > 0) {
return extracted.framePaths.get(extracted.totalFrames - 1) || null;
if (frameIndex < 0 || extracted.totalFrames <= 0) return null;
if (frameIndex >= extracted.totalFrames) {
return loop || holdLastFrame ? extracted.totalFrames - 1 : null;
}
if (frameIndex < 0 || frameIndex >= extracted.totalFrames) return null;
return extracted.framePaths.get(frameIndex) || null;
return frameIndex;
}
const HOLD_LAST_FRAME_TOLERANCE_FRAMES = 2;
export function getFrameAtTime(
extracted: ExtractedFrames,
globalTime: number,
videoStart: number,
loop = false,
mediaStart = 0,
): string | null {
const frameIndex = getFrameIndexAtTime(extracted, globalTime, videoStart, loop, mediaStart);
return frameIndex == null ? null : extracted.framePaths.get(frameIndex) || null;
}
/**
* Whether a clip's source is shorter than its `data-duration` slot by more than
* the compiler tolerates before clamping the slot to the media
* (MEDIA_DURATION_CLAMP_EPSILON_SECONDS) — the case worth warning about. Shared
* by the render and `validate` warnings. `null` when the media covers the slot,
* the clip loops, or inputs are unusable.
* Whether a media source is shorter than its `data-duration` slot by more than
* the compiler tolerance. The calculation stays tag-agnostic; current in-repo
* warnings call it for audio only because video slots may intentionally outlive
* their source and hold the final frame.
*/
export function analyzeClipMediaFit(params: {
/** Timeline slot length in seconds — `end - start` (a.k.a. data-duration). */
@@ -1325,7 +1334,15 @@ export class FrameLookupTable {
const video = this.videos.get(videoId);
if (!video) return null;
if (globalTime < video.start || globalTime > video.end) return null;
return getFrameAtTime(video.extracted, globalTime, video.start, video.loop, video.mediaStart);
const frameIndex = getFrameIndexAtTime(
video.extracted,
globalTime,
video.start,
video.loop,
video.mediaStart,
true,
);
return frameIndex == null ? null : video.extracted.framePaths.get(frameIndex) || null;
}
private resetActiveState(): void {
@@ -1386,37 +1403,15 @@ export class FrameLookupTable {
for (const videoId of this.activeVideoIds) {
const video = this.videos.get(videoId);
if (!video) continue;
let localTime = globalTime - video.start;
const loopDuration = Math.max(0, video.extracted.metadata.durationSeconds - video.mediaStart);
if (video.loop && loopDuration > 0 && localTime >= loopDuration) {
localTime %= loopDuration;
}
const frameIndex = Math.floor(localTime * video.extracted.fps + 1e-9);
if (video.loop && frameIndex >= video.extracted.totalFrames) {
const framePath = video.extracted.framePaths.get(video.extracted.totalFrames - 1);
if (framePath) {
frames.set(videoId, { framePath, frameIndex: video.extracted.totalFrames - 1 });
}
continue;
}
if (frameIndex < 0 || frameIndex >= video.extracted.totalFrames) {
// Source exhausted. Hold the last frame near the clip end so a media that
// falls a hair short of its slot (e.g. `ffmpeg -t 1.45` → 1.433s at 30fps)
// doesn't flash the background for one frame. A clip that's substantially
// shorter than its slot still blanks for the tail. Tolerance floored at
// the clamp epsilon so the seam is covered at any fps (see that const).
const fps = video.extracted.fps;
const holdTolerance = Math.max(
fps > 0 ? HOLD_LAST_FRAME_TOLERANCE_FRAMES / fps : 0,
MEDIA_DURATION_CLAMP_EPSILON_SECONDS,
);
if (globalTime >= video.end - holdTolerance && video.extracted.totalFrames > 0) {
const lastIndex = video.extracted.totalFrames - 1;
const lastPath = video.extracted.framePaths.get(lastIndex);
if (lastPath) frames.set(videoId, { framePath: lastPath, frameIndex: lastIndex });
}
continue;
}
const frameIndex = getFrameIndexAtTime(
video.extracted,
globalTime,
video.start,
video.loop,
video.mediaStart,
true,
);
if (frameIndex == null) continue;
const framePath = video.extracted.framePaths.get(frameIndex);
if (!framePath) continue;
frames.set(videoId, { framePath, frameIndex });