ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-10-30 09:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): a video after a cut plays in sync instead of about 70 ms behind (#6494)

Browse Source 
* fix(core): a video after a cut starts early by its measured start delay, so it plays in sync

* test(player): playbench reports how far the picture after a cut runs behind the playhead

* fix(core): a looping video no longer skews the learned video start delay

* fix(core): the video start delay is learned in less code and capped at 100 ms under test
This commit is contained in:
Miguel Ángel
2026-10-27 06:09:05 +00:00
committed by GitHub
parent 0fb9a50a63
commit 594df21537
5 changed files with 180 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/runtime/media.test.ts
+100
View File
@@ -7,6 +7,7 @@ import {
readElementPlaybackRate,
readElementPlaybackStart,
refreshRuntimeMediaCache,
resetVideoStartLatency,
resolveRuntimeMediaClipDuration,
syncRuntimeMedia as syncRuntimeMediaWithDuration,
} from "./media";
@@ -2052,6 +2053,105 @@ describe("syncRuntimeMedia", () => {
}
expect(el.pause).not.toHaveBeenCalled();
});
describe("start latency", () => {
afterEach(() => resetVideoStartLatency());
// Plays `clip` at `playAt`, then has its element `lag` s behind the playhead at `sampleAt`.
const startWithLag = (
clip: RuntimeMediaClip,
playAt: number,
sampleAt: number,
lag: number,
) => {
tick(clip, playAt);
expect(clip.el.play).toHaveBeenCalledTimes(1);
Object.defineProperty(clip.el, "paused", {
value: false,
writable: true,
configurable: true,
});
clip.el.currentTime = clip.mediaStart + (sampleAt - clip.start) - lag;
tick(clip, sampleAt);
};
it("does not start an upcoming video early before any start is measured", () => {
const clip = videoClip({ start: 10, end: 20, mediaStart: 0 });
tick(clip, 9.99);
expect(clip.el.play).not.toHaveBeenCalled();
});
it("starts an upcoming video early by the start latency measured on earlier ones", () => {
startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.06);
const due50ms = videoClip({ start: 10, end: 20, mediaStart: 0 });
tick(due50ms, 9.95);
expect(due50ms.el.play).toHaveBeenCalledTimes(1);
const due70ms = videoClip({ start: 10, end: 20, mediaStart: 0 });
tick(due70ms, 9.93);
expect(due70ms.el.play).not.toHaveBeenCalled();
});
it("starts an upcoming video on the tick nearest its lead", () => {
startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.06);
const due75ms = videoClip({ start: 10, end: 20, mediaStart: 0 });
// Now is 15 ms earlier than the 60 ms lead; the next tick, 40 ms on, would be 25 ms late.
syncRuntimeMedia({
clips: [due75ms],
timeSeconds: 9.925,
playing: true,
playbackRate: 1,
cueAheadSeconds: 0.04,
getCompositionDuration: () => 30,
});
expect(due75ms.el.play).toHaveBeenCalledTimes(1);
});
it("learns from a video seeked and started in the same tick", () => {
const seeked = videoClip({ start: 0, end: 10, mediaStart: 0 });
seeked.el.currentTime = 0;
startWithLag(seeked, 5, 5.3, 0.06);
const due25ms = videoClip({ start: 10, end: 20, mediaStart: 0 });
tick(due25ms, 9.975);
expect(due25ms.el.play).toHaveBeenCalledTimes(1);
});
it("keeps its lead when an early start lands on time", () => {
startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.06);
startWithLag(videoClip({ start: 10, end: 20, mediaStart: 0 }), 9.945, 10.3, 0.005);
const due58ms = videoClip({ start: 20, end: 30, mediaStart: 0 });
tick(due58ms, 19.942);
expect(due58ms.el.play).toHaveBeenCalledTimes(1);
});
it("caps the learned start delay at 100 ms", () => {
startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.3);
const due150ms = videoClip({ start: 10, end: 20, mediaStart: 0 });
tick(due150ms, 9.85);
expect(due150ms.el.play).not.toHaveBeenCalled();
});
it("does not learn from a looping video sampled across a lap wrap", () => {
startWithLag(videoClip({ start: 0, end: 10, mediaStart: 0 }), 0, 0.3, 0.06);
const loop = videoClip({
start: 10,
end: 20,
mediaStart: 0,
loop: true,
sourceDuration: 0.2,
});
tick(loop, 10);
Object.defineProperty(loop.el, "paused", {
value: false,
writable: true,
configurable: true,
});
loop.el.currentTime = 0.19; // the playhead has wrapped to 0.06, the element not yet
tick(loop, 10.26);
const due50ms = videoClip({ start: 30, end: 40, mediaStart: 0 });
tick(due50ms, 29.95);
expect(due50ms.el.play).toHaveBeenCalledTimes(1);
});
});
});
describe("playing media drift", () => {
packages/core/src/runtime/media.ts
+38 -5
View File
@@ -236,6 +236,7 @@ export function evictMediaSyncState(el: HTMLMediaElement): void {
lastRuntimeAppliedVolume.delete(el);
forgetSteering(el);
heldAhead.delete(el);
videoStarts.delete(el);
}
/** Test-only seam: whether any per-source sync state is still tracked for `el`. */
@@ -302,6 +303,26 @@ const MIN_NATIVE_PLAYBACK_RATE = 1 / 16;
const VIDEO_LEAD_HOLD_SECONDS = 0.1;
const heldAhead = new WeakSet<HTMLMediaElement>();
// A muted video's clock is its audio track, which starts tens of ms after play(), so an upcoming
// video starts that early, hidden, by the delay learned from earlier starts.
let videoStartLatency: number | null = null;
const videoStarts = new WeakMap<HTMLMediaElement, { offset: number; lead: number; at: number }>();
/** Test-only seam. */
export function resetVideoStartLatency(): void {
videoStartLatency = null;
}
/** A quarter second in: its lag plus how early it started, capped at 0.1 s. */
function learnVideoStartLatency(el: HTMLMediaElement, time: number, offset: number, rate: number) {
const start = videoStarts.get(el);
if (!start || time < start.at) return;
videoStarts.delete(el);
const sample = (offset - start.offset) / rate + start.lead;
const averaged = videoStartLatency === null ? sample : (videoStartLatency + sample) / 2;
videoStartLatency = Math.min(0.1, Math.max(0, averaged));
}
// fallow-ignore-next-line complexity
export function syncRuntimeMedia(params: {
clips: RuntimeMediaClip[];
@@ -364,15 +385,18 @@ export function syncRuntimeMedia(params: {
const isNonLoopVideo = el.tagName === "VIDEO" && !clip.loop;
const inWindow = isInClipWindow(params.timeSeconds, clip.start, clip.end);
const dueIn = clip.start - params.timeSeconds;
const cueAhead =
el.tagName === "AUDIO"
? (params.cueAheadSeconds ?? 0)
: params.playing && videoStartLatency !== null
? // Half a tick: it starts on the tick nearest its lead.
videoStartLatency * params.playbackRate + (params.cueAheadSeconds ?? 0) / 2
: 0;
const startsEarly =
el.tagName === "AUDIO" &&
!inWindow &&
dueIn > 0 &&
dueIn * Math.max(1, rateAt(clipRate, clip.start - origin)) <=
Math.max(
params.cueAheadSeconds ?? 0,
startedEarly.has(el) ? MEDIA_SYNC_TOLERANCE_SECONDS : 0,
);
Math.max(cueAhead, startedEarly.has(el) ? MEDIA_SYNC_TOLERANCE_SECONDS : 0);
if (startsEarly) startedEarly.add(el);
else startedEarly.delete(el);
// A video that runs to the composition end stays the visible frame at and past it, so it
@@ -569,6 +593,7 @@ export function syncRuntimeMedia(params: {
}
const holdingAhead = heldAhead.has(el);
const isPlayingMedia = !el.paused && playing;
if (isPlayingMedia) learnVideoStartLatency(el, params.timeSeconds, offset, baseRate);
// Only apply strict sync when offset has stabilized (not growing).
// During initial buffering, offset grows ~16ms/tick as the timeline
// advances while media stays at 0. Accumulated drift from pause/play
@@ -641,6 +666,7 @@ export function syncRuntimeMedia(params: {
}
playRequested.delete(el);
heldAhead.delete(el);
videoStarts.delete(el);
} else if (!playing) {
holdSeekBarrierUntilVideoLands(el);
}
@@ -666,6 +692,13 @@ export function syncRuntimeMedia(params: {
// fire between `load()` and `addEventListener` attachment, wedging
// the element waiting for a callback that never came.
markPlayRequested(el);
// Offset read after any seek above; a loop could wrap a lap before the sample.
if (isNonLoopVideo)
videoStarts.set(el, {
offset: relTime - (el.currentTime || 0),
lead: Math.max(0, clip.start - params.timeSeconds) / params.playbackRate,
at: Math.max(params.timeSeconds, clip.start) + 0.25 * params.playbackRate,
});
void el.play().catch((err: unknown) => {
// If play() rejects — e.g. autoplay blocked, element removed
// mid-flight — drop the in-flight flag so a future sync tick can
packages/player/tests/playbench/analyze.ts
+11
View File
@@ -46,6 +46,9 @@ export type Metrics = {
/** Problem 4 */
secondFrameGapMedianMs: number | null;
secondFrameGapMaxMs: number | null;
/** How far the incoming picture sits behind the playhead over its take's first 0.5 s, median per cut. */
cutPictureLagMedianMs: number | null;
cutPictureLagMaxMs: number | null;
playheadHoldMedianMs: number | null;
playheadHoldP90Ms: number | null;
holdsOver100: number;
@@ -222,7 +225,13 @@ export function analyze(run: Run): Metrics {
// ---- Problem 4: what a cut costs
const gaps: number[] = [];
const holds: number[] = [];
const lags: number[] = [];
for (const t of takes.slice(1)) {
const opening = byId(samples, `v${t.index}`).filter(
(s) => s[3] === 0 && s[7] !== null && s[7] >= t.start && s[7] <= t.start + 0.5,
);
const lag = median(opening.map((s) => (t.inPoint + s[7]! - t.start - s[2]) * 1000));
if (lag !== null) lags.push(lag);
const fr = comp.frames
.filter((f) => f[1] === `v${t.index}` && f[4] === 1)
.sort((a, b) => a[0] - b[0]);
@@ -234,6 +243,8 @@ export function analyze(run: Run): Metrics {
m.cuts = takes.length - 1;
m.secondFrameGapMedianMs = round(median(gaps));
m.secondFrameGapMaxMs = round(gaps.length ? Math.max(...gaps) : null);
m.cutPictureLagMedianMs = round(median(lags));
m.cutPictureLagMaxMs = round(lags.length ? Math.max(...lags) : null);
m.playheadHoldMedianMs = round(median(holds));
m.playheadHoldP90Ms = round(pct(holds, 0.9));
m.holdsOver100 = holds.filter((h) => h > 100).length;
packages/player/tests/playbench/playbench.test.ts
+21
View File
@@ -121,6 +121,27 @@ describe("analyze", () => {
expect(analyze(run).playheadHoldMedianMs).toBeCloseTo(30, 0);
});
it("reports an incoming picture that runs 50 ms behind the playhead after its cut", () => {
const run = base();
for (const t of edit.takes.slice(1))
for (let i = 0; i <= 50; i++) {
const ph = t.start + i * 0.01;
run.comp.samples.push([
1000 + ph * 1000,
`v${t.index}`,
t.inPoint + ph - t.start - 0.05,
0,
0,
1,
1,
ph,
]);
}
const m = analyze(run);
expect(m.cutPictureLagMedianMs).toBeCloseTo(50, 0);
expect(m.cutPictureLagMaxMs).toBeCloseTo(50, 0);
});
it("counts a take as heard only when the audio reaching the output comes from that take's stretch of the file", () => {
const run = base();
run.ownership = [[1000, "parent"]];
packages/player/tests/playbench/run.ts
+10
View File
@@ -229,6 +229,16 @@ const ROWS: [string, string, (m: Metrics) => number | boolean | null][] = [
["4", "cuts", (m) => m.cuts],
["4", "incoming second-frame gap median (ms)", (m) => m.secondFrameGapMedianMs],
["4", "incoming second-frame gap max (ms)", (m) => m.secondFrameGapMaxMs],
[
"4",
"incoming picture behind the playhead, first 0.5 s, median over cuts (ms)",
(m) => m.cutPictureLagMedianMs,
],
[
"4",
"incoming picture behind the playhead, first 0.5 s, worst cut (ms)",
(m) => m.cutPictureLagMaxMs,
],
["4", "playhead hold per cut median (ms)", (m) => m.playheadHoldMedianMs],
["4", "playhead hold per cut p90 (ms)", (m) => m.playheadHoldP90Ms],
["4", "cuts with a hold over 100 ms", (m) => m.holdsOver100],