fix(core): clips that meet within float rounding count as meeting in edits and playback (#4535)

* fix(core): clips that meet within float rounding count as meeting in edits and playback

* fix(engine): hand a shared clip boundary to the incoming clip in export, as preview does
This commit is contained in:
Miguel Ángel
2026-09-26 22:14:14 +00:00
committed by GitHub
parent 0ae223773a
commit 984ad28c7c
20 changed files with 610 additions and 101 deletions
+48 -13
View File
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication
import { describe, expect, it, vi } from "vitest";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
@@ -147,7 +148,7 @@ describe("snapshot lint preflight", () => {
});
describe("resolveSnapshotVideoFrameTime", () => {
it("keeps media active at the inclusive clip end and samples its last decodable frame", () => {
it("holds a clip ending with the composition on its last decodable frame", () => {
expect(
resolveSnapshotVideoFrameTime({
globalTime: 15,
@@ -155,10 +156,30 @@ describe("resolveSnapshotVideoFrameTime", () => {
clipDuration: 15,
relativeTime: 15,
sourceDuration: 15,
compositionDuration: 15,
}),
).toBeCloseTo(15 - 1 / 30, 6);
});
it.each([
[0.3, 0.1 + 0.2],
[26.2, 19.8 + 6.4],
])(
"samples the first frame of a clip starting on a float sum at %s, as the preview does",
(globalTime, clipStart) => {
expect(
resolveSnapshotVideoFrameTime({
globalTime,
clipStart,
clipDuration: 0.2,
relativeTime: globalTime - clipStart,
sourceDuration: 10,
compositionDuration: 1,
}),
).toBe(0);
},
);
it("keeps ordinary in-window media timestamps unchanged", () => {
expect(
resolveSnapshotVideoFrameTime({
@@ -167,21 +188,30 @@ describe("resolveSnapshotVideoFrameTime", () => {
clipDuration: 15,
relativeTime: 7.5,
sourceDuration: 15,
compositionDuration: 15,
}),
).toBe(7.5);
});
it("does not activate media after the clip end", () => {
expect(
resolveSnapshotVideoFrameTime({
globalTime: 15.001,
clipStart: 0,
clipDuration: 15,
relativeTime: 15.001,
sourceDuration: 15,
}),
).toBeNull();
});
it.each([
[15.001, 0, 15],
[15, 0, 15],
[0.3, 0.1, 0.1 + 0.2],
])(
"leaves a clip that ends before the composition does at %s",
(globalTime, clipStart, clipEnd) => {
expect(
resolveSnapshotVideoFrameTime({
globalTime,
clipStart,
clipDuration: clipEnd - clipStart,
relativeTime: globalTime - clipStart,
sourceDuration: 15,
compositionDuration: 30,
}),
).toBeNull();
},
);
it.each([
{
@@ -192,6 +222,7 @@ describe("resolveSnapshotVideoFrameTime", () => {
clipDuration: 10,
relativeTime: 0,
sourceDuration: 10,
compositionDuration: 15,
},
expected: null,
},
@@ -203,6 +234,7 @@ describe("resolveSnapshotVideoFrameTime", () => {
clipDuration: 10,
relativeTime: -0.1,
sourceDuration: 10,
compositionDuration: 15,
},
expected: null,
},
@@ -214,17 +246,19 @@ describe("resolveSnapshotVideoFrameTime", () => {
clipDuration: 10,
relativeTime: 10,
sourceDuration: 0,
compositionDuration: 15,
},
expected: 10 - 1 / 30,
},
{
name: "offset clip inclusive end",
name: "offset clip held at the composition end",
input: {
globalTime: 15,
clipStart: 5,
clipDuration: 10,
relativeTime: 10,
sourceDuration: 10,
compositionDuration: 15,
},
expected: 10 - 1 / 30,
},
@@ -236,6 +270,7 @@ describe("resolveSnapshotVideoFrameTime", () => {
clipDuration: 10,
relativeTime: 10,
sourceDuration: 10,
compositionDuration: 15,
},
expected: 10 - 1 / 30,
},
+13 -14
View File
@@ -15,6 +15,8 @@ import {
type ZoomTarget,
} from "../capture/captureCompositionFrame.js";
import {
isClipVisibleAt,
isInClipWindow,
readElementRateSpec,
sourceTimeAt,
timeAtSourceTime,
@@ -80,28 +82,23 @@ export function formatSnapshotTimestamp(time: number): string {
return `${Number(time.toFixed(3))}s`;
}
/** Keep an exact clip-end snapshot aligned with the renderer's inclusive media
* window. This intentionally differs from the live player's exclusive-end
* visibility so an explicit end-boundary review does not become blank. FFmpeg
* cannot decode at a source's exclusive duration, so sample one nominal 30fps
* frame inside the source. This also clamps clips whose configured media window
* extends beyond the source. An infinite clip duration intentionally never
* enters the end-boundary branch. */
/** Shows media by the runtime's visibility rule. A clip held at the composition end samples one
* nominal 30fps frame inside its source, since FFmpeg cannot decode at a source's exclusive duration. */
export function resolveSnapshotVideoFrameTime(input: {
globalTime: number;
clipStart: number;
clipDuration: number;
relativeTime: number;
sourceDuration: number;
compositionDuration: number;
}): number | null {
const { globalTime, clipStart, clipDuration, relativeTime, sourceDuration } = input;
const { globalTime, clipStart, clipDuration, sourceDuration, compositionDuration } = input;
const clipEnd = clipStart + clipDuration;
const clipEndTolerance = 1e-9;
if (globalTime < clipStart || globalTime > clipEnd + clipEndTolerance || relativeTime < 0)
return null;
const atClipEnd = Math.abs(globalTime - clipEnd) <= clipEndTolerance;
if (!atClipEnd) return relativeTime;
if (!isClipVisibleAt(globalTime, clipStart, clipEnd, compositionDuration)) return null;
const relativeTime =
globalTime < clipStart ? Math.max(0, input.relativeTime) : input.relativeTime;
if (relativeTime < 0) return null;
if (isInClipWindow(globalTime, clipStart, clipEnd)) return relativeTime;
const sourceEnd = sourceDuration > 0 ? sourceDuration : relativeTime;
return Math.max(0, Math.min(relativeTime, sourceEnd - 1 / 30));
@@ -480,6 +477,7 @@ async function captureSnapshots(
};
});
});
const compositionDuration = duration;
const active = candidates.flatMap((candidate) => {
const start = resolveSnapshotVideoClipStart(candidate);
const playbackRate = resolveSnapshotVideoRateSpec(candidate);
@@ -508,6 +506,7 @@ async function captureSnapshots(
clipDuration: duration,
relativeTime: relTime,
sourceDuration: candidate.srcDuration,
compositionDuration,
});
return frameTime === null
? []
+22 -5
View File
@@ -6,6 +6,7 @@ import {
} from "@hyperframes/studio-server";
import type { AppliedFileMutation, PatchOperation } from "@hyperframes/studio-server";
import { fpsToNumber, parseFpsWithDefault } from "@hyperframes/core";
import { isInsideSpan, sameInstant, spansShareTime } from "@hyperframes/core/clip-facts";
import { readCompositionFps } from "../utils/compositionFps.js";
import { readFileSync } from "node:fs";
import type { ProjectTimeline, TimelineRow } from "./describeProject.js";
@@ -147,7 +148,7 @@ function overlap(
candidate !== row &&
candidate.file === row.file &&
candidate.trackIndex === row.trackIndex &&
Math.max(start, candidate.start) < Math.min(end, candidate.end),
spansShareTime(start, end, candidate.start, candidate.end),
);
}
@@ -255,6 +256,9 @@ function trimInput(args: Record<string, unknown>) {
};
}
const leavesNoLength = (start: number, duration: number) =>
duration <= 0 || sameInstant(start, start + duration);
function finishTrim(
context: MutationContext,
nextStart: number,
@@ -263,8 +267,9 @@ function finishTrim(
): MutationDecision | { ok: true; nextStart: number; nextDuration: number } {
if (end && !end.ok) return end;
if (duration && !duration.ok) return duration;
const nextDuration = duration?.seconds ?? (end ? end.seconds - nextStart : context.row.duration);
if (nextDuration <= 0) {
const nextDuration =
duration?.seconds ?? (end ? durationUntil(nextStart, end.seconds) : context.row.duration);
if (leavesNoLength(nextStart, nextDuration)) {
return {
ok: false,
reason: "trim duration must be positive",
@@ -279,6 +284,19 @@ function trimStart(context: MutationContext, expression: string | undefined) {
return parseMutationTime(context, expression, "pass a valid time expression");
}
function durationUntil(start: number, end: number): number {
let duration = end - start;
while (duration > 0 && start + duration > end) duration = nextSmaller(duration);
return duration;
}
const float = new DataView(new ArrayBuffer(8));
function nextSmaller(positive: number): number {
float.setFloat64(0, positive);
float.setBigUint64(0, float.getBigUint64(0) - 1n);
return float.getFloat64(0);
}
function trimEnd(context: MutationContext, expression: string | undefined) {
if (!expression) return undefined;
return parseMutationTime(context, expression, "pass a valid time expression");
@@ -413,8 +431,7 @@ export function mutationConflict(
(candidate) =>
candidate.file === row.file &&
candidate.trackIndex === row.trackIndex &&
candidate.start < nextStart &&
nextStart < candidate.end,
isInsideSpan(nextStart, candidate.start, candidate.end),
);
if (!conflict) return null;
return {
@@ -3,6 +3,7 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { isClipVisibleAt } from "@hyperframes/core";
import { describe, expect, it } from "vitest";
const cliEntry = resolve(fileURLToPath(import.meta.url), "..", "..", "cli.ts");
@@ -102,6 +103,135 @@ describe("timeline edit command", () => {
}
});
const meeting = (aStart: string, aDuration: string, bStart: string, fps = "") =>
`<div data-composition-id="main"${fps} data-duration="40"><div id="a" data-hf-id="a" data-start="${aStart}" data-duration="${aDuration}" data-track-index="0"></div><div id="b" data-hf-id="b" data-start="${bStart}" data-duration="2" data-track-index="0"></div></div>`;
const clip = (html: string, id: string) => {
const tag = new RegExp(`<div[^>]*\\sid="${id}"[^>]*>`).exec(html)?.[0] ?? "";
const attr = (name: string) => new RegExp(`${name}="([^"]+)"`).exec(tag)?.[1];
return { start: attr("data-start"), duration: attr("data-duration") };
};
const endOf = (html: string, id: string) =>
Number(clip(html, id).start) + Number(clip(html, id).duration);
const visibleAt = (html: string, time: number, ids: string[]) =>
ids.filter((id) => isClipVisibleAt(time, Number(clip(html, id).start), endOf(html, id), 40));
const composition = (fps: string, ...clips: [string, string, string][]) =>
`<div data-composition-id="main"${fps} data-duration="40">${clips
.map(
([id, start, duration]) =>
`<div id="${id}" data-hf-id="${id}" data-start="${start}" data-duration="${duration}" data-track-index="0"></div>`,
)
.join("")}</div>`;
it.each([
[
"move",
["#b", "20f"],
composition(' data-fps="24"', ["a", "0", String(20 / 24 + 5e-7)], ["b", "2", "0.25"]),
],
["duplicate", ["#c", "--at", "1"], composition("", ["a", "0", "1.0000005"], ["c", "3", "1"])],
])("refuses a %s that lands half a microsecond inside another clip", (verb, args, html) => {
const dir = project();
try {
writeFileSync(join(dir, "index.html"), html);
expect(run(dir, verb, ...args).status).not.toBe(0);
expect(readFileSync(join(dir, "index.html"), "utf8")).toBe(html);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it.each([
[
"a clip wholly before the insertion point",
["#x", "--at", "5"],
composition("", ["x", "0", "1"], ["t", "4.9999995", "0.0000002"]),
4.9999995,
],
[
"a clip half a microsecond after it",
["#a"],
composition("", ["a", "0", "1"], ["t", "1.0000005", "1"]),
1.0000005 + 1,
],
])("duplicates without snapping %s to the copy", (_, args, html, tStart) => {
const dir = project();
try {
writeFileSync(join(dir, "index.html"), html);
expect(run(dir, "duplicate", ...args).status).toBe(0);
expect(Number(clip(readFileSync(join(dir, "index.html"), "utf8"), "t").start)).toBe(tStart);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("trims a clip that only meets the one before it, ending where asked", () => {
const dir = project();
try {
writeFileSync(join(dir, "index.html"), meeting("19.8", "6.4", "26.2"));
expect(run(dir, "trim", "#b", "--end", "29").status).toBe(0);
const html = readFileSync(join(dir, "index.html"), "utf8");
expect(clip(html, "b")).toEqual({ start: "26.2", duration: "2.8000000000000007" });
expect(endOf(html, "b")).toBe(29);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("trims up to the next clip's start without ending past it, where no duration lands exactly", () => {
const dir = project();
try {
writeFileSync(join(dir, "index.html"), meeting("4.74", "10", "25.74"));
expect(run(dir, "trim", "#a", "--end", "25.74").status).toBe(0);
const html = readFileSync(join(dir, "index.html"), "utf8");
expect(endOf(html, "a")).toBeLessThanOrEqual(25.74);
expect(visibleAt(html, 25.74, ["a", "b"])).toEqual(["b"]);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("trims to frame 20 at 30 fps so that frame shows the next clip and not this one", () => {
const dir = project();
try {
writeFileSync(join(dir, "index.html"), meeting("0", "2", "2", ' data-fps="30"'));
expect(run(dir, "trim", "#a", "--end", "20f").status).toBe(0);
expect(run(dir, "trim", "#b", "--start", "20f").status).toBe(0);
const html = readFileSync(join(dir, "index.html"), "utf8");
expect([clip(html, "a").duration, clip(html, "b").start]).toEqual([
"0.6666666666666666",
"0.6666666666666666",
]);
expect(visibleAt(html, 19 / 30, ["a", "b"])).toEqual(["a"]);
expect(visibleAt(html, 20 / 30, ["a", "b"])).toEqual(["b"]);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it.each([
["19.8", "6.4", "26.2"],
["0", "0.6666666666666666", "0.6666666666666666"],
["0.1", "1.1", "1.2"],
])(
"duplicates a clip at %s lasting %s up against the clip at %s, each boundary exact",
(aStart, aDuration, bStart) => {
const dir = project();
try {
writeFileSync(join(dir, "index.html"), meeting(aStart, aDuration, bStart));
expect(run(dir, "duplicate", "#a").status).toBe(0);
const html = readFileSync(join(dir, "index.html"), "utf8");
const ids = ["a", "a-copy", "b"];
expect(Number(clip(html, "a-copy").start)).toBe(endOf(html, "a"));
expect(Number(clip(html, "b").start)).toBe(endOf(html, "a-copy"));
expect(visibleAt(html, endOf(html, "a"), ids)).toEqual(["a-copy"]);
expect(visibleAt(html, endOf(html, "a-copy"), ids)).toEqual(["b"]);
} finally {
rmSync(dir, { recursive: true, force: true });
}
},
);
it("refuses an ambiguous reference", () => {
const dir = project();
try {
@@ -257,6 +387,36 @@ describe("timeline edit command", () => {
}
});
it("refuses a trim that would leave a clip one float rounding step long", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-timeline-cli-"));
try {
const indexPath = join(dir, "index.html");
writeFileSync(indexPath, meeting("19.8", "6.4", "26.2"));
const result = run(dir, "trim", "#b", "--end", "26.200000000000003");
expect(result.status).toBe(2);
expect(result.stderr).toContain("trim duration must be positive");
expect(clip(readFileSync(indexPath, "utf8"), "b").duration).toBe("2");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("moves a clip over pending media, whose unknown length counts as zero", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-timeline-cli-"));
try {
const indexPath = join(dir, "index.html");
writeFileSync(
indexPath,
`<div data-composition-id="main" data-duration="12"><div id="a" data-hf-id="a" data-start="3" data-duration="1" data-track-index="0"></div><video id="pending" data-hf-id="pending" src="https://example.com/v.mp4" data-start="1" data-track-index="0"></video></div>`,
);
const result = run(dir, "move", "#a", "0.5");
expect(result.status, result.stderr).toBe(0);
expect(clip(readFileSync(indexPath, "utf8"), "a").start).toBe("0.5");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("refuses duplicate insertion inside a spanning clip", () => {
const dir = project();
try {
+59 -1
View File
@@ -1,5 +1,13 @@
import { describe, expect, it } from "vitest";
import { byStart, formatClipLine, type ClipFact } from "./clipFacts.js";
import {
byStart,
formatClipLine,
isInsideSpan,
sameInstant,
spansOverlap,
spansShareTime,
type ClipFact,
} from "./clipFacts.js";
const clip = (over: Partial<ClipFact> = {}): ClipFact => ({
id: "a",
@@ -60,3 +68,53 @@ describe("byStart", () => {
expect(rows.sort(byStart).map((r) => r.id)).toEqual(["a", "b", "c"]);
});
});
describe("spansOverlap", () => {
it("counts clips that only meet as apart, though the float sum of the first overshoots", () => {
expect(19.8 + 6.4).toBeGreaterThan(26.2);
expect(spansOverlap(19.8, 19.8 + 6.4, 26.2, 29)).toBe(false);
expect(spansOverlap(26.2, 29, 19.8, 19.8 + 6.4)).toBe(false);
});
it("counts clips that share time as overlapping, down to half a microsecond", () => {
expect(spansOverlap(0, 2, 1.9, 3)).toBe(true);
expect(spansOverlap(1, 2, 0, 5)).toBe(true);
expect(spansOverlap(0, 20 / 24 + 5e-7, 20 / 24, 20 / 24 + 0.25)).toBe(true);
});
it("counts a zero-length clip inside another, so the timeline gives it a lane of its own", () => {
expect(spansOverlap(1, 1, 0.5, 1.5)).toBe(true);
});
});
describe("spansShareTime", () => {
it("never counts a zero-length span, such as pending media, as sharing time", () => {
expect(spansShareTime(1, 1, 0.5, 1.5)).toBe(false);
expect(spansShareTime(0.5, 1.5, 1, 1)).toBe(false);
});
it("agrees with spansOverlap on spans with length, float sums included", () => {
expect(spansShareTime(19.8, 19.8 + 6.4, 26.2, 29)).toBe(false);
expect(spansShareTime(0, 20 / 24 + 5e-7, 20 / 24, 20 / 24 + 0.25)).toBe(true);
expect(spansShareTime(1, 2, 0, 5)).toBe(true);
});
});
describe("isInsideSpan", () => {
it("counts a time strictly inside, and not one on either edge within float rounding", () => {
expect(isInsideSpan(3, 1, 5)).toBe(true);
expect(isInsideSpan(26.2, 19.8, 19.8 + 6.4)).toBe(false);
expect(isInsideSpan(1, 1, 5)).toBe(false);
});
});
describe("sameInstant", () => {
it("joins times a float rounding step apart, and nothing wider", () => {
expect(sameInstant(19.8 + 6.4, 26.2)).toBe(true);
expect(sameInstant(0.1 + 1.1, 1.2)).toBe(true);
expect(sameInstant(3600.1 + 0.2, 3600.3)).toBe(true);
expect(sameInstant(20 / 24 + 5e-7, 20 / 24)).toBe(false);
expect(sameInstant(1.0000005, 1)).toBe(false);
expect([sameInstant(5, Infinity), sameInstant(Infinity, Infinity)]).toEqual([false, true]);
});
});
+27
View File
@@ -34,6 +34,33 @@ const roundTo3 = (n: number) => Math.round(n * 1000) / 1000;
const num = (n: number) => String(roundTo3(n));
export const instantTolerance = (time: number) => 4 * Number.EPSILON * Math.max(1, Math.abs(time));
/** Float sums like 19.8 + 6.4 miss 26.2 by a rounding step; a few steps, scaled to the time, are one instant. */
export function sameInstant(a: number, b: number): boolean {
if (!Number.isFinite(a) || !Number.isFinite(b)) return a === b;
return Math.abs(a - b) <= instantTolerance(Math.max(Math.abs(a), Math.abs(b)));
}
const isBefore = (a: number, b: number) => a < b && !sameInstant(a, b);
/** For timeline lanes: a zero-length clip inside another still counts, so it gets a lane of its own. */
export function spansOverlap(aStart: number, aEnd: number, bStart: number, bEnd: number): boolean {
return isBefore(aStart, bEnd) && isBefore(bStart, aEnd);
}
export function spansShareTime(
aStart: number,
aEnd: number,
bStart: number,
bEnd: number,
): boolean {
return isBefore(Math.max(aStart, bStart), Math.min(aEnd, bEnd));
}
export const isInsideSpan = (time: number, start: number, end: number) =>
isBefore(start, time) && isBefore(time, end);
export function formatClipLine(clip: ClipFact): string {
const parts = [
`${clip.kind} "${clip.id}"`,
+1 -1
View File
@@ -320,7 +320,7 @@ export {
// (verify:packed-manifests catches exactly that).
export { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
// Also exposed via the ./runtime/clip-window subpath; re-exported here for the same dist-emit reason.
export { isClipVisibleAt, isInClipWindow } from "./runtime/clipWindow.js";
export { hasClipStarted, isClipVisibleAt, isInClipWindow } from "./runtime/clipWindow.js";
export {
normalizePlaybackRate,
normalizeRateSpec,
@@ -5,6 +5,18 @@ describe("isInClipWindow", () => {
it("includes the start and excludes the end", () => {
expect([1, 1.5, 2].map((t) => isInClipWindow(t, 1, 2))).toEqual([true, true, false]);
});
it("hands the instant a float sum misses by a rounding step to the next clip only", () => {
expect([isInClipWindow(26.2, 19.8, 19.8 + 6.4), isInClipWindow(26.2, 26.2, 28.2)]).toEqual([
false,
true,
]);
expect(isInClipWindow(20 / 24, 0, 20 / 24 + 5e-7)).toBe(true);
});
it("keeps a clip with no known end in its window", () => {
expect(isInClipWindow(5, 0, Number.POSITIVE_INFINITY)).toBe(true);
});
});
describe("isClipVisibleAt", () => {
+8 -3
View File
@@ -1,6 +1,11 @@
/** Half-open: two back-to-back clips never both hold the shared boundary instant. */
import { sameInstant } from "../clipFacts";
export const hasClipStarted = (time: number, start: number) =>
time >= start || sameInstant(time, start);
/** Half-open: two back-to-back clips never both hold the shared boundary instant, float sums included. */
export const isInClipWindow = (time: number, start: number, end: number): boolean =>
time >= start && time < end;
hasClipStarted(time, start) && time < end && !sameInstant(time, end);
const TERMINAL_EPSILON_SECONDS = 1e-6;
@@ -15,6 +20,6 @@ export const isClipVisibleAt = (
compositionDuration: number,
): boolean =>
isInClipWindow(time, start, end) ||
(time >= start &&
(hasClipStarted(time, start) &&
compositionDuration > 0 &&
end >= compositionDuration - TERMINAL_EPSILON_SECONDS);
@@ -204,6 +204,26 @@ describe("per-seek media clip index", () => {
expect(mediaSpy.visits[0]!).toBe(16);
});
it("visits a clip whose float-sum start is the same instant as the seek, as the full pass does", () => {
const seekOntoFloatSumStart = (disableNarrowing: boolean) => {
mediaSpy.disableNarrowing = disableNarrowing;
document.body.innerHTML = `<div data-composition-id="main" data-root="true" data-start="0">
<video id="late" data-start="${19.8 + 0.1}" data-duration="5" data-playback-start="3"></video></div>`;
Object.defineProperty(document.querySelector("video"), "duration", { value: 10 });
window.__timelines = { main: createMockTimeline(30) };
initSandboxRuntimeModular();
window.__player!.seek(19.8);
window.__player!.seek(19.9);
const state = mediaState();
window.__hfRuntimeTeardown?.();
return state;
};
const indexed = seekOntoFloatSumStart(false);
expect(indexed).toEqual(seekOntoFloatSumStart(true));
expect(indexed[0]).toMatchObject({ currentTime: 3 });
});
it("re-derives the index when a clip is moved on the timeline", () => {
mountClips(16);
initSandboxRuntimeModular();
+6 -3
View File
@@ -1,5 +1,6 @@
// fallow-ignore-file code-duplication complexity
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
import { instantTolerance } from "../clipFacts";
import { isInClipWindow } from "./clipWindow";
import { revealTimedClipsAfterFirstPass } from "./timedClipHide";
import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics";
@@ -2742,7 +2743,7 @@ export function initSandboxRuntimeModular(): void {
* The media elements this pass must visit, and the argument that the rest can
* be skipped.
*
* `isActive` requires `start <= t < end`, so a clip whose window excludes the
* `isActive` requires `isInClipWindow`, so a clip whose window excludes the
* new time is inactive there no matter what its element state is. A clip that
* is in neither the previous in-window set nor the set of windows whose
* endpoint the transport just crossed was therefore out of window BEFORE and
@@ -2760,8 +2761,10 @@ export function initSandboxRuntimeModular(): void {
): Array<HTMLVideoElement | HTMLAudioElement> => {
const fromSeconds = lastSyncedMediaTimeSeconds;
if (fromSeconds === null) return index.clips.map((clip) => clip.el);
const lo = Math.min(fromSeconds, toSeconds);
const hi = Math.max(fromSeconds, toSeconds);
const sameInstantMargin =
2 * instantTolerance(Math.max(Math.abs(fromSeconds), Math.abs(toSeconds)));
const lo = Math.min(fromSeconds, toSeconds) - sameInstantMargin;
const hi = Math.max(fromSeconds, toSeconds) + sameInstantMargin;
const visiting = new Set<HTMLVideoElement | HTMLAudioElement>();
for (const clip of mediaClipsInWindow) visiting.add(clip.el);
for (const clip of clipsWithEndpointBetween(index.byStart, (c) => c.start, lo, hi)) {
+26
View File
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication
import { describe, it, expect, vi, afterEach } from "vitest";
import {
evictMediaSyncState,
@@ -376,6 +377,14 @@ describe("syncRuntimeMedia", () => {
document.body.innerHTML = "";
});
it("seeks a clip whose start is a float sum to its first frame, never before it", () => {
const clip = createMockClip({ start: 19.8 + 0.1, end: 25 });
Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
clip.el.currentTime = 3;
syncRuntimeMedia({ clips: [clip], timeSeconds: 19.9, playing: false, playbackRate: 1 });
expect(clip.el.currentTime).toBe(0);
});
describe("speed ramp", () => {
it("seeks to the integrated source time and plays at the instantaneous rate", () => {
const rate = {
@@ -1032,6 +1041,23 @@ describe("syncRuntimeMedia", () => {
expect(clip.el.play).not.toHaveBeenCalled();
});
it("holds a video ending on a float sum at the composition end on its last frame", () => {
const clip = createMockClip({
start: 19.8,
end: 19.8 + 6.4,
duration: 6.4,
sourceDuration: 10,
});
syncRuntimeMedia({
clips: [clip],
timeSeconds: 26.2,
playing: false,
playbackRate: 1,
getCompositionDuration: () => 26.2,
});
expect(clip.el.currentTime).toBeCloseTo(6.4, 9);
});
it("holds a speed-ramped video that runs to the composition end at the source time of its own end", () => {
const lane: HfAutomationLane = {
target: "rate",
+4 -2
View File
@@ -1,5 +1,6 @@
import { swallow } from "./diagnostics";
import { isClipVisibleAt, isInClipWindow } from "./clipWindow";
import { sameInstant } from "../clipFacts";
import { interpolateVolumeGain, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
import { elementVolumeLaneGain } from "./audioAutomationVolume.js";
import { fadeGain, NO_FADES, readElementFades, type AudioFades } from "../audioFade.js";
@@ -324,10 +325,11 @@ export function syncRuntimeMedia(params: {
const isTerminalVideo =
isNonLoopVideo &&
!inWindow &&
params.timeSeconds >= clip.end &&
(params.timeSeconds >= clip.end || sameInstant(params.timeSeconds, clip.end)) &&
isClipVisibleAt(params.timeSeconds, clip.start, clip.end, params.getCompositionDuration());
let relTime =
sourceTimeAt(clipRate, Math.min(params.timeSeconds, clip.end) - clip.start) + clip.mediaStart;
sourceTimeAt(clipRate, Math.max(0, Math.min(params.timeSeconds, clip.end) - clip.start)) +
clip.mediaStart;
const isHeldVideoTail =
isTerminalVideo ||
(isNonLoopVideo && clip.sourceDuration != null && relTime >= clip.sourceDuration && inWindow);
@@ -0,0 +1,116 @@
// @vitest-environment node
// fallow-ignore-file code-duplication
import { afterEach, describe, expect, it } from "vitest";
import { parseHTML } from "linkedom";
import { type Page } from "puppeteer-core";
import { createFrameLookupTable, type ExtractedFrames } from "./videoFrameExtractor.js";
import { createVideoFrameInjector } from "./videoFrameInjector.js";
function frames(videoId: string): ExtractedFrames {
const fps = 30;
const totalFrames = 30;
return {
videoId,
srcPath: `${videoId}.webm`,
outputDir: `/tmp/${videoId}`,
framePattern: "frame-%05d.jpg",
fps,
totalFrames,
metadata: {
durationSeconds: totalFrames / fps,
width: 320,
height: 180,
fps,
hasAudio: false,
videoCodec: "vp9",
colorSpace: { colorTransfer: "bt709", colorPrimaries: "bt709", colorSpace: "bt709" },
isVFR: false,
hasAlpha: false,
},
framePaths: new Map(Array.from({ length: totalFrames }, (_, i) => [i, `${videoId}-${i}`])),
};
}
const globals = globalThis as unknown as { window?: unknown; document?: unknown };
const previous = { window: globals.window, document: globals.document };
afterEach(() => {
globals.window = previous.window;
globals.document = previous.document;
});
function mountPage(): { page: Page; paintedFrames: () => Record<string, string> } {
const { window, document } = parseHTML(
`<html><body>
<div><video id="a" data-start="0.1" data-duration="0.2"></video></div>
<div><video id="b" data-start="0.3" data-duration="0.2"></video></div>
</body></html>`,
);
Object.defineProperty(window.HTMLImageElement.prototype, "decode", {
configurable: true,
value: () => Promise.resolve(),
});
Object.defineProperty(window, "getComputedStyle", {
configurable: true,
value: () => ({
display: "block",
visibility: "visible",
opacity: "1",
objectFit: "fill",
objectPosition: "50% 50%",
zIndex: "auto",
getPropertyValue: () => "",
}),
});
for (const video of document.querySelectorAll("video")) {
video.getBoundingClientRect = () => ({ width: 320, height: 180 }) as DOMRect;
}
globals.window = window;
globals.document = document;
const page = {
evaluate: async (fn: (...args: unknown[]) => unknown, ...args: unknown[]) => fn(...args),
} as unknown as Page;
const paintedFrames = () =>
Object.fromEntries(
Array.from(document.querySelectorAll("img.__render_frame__"))
.filter((img) => (img as HTMLImageElement).style.visibility === "visible")
.map((img) => [img.previousElementSibling?.id ?? "", img.getAttribute("src") ?? ""]),
);
return { page, paintedFrames };
}
describe("export frame injection at a shared clip boundary", () => {
it("paints only the incoming clip at the instant the outgoing one ends", async () => {
const table = createFrameLookupTable(
[
{
id: "a",
src: "a.webm",
start: 0.1,
end: 0.1 + 0.2,
mediaStart: 0,
loop: false,
hasAudio: false,
},
{
id: "b",
src: "b.webm",
start: 0.3,
end: 0.5,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
[frames("a"), frames("b")],
);
const inject = createVideoFrameInjector(table, { frameSrcResolver: (path) => `data:${path}` })!;
const { page, paintedFrames } = mountPage();
await inject(page, 8 / 30);
expect(paintedFrames()).toEqual({ a: "data:a-5" });
await inject(page, 9 / 30);
expect(paintedFrames()).toEqual({ b: "data:b-0" });
});
});
@@ -948,6 +948,31 @@ describe("FrameLookupTable", () => {
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
});
it("shows a copy starting on a float sum at that instant, as the preview does", () => {
const copyStart = 0.1 + 0.2;
const clip = (id: string, start: number) => ({
id,
src: `${id}.webm`,
start,
end: start + 0.2,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const frames = (videoId: string) => ({ ...fakeExtracted(30, 30), videoId });
const table = () =>
createFrameLookupTable(
[clip("a", 0.1), clip("copy", copyStart)],
[frames("a"), frames("copy")],
);
expect(table().getActiveFramePayloads(0.3).get("copy")?.frameIndex).toBe(0);
const stepping = table();
stepping.getActiveFramePayloads(0.2);
expect(stepping.getActiveFramePayloads(0.3).get("copy")?.frameIndex).toBe(0);
expect(getFrameAtTime(frames("copy"), 0.3, copyStart)).toBe("frame-0.jpg");
});
it("selects source frames at the authored constant playback rate", () => {
const videos = parseVideoElements(
'<video id="hero" src="clip.webm" data-start="0" data-duration="2" data-playback-rate="2"></video>',
@@ -1038,10 +1063,7 @@ describe("FrameLookupTable", () => {
expect(table.getActiveFramePayloads(31).has("main")).toBe(false); // after resolved end (30)
});
it("holds the last frame at the inclusive clip end (t === end)", () => {
// clip [1,3] with exactly 2s of source frames (60 @ 30fps). The frame
// landing on t === end used to deactivate one frame early and render blank,
// while the runtime keeps the element visible on its last frame.
it("shows the last frame just before the end and leaves at t === end, as the runtime does", () => {
const table = createFrameLookupTable(
[
{
@@ -1056,10 +1078,10 @@ describe("FrameLookupTable", () => {
],
[fakeExtracted(60, 30)],
);
const atEnd = table.getActiveFramePayloads(3.0).get("hero");
expect(atEnd?.frameIndex).toBe(59);
// mid-clip is unaffected
expect(table.getActiveFramePayloads(2.5).get("hero")?.frameIndex).toBe(45);
expect(table.getActiveFramePayloads(2.99).get("hero")?.frameIndex).toBe(59);
expect(table.getActiveFramePayloads(3.0).has("hero")).toBe(false);
expect(table.getFrame("hero", 3.0)).toBeNull();
});
it("holds the last frame across the tail when the source is shorter than the window", () => {
@@ -1080,7 +1102,7 @@ describe("FrameLookupTable", () => {
[fakeExtracted(30, 30)],
);
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
expect(table.getActiveFramePayloads(5.0).get("hero")?.frameIndex).toBe(29);
expect(table.getActiveFramePayloads(4.99).get("hero")?.frameIndex).toBe(29);
});
it("holds the last frame when the source is a sub-frame shorter than the slot", () => {
@@ -1107,27 +1129,38 @@ describe("FrameLookupTable", () => {
expect(table.getActiveFramePayloads(3.4).get("hero")?.frameIndex).toBe(42);
// source exhausted but within tolerance of the end → hold, don't blank
expect(table.getActiveFramePayloads(3.44).get("hero")?.frameIndex).toBe(42);
expect(table.getActiveFramePayloads(3.45).get("hero")?.frameIndex).toBe(42);
});
it("keeps both clips active at a shared adjacent boundary, matching the runtime", () => {
// clip A ends at 3.0, clip B starts at 3.0. The runtime shows both at the
// shared instant; the active set must too.
const table = createFrameLookupTable(
[
{ id: "a", src: "a.webm", start: 0, end: 3, mediaStart: 0, loop: false, hasAudio: false },
{ id: "b", src: "b.webm", start: 3, end: 6, mediaStart: 0, loop: false, hasAudio: false },
],
// createFrameLookupTable maps each clip to extracted frames by id.
[
{ ...fakeExtracted(90, 30), videoId: "a" },
{ ...fakeExtracted(90, 30), videoId: "b" },
],
);
const payloads = table.getActiveFramePayloads(3.0);
expect(payloads.has("a")).toBe(true);
expect(payloads.has("b")).toBe(true);
});
it.each([
["an exact", 0.1, 0.3, 0.3],
["a float-sum", 0.1, 0.1 + 0.2, 9 / 30],
])(
"hands %s shared boundary to the incoming clip only, as the runtime does",
(_, aStart, aEnd, t) => {
const clip = (id: string, start: number, end: number) => ({
id,
src: `${id}.webm`,
start,
end,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const table = () =>
createFrameLookupTable(
[clip("a", aStart, aEnd), clip("b", 0.3, 0.5)],
[
{ ...fakeExtracted(90, 30), videoId: "a" },
{ ...fakeExtracted(90, 30), videoId: "b" },
],
);
const stepping = table();
expect([...stepping.getActiveFramePayloads(8 / 30).keys()]).toEqual(["a"]);
expect([...stepping.getActiveFramePayloads(t).keys()]).toEqual(["b"]);
expect([...table().getActiveFramePayloads(t).keys()]).toEqual(["b"]);
expect(table().getFrame("a", t)).toBeNull();
},
);
});
describe("analyzeClipMediaFit", () => {
@@ -21,6 +21,8 @@ import {
sourceTimeAt,
timeAtSourceTime,
type RateSpec,
hasClipStarted,
isInClipWindow,
parseStrictFiniteTimingNumber,
readMediaStart,
toFps,
@@ -1116,8 +1118,7 @@ export function resolveTimelineExtractionWindow(
if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) {
const phaseOffset = trimmedSourcePreroll % sourceRemaining;
const phaseRemaining = sourceRemaining - phaseOffset;
// The element visibility contract includes its end boundary. Preserve a
// complete cycle on equality as well, otherwise a rebased suffix would
// Keep a complete cycle on equality too, otherwise a rebased suffix would
// wrap to its own first frame instead of the source cycle's first frame.
if (visibleSourceDuration >= phaseRemaining) {
return withTimelineDuration(
@@ -2198,8 +2199,8 @@ function getFrameIndexAtTime(
holdLastFrame = false,
playbackRate: RateSpec = 1,
): number | null {
let localTime = globalTime - videoStart;
if (localTime < 0) return null;
if (!hasClipStarted(globalTime, videoStart)) return null;
let localTime = Math.max(0, globalTime - videoStart);
const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
const loopDuration = timeAtSourceTime(
normalizedPlaybackRate,
@@ -2313,7 +2314,7 @@ export class FrameLookupTable {
getFrame(videoId: string, globalTime: number): string | null {
const video = this.videos.get(videoId);
if (!video) return null;
if (globalTime < video.start || globalTime > video.end) return null;
if (!isInClipWindow(globalTime, video.start, video.end)) return null;
const frameIndex = getFrameIndexAtTime(
video.extracted,
globalTime,
@@ -2333,23 +2334,17 @@ export class FrameLookupTable {
}
private refreshActiveSet(globalTime: number): void {
// The active window is [start, end] INCLUSIVE of the end, mirroring the
// runtime's element-visibility contract (core/runtime init.ts keeps an
// element visible through `currentTime <= end`). An exclusive end-bound
// here deactivated the video one frame early, so the frame landing exactly
// on a clip's end rendered blank while the runtime still showed it.
// The runtime's half-open window. Rendered times stay below the composition end, so its
// terminal hold (isClipVisibleAt) never applies here.
if (this.lastTime == null || globalTime < this.lastTime) {
this.activeVideoIds.clear();
this.startCursor = 0;
for (const entry of this.orderedVideos) {
if (entry.start <= globalTime && globalTime <= entry.end) {
if (!hasClipStarted(globalTime, entry.start)) break;
if (isInClipWindow(globalTime, entry.start, entry.end)) {
this.activeVideoIds.add(entry.videoId);
}
if (entry.start <= globalTime) {
this.startCursor += 1;
} else {
break;
}
this.startCursor += 1;
}
this.lastTime = globalTime;
return;
@@ -2358,10 +2353,10 @@ export class FrameLookupTable {
while (this.startCursor < this.orderedVideos.length) {
const candidate = this.orderedVideos[this.startCursor];
if (!candidate) break;
if (candidate.start > globalTime) {
if (!hasClipStarted(globalTime, candidate.start)) {
break;
}
if (globalTime <= candidate.end) {
if (isInClipWindow(globalTime, candidate.start, candidate.end)) {
this.activeVideoIds.add(candidate.videoId);
}
this.startCursor += 1;
@@ -2369,7 +2364,7 @@ export class FrameLookupTable {
for (const videoId of Array.from(this.activeVideoIds)) {
const video = this.videos.get(videoId);
if (!video || globalTime < video.start || globalTime > video.end) {
if (!video || !isInClipWindow(globalTime, video.start, video.end)) {
this.activeVideoIds.delete(videoId);
}
}
@@ -160,8 +160,8 @@ describe("distributed video metadata", () => {
const lookup = createFrameLookupTable(result.videos, [extractedFrames(metadata)]);
expect(lookup.getActiveFramePayloads(7).get("hero")?.frameIndex).toBe(expectedFrame);
expect(lookup.getFrame("hero", 8)).not.toBeNull();
expect(lookup.getFrame("hero", 8.01)).toBeNull();
expect(lookup.getFrame("hero", 8 - 1 / 30)).not.toBeNull();
expect(lookup.getFrame("hero", 8)).toBeNull();
},
);
});
@@ -1,3 +1,4 @@
import { sameInstant } from "@hyperframes/core/clip-facts";
import { ensureHfIds } from "@hyperframes/parsers/hf-ids";
import {
findTargetElement,
@@ -59,9 +60,10 @@ function rippleElements(
continue;
}
const start = numericAttribute(candidate, "data-start");
if (start !== null && start >= at) {
candidate.setAttribute("data-start", String(start + duration));
}
if (start === null) continue;
const metInsertionPoint = sameInstant(start, at);
if (start < at && !metInsertionPoint) continue;
candidate.setAttribute("data-start", String((metInsertionPoint ? at : start) + duration));
}
}
@@ -17,6 +17,8 @@
* style) and apply the returned `StackingPatch[]` however they persist styles.
*/
import { spansOverlap } from "@hyperframes/core/clip-facts";
/** Minimal element view this module reasons over. */
export interface StackingElement {
/** Stable identity (TimelineElement.key ?? id). */
@@ -69,8 +71,6 @@ export interface StackingPatch {
zIndex: number;
}
const EPS = 1e-6;
/**
* Canonical paint-scope key: leaf z-indexes are comparable only within the same
* source document and CSS stacking context. The ONLY place this normalization
@@ -90,17 +90,17 @@ export function samePaintScope(
/**
* Two clips overlap in time when their half-open [start, end) intervals intersect.
*
* NOTE the `- EPS`: this DELIBERATELY diverges from `timeRangesOverlap`'s exact
* NOTE the float slack: this DELIBERATELY diverges from `timeRangesOverlap`'s exact
* strict-`<` (timelineCollision.ts). A boolean collision decision is idempotent, so
* exact `<` is fine there; here the result drives a VISIBLE stacking re-lane, so the
* epsilon guards against float fuzz (e.g. 5.0000001 vs 5) spuriously overlapping two
* epsilon guards against float fuzz (e.g. 19.8 + 6.4 vs 26.2) spuriously overlapping two
* abutting clips and shuffling lanes. The two are intended to differ, not align.
*/
function overlapsInTime(
a: Pick<StackingElement, "start" | "duration">,
b: Pick<StackingElement, "start" | "duration">,
): boolean {
return a.start < b.start + b.duration - EPS && b.start < a.start + a.duration - EPS;
return spansOverlap(a.start, a.start + a.duration, b.start, b.start + b.duration);
}
/**
@@ -1,3 +1,4 @@
import { spansOverlap } from "@hyperframes/core/clip-facts";
import type { TimelineElement } from "../store/playerStore";
import { isAudioTimelineElement } from "../../utils/timelineInspector";
import { isTransitionPair } from "./timelineTransitionSeams";
@@ -23,11 +24,9 @@ export function isMainTrackElement(el: TimelineElement): boolean {
const keyOf = (el: TimelineElement) => el.key ?? el.id;
const EPS = 1e-6;
/** Two clips overlap when their half-open [start, end) intervals intersect. */
function overlaps(a: TimelineElement, b: TimelineElement): boolean {
return a.start < b.start + b.duration - EPS && b.start < a.start + a.duration - EPS;
return spansOverlap(a.start, a.start + a.duration, b.start, b.start + b.duration);
}
/** Deterministic order on the stable clip id (never the mutated lane/track). */