mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
* perf(core): playback housekeeping runs on a timer, and rebinds stop re-sampling silent media
While playing, the transport re-posted the clip manifest every 20 frames and
re-bound media every 30, frame-counted, so a 120 Hz display did both twice as
often for a film that was not changing. Playing, those jobs now run when the
composition changes (rate-limited as before) and on a one-second poll for
what nothing can push.
A rebind re-probed every media element's volume by seeking the whole timeline
60 times per second of clip, about 1,500 seeks and 50-140 ms in one frame.
The probe seeks with events suppressed, so only a tween on the element that
names volume can move it; an element no tween touches is now answered from
the tweens without seeking.
* fix(core): a data-width or data-height change applies while playing and parked
postTimeline applies the root's sizing, and it now runs when the composition
revision moves. The revision only watched timing attributes, so a size edit
waited for a pause while playing, and on main it was missed while parked
too. The observer now also watches data-width and data-height. The tick
comment is trimmed to three lines, the tick reuses its playing flag, and the
cadence test is renamed to what it checks now.
* test(core): share one 120 Hz frame step in the transport park tests
The two playing tests each defined the same frame helper; one frame120Hz
now sits beside settle and all four loops use it. No source change.
* refactor(core): drop the playing poll doc comments
The comment ratchet does not let a file's comment share rise; the reason
lives in the PR body.
* fix(core): a playing clip's track, label and z-index reach Studio again
The playing path had dropped the periodic manifest post, and the
composition observer does not see data-track-index, data-timeline-label or
an inline z-index on an existing clip (watching style would fire on every
GSAP frame). On the playing poll the runtime now builds the manifest and
posts it only when it differs from the last one posted, so an edit lands
within a second and a steady film still posts nothing.
* fix(core): keep sampling volume when a tween drives a non-DOM object
A tween on a plain-object proxy whose setter writes the element's volume
names neither the element nor volume, so the fast skip dropped it and
WebAudio-routed preview played the static data-volume (0 where the film
asks for 0.375). The skip now applies only when every tween targets DOM
nodes; any other target falls back to the full volume probe, as on main.
* fix(core): skip the volume probe again for spacers, calls and counters
The non-DOM fallback fired on any plain-object or function target, so a
tl.to({}, {duration}) spacer, a tl.call or a tweened counter brought back the
full probe on ordinary films. It now falls back only when a tweened property
resolves to a setter on the target or its prototype chain, which is what a
gain proxy or a class with a volume setter uses; data properties skip.
* fix(core): post a clip edit made while playing when playback stops
A label, track or inline z-index edited while playing and then paused before
the next one-second poll was never posted: once parked, nothing watches it.
The tick where playback stops now runs the same manifest diff once and posts
only if the manifest changed.
314 lines
11 KiB
TypeScript
314 lines
11 KiB
TypeScript
import type { RuntimeTimelineLike } from "./types";
|
||
import { clampAudioGain, withUnclampedVolume } from "../audioGain.js";
|
||
import { parseStrictFiniteTimingNumber } from "./playbackRate";
|
||
import { createRuntimeStartTimeResolver } from "./startResolver";
|
||
import { isMediaElement } from "./domRealm";
|
||
|
||
/**
|
||
* Shared volume-automation utilities used by both the renderer (offline PCM
|
||
* baking in audioVolumeEnvelope.ts) and the preview runtime (per-tick gain
|
||
* applied in syncRuntimeMedia).
|
||
*
|
||
* Keeping the two concerns in one place ensures preview and render derive the
|
||
* envelope from the same logic and the same probe samples.
|
||
*/
|
||
|
||
export interface VolumeKeyframe {
|
||
time: number;
|
||
volume: number;
|
||
}
|
||
|
||
/**
|
||
* Normalise raw keyframes to track-relative seconds: subtract `trackStart`,
|
||
* clamp to [0,1], sort, de-duplicate, and prepend a `baseVolume` anchor at
|
||
* t=0 when the first keyframe starts after the clip's begin.
|
||
*
|
||
* Returns an empty array when all keyframes are invalid — the caller should
|
||
* treat an empty envelope as "no automation, use static volume."
|
||
*/
|
||
export function normaliseEnvelope(
|
||
keyframes: VolumeKeyframe[],
|
||
trackStart: number,
|
||
baseVolume: number,
|
||
): VolumeKeyframe[] {
|
||
const points = keyframes
|
||
.filter((k) => Number.isFinite(k.time) && Number.isFinite(k.volume))
|
||
.map((k) => ({
|
||
time: Math.max(0, k.time - trackStart),
|
||
volume: clampAudioGain(k.volume),
|
||
}))
|
||
.sort((a, b) => a.time - b.time);
|
||
|
||
const deduped: VolumeKeyframe[] = [];
|
||
for (const point of points) {
|
||
const previous = deduped.at(-1);
|
||
if (previous && Math.abs(previous.time - point.time) < 1e-9) {
|
||
previous.volume = point.volume;
|
||
} else {
|
||
deduped.push(point);
|
||
}
|
||
}
|
||
|
||
if (deduped.length === 0) return deduped;
|
||
if (deduped[0]!.time > 0) {
|
||
deduped.unshift({ time: 0, volume: clampAudioGain(baseVolume) });
|
||
}
|
||
return deduped;
|
||
}
|
||
|
||
/**
|
||
* Linearly interpolate the gain at time `t` (track-relative seconds) from a
|
||
* normalised envelope produced by `normaliseEnvelope`. Returns 1 when the
|
||
* envelope is empty.
|
||
*/
|
||
export function interpolateVolumeGain(envelope: VolumeKeyframe[], t: number): number {
|
||
if (envelope.length === 0) return 1;
|
||
|
||
let segment = 0;
|
||
// The PCM baker intentionally inlines this lookup with a monotonic cursor
|
||
// because calling this preview-oriented helper per sample would be O(N×M).
|
||
// fallow-ignore-next-line code-duplication
|
||
while (segment < envelope.length - 2 && t >= envelope[segment + 1]!.time) {
|
||
segment += 1;
|
||
}
|
||
|
||
const a = envelope[segment]!;
|
||
const b = envelope[segment + 1] ?? a;
|
||
const span = b.time - a.time;
|
||
const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (t - a.time) / span));
|
||
return a.volume + (b.volume - a.volume) * progress;
|
||
}
|
||
|
||
function recordVolumeSample(
|
||
keyframes: VolumeKeyframe[],
|
||
previousSample: VolumeKeyframe | undefined,
|
||
sample: VolumeKeyframe,
|
||
isFinalSample: boolean,
|
||
): void {
|
||
const last = keyframes.at(-1);
|
||
if (!last || Math.abs(last.volume - sample.volume) > 0.0001) {
|
||
// Change-only compression must retain the preceding real sample so a
|
||
// flat run stays flat instead of being interpolated into the next value.
|
||
// During a continuous ramp, that sample is already the last keyframe.
|
||
if (last && previousSample && previousSample.time > last.time) {
|
||
keyframes.push(previousSample);
|
||
}
|
||
keyframes.push(sample);
|
||
} else if (isFinalSample && sample.time > last.time) {
|
||
keyframes.push(sample);
|
||
}
|
||
}
|
||
|
||
function parseVolumeNumber(value: string | undefined): number | undefined {
|
||
const parsed = Number.parseFloat(value ?? "");
|
||
return Number.isFinite(parsed) ? parsed : undefined;
|
||
}
|
||
|
||
interface ProbeWindow {
|
||
start: number;
|
||
end: number;
|
||
staticVolume: number;
|
||
}
|
||
|
||
function resolveVolumeProbeWindow(
|
||
el: HTMLAudioElement | HTMLVideoElement,
|
||
compositionDuration: number,
|
||
): ProbeWindow {
|
||
// Probe samples are stamped with ROOT-timeline seek times, and
|
||
// `normaliseEnvelope` rebases them by this start — so it has to be the same
|
||
// absolute start the transport plays the clip at. Reading `data-start`
|
||
// directly gave a composition-local value, which put a nested clip's whole
|
||
// envelope at the wrong origin.
|
||
const start = createRuntimeStartTimeResolver({
|
||
timelineRegistry: (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> })
|
||
.__timelines,
|
||
includeAuthoredTimingAttrs: true,
|
||
}).resolveMediaStartForElement(el);
|
||
const endAttr = parseStrictFiniteTimingNumber(el.dataset.end) ?? undefined;
|
||
const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration) ?? undefined;
|
||
let end = compositionDuration;
|
||
if (durAttr !== undefined && durAttr > 0) {
|
||
end = start + durAttr;
|
||
} else if (endAttr !== undefined && endAttr > start) {
|
||
end = endAttr;
|
||
}
|
||
const staticAttr = parseVolumeNumber(el.dataset.volume) ?? 1;
|
||
return { start, end, staticVolume: clampAudioGain(staticAttr) };
|
||
}
|
||
|
||
/**
|
||
* Probe a single media element's volume automation by seeking a GSAP timeline
|
||
* through the element's active window.
|
||
*
|
||
* Runs synchronously in the browser. The timeline is left at its current
|
||
* position after the probe (the next transport tick re-seeks it to `t`).
|
||
*
|
||
* Returns null when the element has no detectable automation (volume never
|
||
* changes from its initial `data-volume` value).
|
||
*/
|
||
export function probeElementVolumeKeyframes(
|
||
el: HTMLAudioElement | HTMLVideoElement,
|
||
seekTimeline: (t: number) => void,
|
||
compositionDuration: number,
|
||
sampleFps: number,
|
||
): VolumeKeyframe[] | null {
|
||
return probeKeyframesInWindow(
|
||
el,
|
||
seekTimeline,
|
||
compositionDuration,
|
||
sampleFps,
|
||
resolveVolumeProbeWindow(el, compositionDuration),
|
||
);
|
||
}
|
||
|
||
/** Sampling half of the probe, given an already-resolved window. Split out so
|
||
* `probeAndCacheElementVolume` resolves that window ONCE and reuses it for the
|
||
* envelope rebase, instead of deriving the same start twice per element. */
|
||
function probeKeyframesInWindow(
|
||
el: HTMLAudioElement | HTMLVideoElement,
|
||
seekTimeline: (t: number) => void,
|
||
compositionDuration: number,
|
||
sampleFps: number,
|
||
{ start, end, staticVolume }: ProbeWindow,
|
||
): VolumeKeyframe[] | null {
|
||
const step = 1 / Math.min(60, Math.max(1, sampleFps));
|
||
const sampleStart = Math.max(0, start);
|
||
const sampleEnd = Math.min(compositionDuration, end);
|
||
|
||
const keyframes: VolumeKeyframe[] = withUnclampedVolume(el, () => {
|
||
// Reset to data-volume so GSAP captures the correct FROM value. Above
|
||
// unity that only survives because the shadow accessor is installed.
|
||
el.volume = staticVolume;
|
||
|
||
const samples: VolumeKeyframe[] = [];
|
||
let previousSample: VolumeKeyframe | undefined;
|
||
for (let t = sampleStart; t <= sampleEnd + 1e-6; t = Math.min(sampleEnd, t + step)) {
|
||
seekTimeline(t);
|
||
const raw = Number(el.volume);
|
||
if (Number.isFinite(raw)) {
|
||
const sample = {
|
||
time: Number(t.toFixed(6)),
|
||
volume: Number(clampAudioGain(raw).toFixed(6)),
|
||
};
|
||
recordVolumeSample(samples, previousSample, sample, t === sampleEnd);
|
||
previousSample = sample;
|
||
}
|
||
if (t === sampleEnd) break;
|
||
}
|
||
return samples;
|
||
});
|
||
|
||
const hasAutomation = keyframes.some((kf) => Math.abs(kf.volume - staticVolume) > 0.0001);
|
||
return hasAutomation ? keyframes : null;
|
||
}
|
||
|
||
export type RuntimeTimelineRef = Partial<
|
||
Pick<RuntimeTimelineLike, "totalTime" | "seek" | "getChildren">
|
||
>;
|
||
|
||
export interface VolumeProbeOptions {
|
||
/**
|
||
* Render/probe pages must not sample the live visual timeline during runtime
|
||
* initialization. The producer discovers audio automation in its own
|
||
* isolated pass and bakes it before frame capture, so seeking here is both
|
||
* redundant and capable of materializing future zero-duration GSAP state.
|
||
*
|
||
* Preview callers omit this option and retain live automation discovery.
|
||
*/
|
||
allowLiveTimelineSeek?: boolean;
|
||
}
|
||
|
||
function namesKey(vars: unknown, matches: (key: string) => boolean, depth = 0): boolean {
|
||
if (depth > 3 || vars === null || typeof vars !== "object") return false;
|
||
if (Array.isArray(vars)) return vars.some((item) => namesKey(item, matches, depth + 1));
|
||
if (Object.getPrototypeOf(vars) !== Object.prototype) return false;
|
||
return Object.entries(vars).some(
|
||
([key, value]) => matches(key) || namesKey(value, matches, depth + 1),
|
||
);
|
||
}
|
||
|
||
const isDomNode = (target: unknown): boolean =>
|
||
typeof (target as { nodeType?: unknown } | null)?.nodeType === "number";
|
||
|
||
function hasSetter(target: unknown, key: string): boolean {
|
||
for (let o = Object(target); o; o = Object.getPrototypeOf(o)) {
|
||
const descriptor = Object.getOwnPropertyDescriptor(o, key);
|
||
if (descriptor) return typeof descriptor.set === "function";
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* Whether seeking `timeline` can move `el.volume`: a tween on the element that names `volume`, or a tween
|
||
* on a non-DOM object whose tweened property is a setter (a gain proxy). Spacers, calls and counters skip.
|
||
*/
|
||
function timelineCanMoveVolume(timeline: RuntimeTimelineRef, el: HTMLMediaElement): boolean {
|
||
if (typeof timeline.getChildren !== "function") return true;
|
||
try {
|
||
return timeline.getChildren(true, true, false).some((tween) => {
|
||
const targets: unknown[] = tween.targets?.() ?? [];
|
||
return targets.some((target) =>
|
||
isDomNode(target)
|
||
? target === el && namesKey(tween.vars, (key) => key === "volume")
|
||
: namesKey(tween.vars, (key) => hasSetter(target, key)),
|
||
);
|
||
});
|
||
} catch {
|
||
return true;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Probe a media element and, if volume automation is detected, store a
|
||
* NORMALISED envelope in `cache`. Safe to call with a null timeline — returns
|
||
* early.
|
||
*
|
||
* `probeElementVolumeKeyframes` stamps each keyframe with the timeline seek
|
||
* time it was sampled at. Everything downstream of this cache — like the
|
||
* renderer's PCM baker — indexes an envelope by track-relative seconds, so the
|
||
* rebase belongs here, at the one point that fills the cache, rather than at
|
||
* each read.
|
||
*/
|
||
export function probeAndCacheElementVolume(
|
||
mediaEl: HTMLMediaElement,
|
||
timeline: RuntimeTimelineRef | null | undefined,
|
||
compositionDuration: number,
|
||
cache: WeakMap<HTMLMediaElement, VolumeKeyframe[]>,
|
||
options: VolumeProbeOptions = {},
|
||
): void {
|
||
if (options.allowLiveTimelineSeek === false) return;
|
||
if (!timeline) return;
|
||
if (!isMediaElement(mediaEl)) return;
|
||
if (compositionDuration <= 0) return;
|
||
// Sampling is ~60 whole-timeline seeks per second of clip, paid again on every rebind.
|
||
if (!timelineCanMoveVolume(timeline, mediaEl)) return;
|
||
|
||
const seekFn = (t: number) => {
|
||
try {
|
||
if (typeof timeline.totalTime === "function") {
|
||
timeline.totalTime(t, true);
|
||
} else if (typeof timeline.seek === "function") {
|
||
timeline.seek(t, true);
|
||
}
|
||
} catch {
|
||
// ignore seek failures during probe
|
||
}
|
||
};
|
||
// Sampling seeks the live timeline through the entire composition. Preserve
|
||
// its playhead so the probe cannot perturb the first rendered frame (or any
|
||
// user scrub in the preview).
|
||
const originalTime =
|
||
typeof timeline.totalTime === "function"
|
||
? Number(timeline.totalTime())
|
||
: typeof timeline.seek === "function"
|
||
? Number(timeline.seek())
|
||
: 0;
|
||
const probeWindow = resolveVolumeProbeWindow(mediaEl, compositionDuration);
|
||
const keyframes = probeKeyframesInWindow(mediaEl, seekFn, compositionDuration, 60, probeWindow);
|
||
if (Number.isFinite(originalTime)) seekFn(originalTime);
|
||
if (keyframes) {
|
||
const envelope = normaliseEnvelope(keyframes, probeWindow.start, probeWindow.staticVolume);
|
||
if (envelope.length > 0) cache.set(mediaEl, envelope);
|
||
}
|
||
}
|