ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-04 07:19:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: make creator media edits render-safe (#3322)

Browse Source 
* feat: make creator media edits render-safe

* fix: align media playback timing

* docs: add creator editing recipes

* docs: expand creator editing guidance

* fix: unify media source offsets

* fix: scale natural media duration

* fix: preserve natural media zero spans

* fix: align compiled natural media timing

* test: classify compiler media test as integration

* fix: drop inactive media windows

* fix: unify literal timing parsing

* fix: keep browser media parsing serializable

* fix: keep page timing readers strict

* fix: close remaining preview timing gaps

* fix(core): preserve Studio voice pitch at playback speed

* chore: keep creator contract source-neutral
This commit is contained in:
Miguel Ángel
2026-08-18 10:17:02 -04:00
committed by GitHub
parent 049f5618d7
commit afafca4b96
42 changed files with 2695 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/runtime/webAudioTransport.ts
+126 -22
View File
@@ -7,6 +7,7 @@ import {
import { VOLUME_RANGE } from "../audioAutomation.js";
import { swallow } from "./diagnostics";
import { getDebugSurface } from "./globals.js";
import { readElementPlaybackRate } from "./media.js";
function normalizeRate(rate: number): number {
if (!Number.isFinite(rate) || rate <= 0) return 1;
@@ -43,21 +44,23 @@ function startBoundedSource(
elapsed: number;
mediaStart: number;
scheduledAt: number;
safeRate: number;
globalRate: number;
mediaRate: number;
clipDuration: number;
},
): boolean {
const { elapsed, mediaStart, scheduledAt, safeRate, clipDuration } = opts;
const { elapsed, mediaStart, scheduledAt, globalRate, mediaRate, clipDuration } = opts;
const hasBound = Number.isFinite(clipDuration) && clipDuration > 0;
const clipSourceLen = clipDuration * safeRate;
const clipSourceLen = clipDuration * mediaRate;
if (elapsed >= 0) {
const remaining = clipSourceLen - elapsed;
const sourceElapsed = elapsed * mediaRate;
const remaining = clipSourceLen - sourceElapsed;
if (hasBound && remaining <= 0) return false;
if (hasBound) node.start(0, elapsed + mediaStart, remaining);
else node.start(0, elapsed + mediaStart);
if (hasBound) node.start(0, sourceElapsed + mediaStart, remaining);
else node.start(0, sourceElapsed + mediaStart);
return true;
}
const delay = -elapsed / safeRate;
const delay = -elapsed / globalRate;
if (hasBound) node.start(scheduledAt + delay, mediaStart, clipSourceLen);
else node.start(scheduledAt + delay, mediaStart);
return true;
@@ -77,9 +80,8 @@ function scheduleVolumeLane(
scheduleParamLane([{ param: gainNode.gain }], lane, VOLUME_RANGE.scale, timing);
}
export type ScheduledSource = {
type ScheduledSourceBase = {
el: HTMLMediaElement;
sourceNode: AudioBufferSourceNode;
gainNode: GainNode;
/** FX chain spliced between source and gain, when the element carries one. */
fx?: ElementFxHandle | null;
@@ -87,18 +89,35 @@ export type ScheduledSource = {
mediaStart: number;
scheduledAt: number;
priorMuted: boolean;
priorVolume: number;
mediaPlaybackRate: number;
// The clip had a finite window, so start() was given a fixed duration in
// buffer-sample seconds. That bound can't be rescaled in place on a rate
// change — callers must stopAll()+reschedule (see hasBoundedActiveSources).
bounded: boolean;
};
export type ScheduledSource = ScheduledSourceBase &
(
| { sourceKind: "buffer"; sourceNode: AudioBufferSourceNode }
| { sourceKind: "media-element"; sourceNode: MediaElementAudioSourceNode }
);
function isBufferSource(
source: ScheduledSource,
): source is ScheduledSourceBase & { sourceKind: "buffer"; sourceNode: AudioBufferSourceNode } {
return source.sourceKind === "buffer";
}
export class WebAudioTransport {
private _ctx: AudioContext | null = null;
private _bufferCache = new Map<string, AudioBuffer>();
private _failedSrcs = new Set<string>();
private _mediaElementSources = new WeakMap<HTMLMediaElement, MediaElementAudioSourceNode>();
private _activeSources: ScheduledSource[] = [];
private _masterGain: GainNode | null = null;
private _masterVolume = 1;
private _masterMuted = false;
// Composition-time reference frame: at AudioContext time `_rateAnchorCtx`,
// composition time was `_rateAnchorComp`, and time has been advancing at
// `_rate` composition-seconds per wallclock-second since.
@@ -113,6 +132,7 @@ export class WebAudioTransport {
this._ctx = new AudioContext();
this._masterGain = this._ctx.createGain();
this._masterGain.connect(this._ctx.destination);
this.applyMasterGain();
return true;
} catch {
return false;
@@ -179,6 +199,75 @@ export class WebAudioTransport {
return this._playGeneration;
}
/**
* Route the browser's pitch-preserving HTMLMediaElement transport through the
* same FX, automation, element-gain, and master graph used by final audio.
* The media element remains the source-time/rate owner; Web Audio is strictly
* downstream and therefore never resamples it for Studio global speed.
*/
async scheduleMediaElementPlayback(
el: HTMLMediaElement,
compositionStart: number,
_mediaStart: number,
compositionTime: number,
volume: number,
generation: number,
rate = 1,
): Promise<ScheduledSource | null> {
if (!this._ctx || !this._masterGain) return null;
if (generation !== this._playGeneration) return null;
try {
if (this._ctx.state === "suspended") await this._ctx.resume();
if (generation !== this._playGeneration) return null;
let sourceNode = this._mediaElementSources.get(el);
if (!sourceNode) {
sourceNode = this._ctx.createMediaElementSource(el);
this._mediaElementSources.set(el, sourceNode);
}
const safeRate = normalizeRate(rate);
const gainNode = this._ctx.createGain();
gainNode.gain.value = volume;
const scheduledAt = this._ctx.currentTime;
const elapsed = compositionTime - compositionStart;
const timing: AutomationTiming = { scheduledAt, elapsed, rate: safeRate };
const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing);
gainNode.connect(this._masterGain);
scheduleVolumeLane(el, gainNode, timing);
this._rate = safeRate;
this._rateAnchorCtx = scheduledAt;
this._rateAnchorComp = compositionTime;
const scheduled: ScheduledSource = {
fx,
el,
sourceNode,
sourceKind: "media-element",
gainNode,
compositionStart,
mediaStart: _mediaStart,
scheduledAt,
priorMuted: el.muted,
priorVolume: el.volume,
mediaPlaybackRate: readElementPlaybackRate(el),
bounded: false,
};
// Chrome applies HTMLMediaElement.volume before MediaElementAudioSource.
// Keep that upstream stage at unity so the existing downstream gain is
// the single owner of author × user volume and automation.
el.volume = 1;
this._activeSources.push(scheduled);
this._paused = false;
return scheduled;
} catch (err) {
swallow("webAudioTransport.mediaElementSource", err);
return null;
}
}
async schedulePlayback(
el: HTMLMediaElement,
buffer: AudioBuffer,
@@ -200,10 +289,12 @@ export class WebAudioTransport {
if (generation !== this._playGeneration) return null;
const safeRate = normalizeRate(rate);
const mediaRate = readElementPlaybackRate(el);
const sourceRate = safeRate * mediaRate;
const sourceNode = this._ctx.createBufferSource();
sourceNode.buffer = buffer;
sourceNode.playbackRate.value = safeRate;
sourceNode.playbackRate.value = sourceRate;
const gainNode = this._ctx.createGain();
gainNode.gain.value = volume;
@@ -230,7 +321,8 @@ export class WebAudioTransport {
elapsed,
mediaStart,
scheduledAt,
safeRate,
globalRate: safeRate,
mediaRate,
clipDuration,
})
) {
@@ -249,11 +341,14 @@ export class WebAudioTransport {
fx,
el,
sourceNode,
sourceKind: "buffer",
gainNode,
compositionStart,
mediaStart,
scheduledAt,
priorMuted,
priorVolume: el.volume,
mediaPlaybackRate: mediaRate,
bounded: Number.isFinite(clipDuration) && clipDuration > 0,
};
this._activeSources.push(scheduled);
@@ -314,7 +409,9 @@ export class WebAudioTransport {
this._rate = safeRate;
for (const source of this._activeSources) {
try {
source.sourceNode.playbackRate.value = safeRate;
if (isBufferSource(source)) {
source.sourceNode.playbackRate.value = safeRate * source.mediaPlaybackRate;
}
source.fx?.setRate(safeRate);
} catch (err) {
swallow("webAudioTransport.setRate", err);
@@ -327,29 +424,29 @@ export class WebAudioTransport {
// arg at its original rate; a later rate change can't rescale it in place, so
// the caller must stopAll()+reschedule to keep trimmed clips ending on time.
hasBoundedActiveSources(): boolean {
return this._activeSources.some((s) => s.bounded);
return this._activeSources.some((s) => isBufferSource(s) && s.bounded);
}
stopAll(): void {
for (const source of this._activeSources) {
try {
source.sourceNode.stop();
if (isBufferSource(source)) source.sourceNode.stop();
source.sourceNode.disconnect();
source.fx?.dispose();
source.gainNode.disconnect();
} catch {
// already stopped
}
source.el.muted = source.priorMuted;
if (isBufferSource(source)) source.el.muted = source.priorMuted;
else source.el.volume = source.priorVolume;
}
this._activeSources = [];
this._paused = true;
}
setVolume(volume: number): void {
if (this._masterGain) {
this._masterGain.gain.value = Math.max(0, Math.min(1, volume));
}
this._masterVolume = Math.max(0, Math.min(1, volume));
this.applyMasterGain();
}
setElementVolume(el: HTMLMediaElement, volume: number): void {
@@ -357,6 +454,7 @@ export class WebAudioTransport {
for (const source of this._activeSources) {
if (source.el !== el) continue;
try {
if (source.sourceKind === "media-element") source.el.volume = 1;
source.gainNode.gain.value = safeVolume;
} catch (err) {
swallow("webAudioTransport.setElementVolume", err);
@@ -365,9 +463,12 @@ export class WebAudioTransport {
}
setMuted(muted: boolean): void {
if (this._masterGain) {
this._masterGain.gain.value = muted ? 0 : 1;
}
this._masterMuted = muted;
this.applyMasterGain();
}
private applyMasterGain(): void {
if (this._masterGain) this._masterGain.gain.value = this._masterMuted ? 0 : this._masterVolume;
}
isActive(): boolean {
@@ -377,13 +478,14 @@ export class WebAudioTransport {
/** Whether the transport currently plays THIS element (the runtime mutes it to
* avoid double audio; an unclaimed track stays audible). */
ownsElement(el: HTMLMediaElement): boolean {
return !this._paused && this._activeSources.some((s) => s.el === el);
return !this._paused && this._activeSources.some((s) => s.el === el && isBufferSource(s));
}
destroy(): void {
this.stopAll();
this._bufferCache.clear();
this._failedSrcs.clear();
this._mediaElementSources = new WeakMap();
if (this._ctx) {
try {
void this._ctx.close();
@@ -393,5 +495,7 @@ export class WebAudioTransport {
}
this._ctx = null;
this._masterGain = null;
this._masterVolume = 1;
this._masterMuted = false;
}
}