import { swallow } from "./diagnostics"; import { interpolateVolumeGain, type VolumeKeyframe } from "./mediaVolumeEnvelope.js"; import { elementVolumeLaneGain } from "./audioAutomationVolume.js"; import { readElementPlaybackRate, readMediaStart } from "./playbackRate.js"; import { clampAudioGain } from "../audioGain.js"; import { findInjectedRenderFrame } from "./renderFrameSibling.js"; export { readElementPlaybackRate, resolveNaturalMediaTimelineDuration } from "./playbackRate.js"; export function readElementPlaybackStart(el: Element): number { return readMediaStart(el); } /** * Resolve a media element's timeline window without conflating a video's * authored display slot with the amount of source left to decode. * * An explicit video slot may outlive its source and holds the final frame. * Audio remains source-bounded because it has no visual hold state. */ export function resolveRuntimeMediaClipDuration(params: { isVideo: boolean; sourceDuration: number | null; hostRemaining: number | null; explicitDuration: number | null; }): number | null { const mediaDuration = params.isVideo ? (params.explicitDuration ?? params.sourceDuration) : params.sourceDuration; const candidates = ( params.isVideo ? [mediaDuration, params.hostRemaining] : [mediaDuration, params.hostRemaining, params.explicitDuration] ).filter((value): value is number => value != null && Number.isFinite(value) && value >= 0); return candidates.length > 0 ? Math.min(...candidates) : null; } export type RuntimeMediaClip = { el: HTMLVideoElement | HTMLAudioElement; start: number; mediaStart: number; duration: number; end: number; volume: number | null; playbackRate: number; loop: boolean; /** Source media duration in seconds (from el.duration). Used for loop wrapping. */ sourceDuration: number | null; /** * Probed volume keyframes from the GSAP timeline (same probe the renderer * uses). When present, `syncRuntimeMedia` drives volume from the envelope * rather than from `data-volume` + GSAP-change tracking, eliminating the * race between the 60 Hz transport tick and GSAP's own seek. */ volumeKeyframes?: VolumeKeyframe[]; }; export function refreshRuntimeMediaCache(params?: { resolveStartSeconds?: (element: Element) => number; resolveDurationSeconds?: (element: HTMLVideoElement | HTMLAudioElement) => number | null; shouldIncludeElement?: (element: HTMLVideoElement | HTMLAudioElement) => boolean; }): { timedMediaEls: Array; mediaClips: RuntimeMediaClip[]; videoClips: RuntimeMediaClip[]; maxMediaEnd: number; } { const mediaEls = Array.from(document.querySelectorAll("video, audio")) as Array< HTMLVideoElement | HTMLAudioElement >; const timedMediaEls = params?.shouldIncludeElement ? mediaEls.filter((el) => params.shouldIncludeElement?.(el)) : mediaEls.filter((el) => el.hasAttribute("data-start")); const mediaClips: RuntimeMediaClip[] = []; const videoClips: RuntimeMediaClip[] = []; let maxMediaEnd = 0; for (const el of timedMediaEls) { const start = params?.resolveStartSeconds ? params.resolveStartSeconds(el) : Number.parseFloat(el.dataset.start ?? "0"); if (!Number.isFinite(start)) continue; const mediaStart = readElementPlaybackStart(el); const playbackRate = readElementPlaybackRate(el); const loop = el.loop; const sourceDuration = Number.isFinite(el.duration) && el.duration > 0 ? el.duration : null; let duration = params?.resolveDurationSeconds?.(el) ?? Number.parseFloat(el.dataset.duration ?? ""); if ((!Number.isFinite(duration) || duration < 0) && sourceDuration != null) { // Effective duration accounts for playback rate: // at 0.5x, a 10s source plays for 20s on the timeline duration = Math.max(0, (sourceDuration - mediaStart) / playbackRate); } const hasKnownDuration = Number.isFinite(duration) && duration >= 0; const end = hasKnownDuration ? start + duration : Number.POSITIVE_INFINITY; const volumeRaw = Number.parseFloat(el.dataset.volume ?? ""); const clip: RuntimeMediaClip = { el, start, mediaStart, duration: hasKnownDuration ? duration : Number.POSITIVE_INFINITY, end, volume: Number.isFinite(volumeRaw) ? volumeRaw : null, playbackRate, loop, sourceDuration, }; mediaClips.push(clip); if (el.tagName === "VIDEO") videoClips.push(clip); if (Number.isFinite(end)) maxMediaEnd = Math.max(maxMediaEnd, end); } return { timedMediaEls, mediaClips, videoClips, maxMediaEnd }; } // Per-element timeline→media offset from the previous tick. Used to tell a // gradual drift (initial buffer catch-up, where offset grows ~16ms/tick) from // a scrub (where offset jumps in one tick). Cleared when a clip becomes // inactive so the next activation gets a hard resync on its first tick. const lastOffset = new WeakMap(); // Desired source time from the previous active tick. Unlike `forceSync`, which // also covers play/pause and rate changes, a decrease here identifies an actual // backward transport seek within an audio clip. const lastRelativeTime = new WeakMap(); const strictDriftSamples = new WeakMap(); // Elements that had a seek past their buffered range (common with streaming // MP3 where preload="metadata" only fetches the first few seconds). After // setting preload="auto" and calling load(), we mark the element so subsequent // ticks don't restart the fetch in a loop while the browser downloads data. // Cleared when the clip leaves its active window. const seekLoadRetried = new WeakSet(); // Elements whose play() is in flight. The sync runs on a 50 ms poll and with // a 1–2 s buffer that would fire 20–40 spurious play() calls per element — // noise in devtools and, worse, each `.catch(() => {})` would swallow a real // AbortError / NotAllowedError that should surface. Cleared on the `playing` // event (actual playback started) or on `pause`/`error` (state ended). const playRequested = new WeakSet(); function markPlayRequested(el: HTMLMediaElement): void { if (playRequested.has(el)) return; playRequested.add(el); const clear = () => playRequested.delete(el); el.addEventListener("playing", clear, { once: true }); el.addEventListener("pause", clear, { once: true }); el.addEventListener("error", clear, { once: true }); } // HTMLMediaElement.NETWORK_NO_SOURCE — no usable source (404 / unsupported). const MEDIA_NETWORK_NO_SOURCE = 3; // An element that errored or has no source can't play; re-issuing play() every // tick just floods rejections. Skip it until its state changes (src reload). function isUnplayable(el: HTMLMediaElement): boolean { return el.error != null || el.networkState === MEDIA_NETWORK_NO_SOURCE; } const lastRuntimeAppliedVolume = new WeakMap(); function clampVolume(volume: number): number { if (!Number.isFinite(volume)) return 1; return Math.max(0, Math.min(1, volume)); } /** * Drop every per-source sync baseline tracked for `el` — offset drift * samples, the seek-past-buffered-range retry latch, and the last * runtime-applied volume — so the next `syncRuntimeMedia` tick treats it as * a first tick (hard resync, fresh drift baseline) instead of comparing * against state computed for a different file. Used both when a clip leaves * its active window (below) and by the runtime's proxy-swap helper * (mediaProxy.ts) right after an in-place `src` swap, which points the same * element at a different file without ever leaving its active window. */ export function evictMediaSyncState(el: HTMLMediaElement): void { lastOffset.delete(el); lastRelativeTime.delete(el); strictDriftSamples.delete(el); seekLoadRetried.delete(el); lastRuntimeAppliedVolume.delete(el); } /** Test-only seam: whether any per-source sync state is still tracked for `el`. */ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean { return ( lastOffset.has(el) || lastRelativeTime.has(el) || strictDriftSamples.has(el) || seekLoadRetried.has(el) || lastRuntimeAppliedVolume.has(el) ); } // fallow-ignore-next-line complexity export function syncRuntimeMedia(params: { clips: RuntimeMediaClip[]; timeSeconds: number; playing: boolean; playbackRate: number; /** Force-mute every element (parent-frame proxy owns all audio). Asserted per * tick so sub-composition media added mid-playback inherits the silence. */ outputMuted?: boolean; /** * User's explicit mute preference (set via `onSetMuted`). Symmetric to * `outputMuted` — also asserted per tick — so a sub-composition that * activates after the user mutes doesn't briefly play at author volume * before the next bridge message lands. */ userMuted?: boolean; /** * User's volume preference (0–1, set via `onSetVolume`). Multiplied with the * per-clip author volume so `data-volume="0.5"` at user volume 0.8 yields 0.4. */ userVolume?: number; /** * Invoked at most once when a media element's `play()` promise rejects with * `NotAllowedError`. The caller is expected to latch and post a single * outbound message; further invocations are suppressed by the caller. */ onAutoplayBlocked?: () => void; onElementVolume?: (el: HTMLMediaElement, effectiveVolume: number, authorVolume: number) => void; /** Is THIS element owned by the Web Audio transport? Owned → mute it (transport * plays it); not owned → leave audible (HTMLMedia fallback). Per-element, not a * global flag, so a not-yet-claimed track isn't muted by other tracks. */ isWebAudioOwned?: (el: HTMLMediaElement) => boolean; /** Native media routed through WebAudio keeps its upstream element volume at * unity; do not mistake that transport write for an authored volume edit. */ isWebAudioRouted?: (el: HTMLMediaElement) => boolean; /** "Hear only this" gate for the HTMLMedia fallback path (video / any audio * not owned by the Web Audio transport, which applies its own dedicated * solo gain instead — see `WebAudioTransport.setSolo`). Absent when solo * isn't wired up at all, which reads as "always audible". */ isAudibleUnderSolo?: (el: HTMLMediaElement) => boolean; forceSync?: boolean; }): void { const forceMuteAll = !!(params.outputMuted || params.userMuted); for (const clip of params.clips) { const { el } = clip; if (!el.isConnected) continue; let relTime = (params.timeSeconds - clip.start) * clip.playbackRate + clip.mediaStart; const isNonLoopVideo = el.tagName === "VIDEO" && !clip.loop; const isHeldVideoTail = isNonLoopVideo && clip.sourceDuration != null && relTime >= clip.sourceDuration && params.timeSeconds >= clip.start && params.timeSeconds < clip.end; if (isHeldVideoTail && clip.sourceDuration != null) { relTime = clip.sourceDuration; } const previousRelativeTime = lastRelativeTime.get(el); const audioReenteredAfterBackwardSeek = el.tagName === "AUDIO" && (previousRelativeTime === undefined || relTime < previousRelativeTime - 0.04); const canSeekEndedMediaBackward = !clip.loop && clip.sourceDuration != null && relTime >= clip.mediaStart && relTime < clip.sourceDuration && (isNonLoopVideo || audioReenteredAfterBackwardSeek); // Ended media can re-enter playable source after a backward timeline seek // without depending on the browser having reset `ended` first. Audio needs // a fresh activation or a measured backward transport seek so ordinary EOF // and non-seek force-sync transitions cannot replay its tail. A non-loop // video additionally remains an active visual through // its authored window, with tail seeks clamped to the final frame. const isActive = params.timeSeconds >= clip.start && params.timeSeconds < clip.end && relTime >= 0 && (!el.ended || clip.loop || isHeldVideoTail || canSeekEndedMediaBackward); if (isActive) { lastRelativeTime.set(el, relTime); // Loop wrapping: when media reaches end, restart from mediaStart if (clip.loop && clip.sourceDuration != null && clip.sourceDuration > 0) { const loopLength = clip.sourceDuration - clip.mediaStart; if (loopLength > 0 && relTime >= clip.sourceDuration) { relTime = clip.mediaStart + ((relTime - clip.mediaStart) % loopLength); } } const userVol = clampVolume(params.userVolume ?? 1); const fallbackAuthorVolume = clampAudioGain(clip.volume ?? 1); const previousRuntimeVolume = lastRuntimeAppliedVolume.get(el); const currentElementVolume = clampVolume(el.volume); let authorVolume: number; // An explicit volume lane owns the fader. It is checked before the probed // keyframes because the two would otherwise fight, and it is the one the // author drew — `lint` warns when a track carries both. // Clip-local, NOT `relTime`. A lane's `t` is "seconds from the start of // the clip" (see HfAutomationPoint), and the render honours that: the wav // is already cut with `-ss mediaStart`, so its t=0 IS the clip's start. // `relTime` is MEDIA time — it carries mediaStart, scales by playbackRate // and wraps on a loop — so feeding it here played the envelope at a // different position than it renders, or ran it off the end entirely on a // trimmed clip. The FX lanes on this same feature use clip-local elapsed; // there is one time base, and this is it. const laneGain = elementVolumeLaneGain(el, params.timeSeconds - clip.start); if (laneGain !== null) { authorVolume = clampAudioGain(laneGain); } else if (clip.volumeKeyframes && clip.volumeKeyframes.length > 0) { // Keyframes probed from the GSAP timeline — same source as the renderer. // Use the interpolated envelope value directly; no need to track GSAP changes. // Index by elapsed time on the TIMELINE since the clip began, which is what // a normalised envelope is keyed by (and what the renderer's PCM baker uses). // `relTime` is a position inside the media SOURCE — it carries `mediaStart` // and the playback rate — so it only coincides with the envelope's time base // for an untrimmed clip playing at 1x from t=0. const elapsedInClip = params.timeSeconds - clip.start; authorVolume = clampAudioGain(interpolateVolumeGain(clip.volumeKeyframes, elapsedInClip)); } else if (params.isWebAudioRouted?.(el)) { authorVolume = fallbackAuthorVolume; } else if (previousRuntimeVolume === undefined) { // First tick this clip is active. The transport has already seeked GSAP // to the current time (seekTimelineAndAdapters runs before syncRuntimeMedia), // so el.volume reflects the animated value — trust it rather than falling // back to data-volume, which would clobber the GSAP-seeked position. // // Except above unity. `el.volume` is spec-bound to [0,1], so it cannot // represent an authored boost, and reading it back can only lose the // gain. Without this, a boosted clip opened at 0 dB for one tick and // then jumped once the unchanged-since-last-tick branch below took over // — audible, and invisible to any test that ticks more than once. authorVolume = fallbackAuthorVolume > 1 ? fallbackAuthorVolume : currentElementVolume; } else if (Math.abs(currentElementVolume - previousRuntimeVolume) > 0.0001) { // GSAP (or user code) changed el.volume between ticks — track it. // // Unity-capped on purpose, and it is not a hole in the ceiling: this // reads back through `el.volume`, which the spec pins to [0,1], so it // cannot observe an above-unity value however wide the clamp gets. A // clip whose volume is actually animated takes the probed-keyframes // branch above, which carries the authored gain unclamped; this branch // is the fallback for elements no probe ran on. authorVolume = currentElementVolume; } else { // Volume unchanged since last tick — use data-volume as the baseline. authorVolume = fallbackAuthorVolume; } // A data-hidden ancestor is silent in the export (audioMixer.ts drops // it); preview must match. Folded into the per-tick volume, not // el.muted (RULES trap: el.muted is the transport's ownership flag). // Solo rides the same fold for the same reason — never el.muted, and // never touching any attribute (it is session-only, unlike hidden). const silencedBySolo = params.isAudibleUnderSolo ? !params.isAudibleUnderSolo(el) : false; const effectiveVolume = el.closest("[data-hidden]") || silencedBySolo ? 0 : clampVolume(authorVolume * userVol); el.volume = effectiveVolume; lastRuntimeAppliedVolume.set(el, effectiveVolume); params.onElementVolume?.(el, effectiveVolume, authorVolume); // Mute only when force-muted or the transport owns this element; an unclaimed // track stays audible via the HTMLMedia fallback. if (forceMuteAll || params.isWebAudioOwned?.(el)) el.muted = true; // Ensure full preload for every active media element. Streaming // formats (MP3) may arrive with preload="metadata", which only // buffers the first few seconds and causes seeks to silently fail // past the buffered range. Setting this on every tick is cheap // (no-op when already "auto") and catches elements whose preload // was overridden after init.ts set it. if (el.preload !== "auto") el.preload = "auto"; try { // Per-element rate × global transport rate el.playbackRate = clip.playbackRate * params.playbackRate; } catch (err) { // ignore unsupported playbackRate swallow("runtime.media.site1", err); } // Drift correction — three tiers: // // 1. Hard sync (0.5s): first tick, timeline jumps (scrub), catastrophic // drift (>3s). Unconditional seek — accepts brief rebuffer cost. // Forcing el.currentTime every frame causes audible seek hiccups // (readyState drops briefly), so we only hard-seek when necessary. // // 2. Strict sync (40ms, 2 consecutive samples): catches accumulated // drift from pause/play toggling or browser media pipeline latency. // Offset-stabilization guard (4ms/tick) prevents false corrections // during initial buffering where offset grows naturally. // // 3. Force sync (20ms): on play/pause/seek/rate transitions, correct // any drift >20ms immediately via the forceSync one-shot flag. // // The first tick a clip is active has no previous offset to compare — // treated as hard resync so sub-compositions with non-zero mediaStart // land on the right frame. const STRICT_DRIFT_THRESHOLD = 0.04; const STRICT_REQUIRED_SAMPLES = 2; const currentElTime = el.currentTime || 0; const drift = Math.abs(currentElTime - relTime); const offset = relTime - currentElTime; const prevOffset = lastOffset.get(el); lastOffset.set(el, offset); const firstTickOfClip = prevOffset === undefined; const offsetJumped = !firstTickOfClip && Math.abs(offset - prevOffset!) > 0.5; const catastrophicDrift = drift > 3; // A short audio clip can leave its native element paused just before EOF. // When the timeline re-enters the clip, rewind stale forward state at the // strict threshold; do not force cold audio forward while it buffers. const staleAudioOnFirstTick = el.tagName === "AUDIO" && firstTickOfClip && currentElTime - relTime > STRICT_DRIFT_THRESHOLD; const hardSync = (isHeldVideoTail && drift > 0.001) || (el.ended && canSeekEndedMediaBackward && drift > 0.001) || staleAudioOnFirstTick || (drift > 0.5 && (firstTickOfClip || offsetJumped || catastrophicDrift)); // Playing video elements use the browser's native decoder pipeline for // timing. Seeking a playing video resets the decoder, causing a ~150ms // freeze while it re-buffers — during which the monotonic clock advances, // creating a perpetual seek→freeze→drift→seek stutter loop. Skip strict // and force sync for playing videos; only hard sync (>0.5s) warrants // the decoder-reset cost. const isPlayingVideo = el.tagName === "VIDEO" && !el.paused; // Only apply strict sync when offset has stabilized (not growing). // During initial buffering, offset grows ~16ms/tick as the timeline // advances while media stays at 0. Accumulated drift from pause/play // toggling shows up as a stable, non-zero offset (delta near 0). const offsetStabilized = prevOffset !== undefined && Math.abs(offset - prevOffset) < 0.004; let strictSync = false; if ( !isPlayingVideo && !hardSync && !firstTickOfClip && offsetStabilized && drift > STRICT_DRIFT_THRESHOLD ) { const samples = (strictDriftSamples.get(el) ?? 0) + 1; strictDriftSamples.set(el, samples); if (samples >= STRICT_REQUIRED_SAMPLES) { strictSync = true; strictDriftSamples.set(el, 0); } } else if (drift <= STRICT_DRIFT_THRESHOLD) { strictDriftSamples.set(el, 0); } const forceSync = !isPlayingVideo && params.forceSync && drift > 0.02; if (hardSync || strictSync || forceSync) { // Skip the per-tick seek (and the `el.load()` drift-recovery retry // below) for `