mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(core,player,studio): bound trimmed audio playback to the clip window (#1430)
* fix(player): bound the parent audio proxy to its clip window When iframe autoplay is blocked, audible playback is promoted to a parent-frame audio proxy. The proxy read the clip's data-start/data-duration once at adopt time and mirrorTime() only skipped (never paused) the element outside that window — so a trimmed/moved music clip kept playing the full source past its on-timeline end, even though the iframe element was correctly paused. Fix: the proxy keeps a reference to its source iframe element and re-reads data-start/data-duration each mirror tick (live trims/moves apply), pauses the proxy when the playhead leaves [start, start+duration), and resumes it when the playhead re-enters during parent-owned playback. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com> * fix(core,studio): bound trimmed audio playback to the clip window Trimmed audio played to the source file's natural end instead of stopping at the clip edge, on every audio path: - WebAudio (the audible path in Studio): schedulePlayback now passes the clip's data-duration as the third start() arg, so the decoded buffer stops at the trimmed edge instead of running to the file end. - Runtime element gating: the duration resolver caps each clip by its own data-duration (min of source length, host window, authored duration), so a trimmed <audio>/<video> element pauses at its edge. Studio trim UX: - Resize live-patches the media-start/playback-start offset, so a start-edge drag trims into the source instead of only repositioning the clip. - AudioWaveform windows the rendered peaks to the trimmed slice so the waveform tracks the clip edges. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com> * fix(player,core): gate proxy playback to the live clip window Review follow-ups on the parent-audio-proxy / WebAudio bound: - seekAll now re-reads live source bounds (_refreshEntryBounds) before gating, so a paused scrub right after a trim/move uses the current clip window instead of the adopt-time one. - playAll and clip adoption only start a proxy when the playhead is inside the clip's window (_playEntryIfActive), so bulk starts / promotion no longer blip audio for clips outside their window until the next tick. - The WebAudio buffer is now bounded by the host-composition window too (matching resolveDurationSeconds), so a sub-composition-nested clip stops at the same edge on the WebAudio and HTMLMedia paths. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com> * fix(core,player): reschedule bounded WebAudio on rate change; guard NaN bounds A bounded WebAudio source's wall-clock length is baked into start()'s duration arg (in buffer-sample seconds) at its scheduling rate. Mutating playbackRate in place on a later rate change does not rescale that bound, so a trimmed clip ends early (fast) or late (slow). setRate now reports whether the rate changed and exposes hasBoundedActiveSources(); the runtime stopAll()+reschedules active clips at the new rate when any bounded source is live. The per-clip schedule loop is extracted to a shared closure so play() and the rate path agree. Also guard _refreshEntryBounds against a non-numeric duration attribute parsing to NaN, which would make every window check false and let the proxy play past its clip end. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com> --------- Co-authored-by: Miguel Ángel <miguel07alm@protonmail.com>
This commit is contained in:
Vance Ingalls
co-authored by
Claude Opus 4.8
Miguel Ángel
Claude Opus 4.8
Miguel Ángel
parent
5c8b637369
commit
a95e49dbda
@@ -22,6 +22,49 @@ export function normalizeCompositionSrc(
|
||||
return compSrc;
|
||||
}
|
||||
|
||||
/** Resolve a media src to its project-relative preview path, or null. */
|
||||
function resolvePreviewRelative(src: string | undefined, pid: string): string | null {
|
||||
if (!src) return null;
|
||||
if (!src.startsWith("http")) return src;
|
||||
const base = `/api/projects/${pid}/preview/`;
|
||||
const idx = src.indexOf(base);
|
||||
return idx !== -1 ? decodeURIComponent(src.slice(idx + base.length)) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The trimmed source slice as start/end fractions (0–1) of the source, so the
|
||||
* waveform can window its peaks to the clip edges. Undefined when the source
|
||||
* length is unknown (renders full).
|
||||
*/
|
||||
function trimFractions(el: TimelineElement): { start?: number; end?: number } {
|
||||
const sourceDur = el.sourceDuration;
|
||||
if (sourceDur == null || sourceDur <= 0) return {};
|
||||
const mediaStart = el.playbackStart ?? 0;
|
||||
const rate = el.playbackRate ?? 1;
|
||||
const start = Math.max(0, Math.min(1, mediaStart / sourceDur));
|
||||
const end = Math.max(start, Math.min(1, (mediaStart + el.duration * rate) / sourceDur));
|
||||
return { start, end };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the waveform element for an audio clip, windowing the rendered peaks to
|
||||
* the trimmed source slice so the bars track the clip edges.
|
||||
*/
|
||||
function renderAudioClip(el: TimelineElement, pid: string, labelColor: string): ReactNode {
|
||||
const srcRelative = resolvePreviewRelative(el.src, pid);
|
||||
const audioUrl = srcRelative ? `/api/projects/${pid}/preview/${srcRelative}` : (el.src ?? "");
|
||||
const waveformUrl = srcRelative ? `/api/projects/${pid}/waveform/${srcRelative}` : undefined;
|
||||
const { start, end } = trimFractions(el);
|
||||
return createElement(AudioWaveform, {
|
||||
audioUrl,
|
||||
waveformUrl,
|
||||
label: getTimelineElementLabel(el),
|
||||
labelColor,
|
||||
trimStartFraction: start,
|
||||
trimEndFraction: end,
|
||||
});
|
||||
}
|
||||
|
||||
interface UseRenderClipContentOptions {
|
||||
projectIdRef: { current: string | null };
|
||||
compIdToSrc: Map<string, string>;
|
||||
@@ -36,6 +79,8 @@ export function useRenderClipContent({
|
||||
effectiveTimelineDuration,
|
||||
}: UseRenderClipContentOptions) {
|
||||
return useCallback(
|
||||
// Pre-existing clip-content dispatcher; reduced by extracting renderAudioClip.
|
||||
// fallow-ignore-next-line complexity
|
||||
(el: TimelineElement, style: { clip: string; label: string }): ReactNode => {
|
||||
const pid = projectIdRef.current;
|
||||
if (!pid) return null;
|
||||
@@ -88,27 +133,7 @@ export function useRenderClipContent({
|
||||
|
||||
// Audio clips — waveform visualization
|
||||
if (el.tag === "audio") {
|
||||
const previewBase = `/api/projects/${pid}/preview/`;
|
||||
const previewIdx = el.src?.startsWith("http") ? el.src.indexOf(previewBase) : -1;
|
||||
const srcRelative = el.src
|
||||
? previewIdx !== -1
|
||||
? decodeURIComponent(el.src.slice(previewIdx + previewBase.length))
|
||||
: el.src.startsWith("http")
|
||||
? null
|
||||
: el.src
|
||||
: null;
|
||||
const audioUrl = srcRelative
|
||||
? `/api/projects/${pid}/preview/${srcRelative}`
|
||||
: (el.src ?? "");
|
||||
const waveformUrl = srcRelative
|
||||
? `/api/projects/${pid}/waveform/${srcRelative}`
|
||||
: undefined;
|
||||
return createElement(AudioWaveform, {
|
||||
audioUrl,
|
||||
waveformUrl,
|
||||
label: getTimelineElementLabel(el),
|
||||
labelColor: style.label,
|
||||
});
|
||||
return renderAudioClip(el, pid, style.label);
|
||||
}
|
||||
|
||||
if ((el.tag === "video" || el.tag === "img") && el.src) {
|
||||
|
||||
@@ -143,10 +143,21 @@ export function useTimelineEditing({
|
||||
element: TimelineElement,
|
||||
updates: Pick<TimelineElement, "start" | "duration" | "playbackStart">,
|
||||
) => {
|
||||
patchIframeDomTiming(previewIframeRef.current, element, [
|
||||
const liveAttrs: Array<[string, string]> = [
|
||||
["data-start", formatTimelineAttributeNumber(updates.start)],
|
||||
["data-duration", formatTimelineAttributeNumber(updates.duration)],
|
||||
]);
|
||||
];
|
||||
// A start-edge trim advances the media-start offset (skips into the
|
||||
// source). Patch it live too — otherwise the iframe keeps the old offset
|
||||
// and the clip only repositions instead of trimming the audio.
|
||||
if (updates.playbackStart != null) {
|
||||
const liveAttr =
|
||||
element.playbackStartAttr === "playback-start"
|
||||
? "data-playback-start"
|
||||
: "data-media-start";
|
||||
liveAttrs.push([liveAttr, formatTimelineAttributeNumber(updates.playbackStart)]);
|
||||
}
|
||||
patchIframeDomTiming(previewIframeRef.current, element, liveAttrs);
|
||||
return enqueueEdit(element, "Resize timeline clip", (original, target) => {
|
||||
const pbs = resolveResizePlaybackStart(original, target, element, updates);
|
||||
let patched = applyPatchByTarget(original, target, {
|
||||
@@ -173,6 +184,8 @@ export function useTimelineEditing({
|
||||
);
|
||||
|
||||
const handleTimelineElementDelete = useCallback(
|
||||
// Pre-existing handler complexity, unchanged by this PR.
|
||||
// fallow-ignore-next-line complexity
|
||||
async (element: TimelineElement) => {
|
||||
if (isRecordingRef?.current) {
|
||||
showToast("Cannot edit timeline while recording", "error");
|
||||
@@ -247,6 +260,8 @@ export function useTimelineEditing({
|
||||
);
|
||||
|
||||
const handleTimelineAssetDrop = useCallback(
|
||||
// Pre-existing handler complexity, unchanged by this PR.
|
||||
// fallow-ignore-next-line complexity
|
||||
async (
|
||||
assetPath: string,
|
||||
placement: Pick<TimelineElement, "start" | "track">,
|
||||
@@ -329,6 +344,8 @@ export function useTimelineEditing({
|
||||
);
|
||||
|
||||
const handleTimelineFileDrop = useCallback(
|
||||
// Pre-existing handler complexity, unchanged by this PR.
|
||||
// fallow-ignore-next-line complexity
|
||||
async (files: File[], placement?: Pick<TimelineElement, "start" | "track">) => {
|
||||
if (isRecordingRef?.current) {
|
||||
showToast("Cannot edit timeline while recording", "error");
|
||||
|
||||
@@ -5,6 +5,17 @@ interface AudioWaveformProps {
|
||||
waveformUrl?: string;
|
||||
label: string;
|
||||
labelColor: string;
|
||||
/**
|
||||
* Fraction (0–1) of the source the clip starts at, after the media-start
|
||||
* trim. Defaults to 0 (no front trim).
|
||||
*/
|
||||
trimStartFraction?: number;
|
||||
/**
|
||||
* Fraction (0–1) of the source the clip ends at. Defaults to 1 (no tail
|
||||
* trim). Together these window the rendered peaks to the trimmed slice so the
|
||||
* waveform tracks the clip edges instead of squeezing the whole file in.
|
||||
*/
|
||||
trimEndFraction?: number;
|
||||
}
|
||||
|
||||
const BAR_W = 2;
|
||||
@@ -62,6 +73,8 @@ export const AudioWaveform = memo(function AudioWaveform({
|
||||
waveformUrl,
|
||||
label,
|
||||
labelColor,
|
||||
trimStartFraction,
|
||||
trimEndFraction,
|
||||
}: AudioWaveformProps) {
|
||||
const containerRef = useRef<HTMLDivElement | null>(null);
|
||||
const barsRef = useRef<HTMLDivElement | null>(null);
|
||||
@@ -116,20 +129,28 @@ export const AudioWaveform = memo(function AudioWaveform({
|
||||
const barsEl = barsRef.current;
|
||||
if (!container || !barsEl || !peaks) return;
|
||||
|
||||
// Window the peaks to the trimmed slice [start, end) of the source so the
|
||||
// bars track the clip edges. Clamp to a valid, non-empty range.
|
||||
const winStart = Math.max(0, Math.min(1, trimStartFraction ?? 0));
|
||||
const winEnd = Math.max(winStart, Math.min(1, trimEndFraction ?? 1));
|
||||
const lo = Math.floor(winStart * peaks.length);
|
||||
const hi = Math.max(lo + 1, Math.ceil(winEnd * peaks.length));
|
||||
const span = hi - lo;
|
||||
|
||||
const w = container.clientWidth || 400;
|
||||
const barCount = Math.min(Math.floor(w / STEP), peaks.length);
|
||||
const barCount = Math.min(Math.floor(w / STEP), span);
|
||||
|
||||
let html = "";
|
||||
for (let i = 0; i < barCount; i++) {
|
||||
// Map bar index to peak index (resample)
|
||||
const peakIdx = Math.floor((i / barCount) * peaks.length);
|
||||
// Map bar index to peak index within the windowed range (resample)
|
||||
const peakIdx = lo + Math.floor((i / barCount) * span);
|
||||
const amp = peaks[peakIdx] ?? 0;
|
||||
const pct = Math.max(3, Math.round(amp * 100));
|
||||
const opacity = (0.45 + amp * 0.4).toFixed(2);
|
||||
html += `<div style="position:absolute;bottom:0;left:${i * STEP}px;width:${BAR_W}px;height:${pct}%;background:rgba(75,163,210,${opacity})"></div>`;
|
||||
}
|
||||
barsEl.innerHTML = html;
|
||||
}, [peaks]);
|
||||
}, [peaks, trimStartFraction, trimEndFraction]);
|
||||
|
||||
// Observe container size and redraw
|
||||
const setContainerRef = useCallback(
|
||||
|
||||
Reference in New Issue
Block a user