From 87955b890e92ec94bbf77f82b6fc567d16127e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Mon, 16 Nov 2026 16:10:12 +0000 Subject: [PATCH] fix(studio): beat analysis never picks a recording longer than a song (#7944) * fix(studio): beat analysis never picks a recording longer than a song * fix(studio): a long music track keeps its saved beats; the cap judges the file alone * fix(studio): a music track's beat edits survive its file length arriving --- .../src/hooks/useMusicBeatAnalysis.test.tsx | 131 ++++++++++++++++++ .../studio/src/hooks/useMusicBeatAnalysis.ts | 117 +++++++--------- .../src/utils/timelineInspector.test.ts | 51 ++++++- .../studio/src/utils/timelineInspector.ts | 23 ++- 4 files changed, 252 insertions(+), 70 deletions(-) create mode 100644 packages/studio/src/hooks/useMusicBeatAnalysis.test.tsx diff --git a/packages/studio/src/hooks/useMusicBeatAnalysis.test.tsx b/packages/studio/src/hooks/useMusicBeatAnalysis.test.tsx new file mode 100644 index 0000000000..296a371a21 --- /dev/null +++ b/packages/studio/src/hooks/useMusicBeatAnalysis.test.tsx @@ -0,0 +1,131 @@ +// @vitest-environment happy-dom +import React, { act } from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { TimelineElement } from "../player"; +import { usePlayerStore } from "../player/store/playerStore"; +import { FileManagerProvider } from "../contexts/FileManagerContext"; +import { mountReactHarness } from "./domSelectionTestHarness"; + +const analyze = vi.hoisted(() => vi.fn()); +vi.mock("@hyperframes/core/beats", async (actual) => ({ + ...(await actual()), + analyzeMusicFromUrl: analyze, +})); + +const { useMusicBeatAnalysis } = await import("./useMusicBeatAnalysis"); +const { serializeBeats } = await import("@hyperframes/core/beats"); + +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +const clip = (overrides: Partial) => + ({ + id: "talk", + domId: "talk", + key: "talk", + tag: "audio", + src: "assets/talk.m4a", + start: 0, + duration: 180, + track: 3, + ...overrides, + }) as TimelineElement; + +let savedBeats = ""; +const files = { + readOptionalProjectFile: async () => savedBeats, + writeProjectFile: async () => {}, +} as unknown as React.ComponentProps["value"]; + +function Probe() { + useMusicBeatAnalysis(); + return null; +} + +/** Studio's timeline opened on `elements`: which files beat analysis decoded, and the beat times it shows. The hook + * caches an analysis per file, so each case names its own. */ +async function opened(elements: TimelineElement[]) { + usePlayerStore.setState({ elements, beatAnalysis: null }); + const root = mountReactHarness( + + + , + ); + await act(async () => { + for (let i = 0; i < 5; i += 1) await Promise.resolve(); + }); + const beats = usePlayerStore.getState().beatAnalysis?.beatTimes ?? null; + act(() => root.unmount()); + return { decoded: analyze.mock.calls.map(([src]) => src), beats }; +} + +beforeEach(() => { + savedBeats = ""; + analyze.mockReset(); + analyze.mockResolvedValue({ beatTimes: [1, 2], beatStrengths: [1, 1], bpm: 120 }); +}); +afterEach(() => + usePlayerStore.setState({ elements: [], beatEdits: null, beatUndo: [], beatRedo: [] }), +); + +// fails without the fix: a 62-minute recording with no role was picked as the music and decoded whole (1.4 GB on +// a Mac, which ran a person's page out of memory while their talk was still being transcribed). +describe("beat analysis decodes only a file no longer than a song", () => { + it("leaves a 62-minute unlabelled recording alone", async () => { + const talk = clip({ src: "assets/talk.m4a", duration: 3720, sourceDuration: 3720 }); + expect((await opened([talk])).decoded).toEqual([]); + }); + + it("leaves a 3-minute clip of a 62-minute file alone, since the decode reads the whole file", async () => { + const cut = clip({ src: "assets/cut.m4a", duration: 180, sourceDuration: 3720 }); + expect((await opened([cut])).decoded).toEqual([]); + }); + + it("leaves a recording alone while its file's length is unknown", async () => { + expect((await opened([clip({ src: "assets/unread.m4a", duration: 180 })])).decoded).toEqual([]); + }); + + it("shows a 62-minute music track's saved beats without decoding it", async () => { + savedBeats = serializeBeats([1, 2], [1, 1], "assets/mix.m4a"); + const mix = clip({ + src: "assets/mix.m4a", + timelineRole: "music", + duration: 3720, + sourceDuration: 3720, + }); + expect(await opened([mix])).toEqual({ decoded: [], beats: [1, 2] }); + }); + + it("still finds the beats of a 3-minute song", async () => { + const song = clip({ src: "assets/song.mp3", sourceDuration: 180 }); + expect(await opened([song])).toEqual({ decoded: ["assets/song.mp3"], beats: [1, 2] }); + }); +}); + +// fails without the fix: the length arriving after the saved beats showed loaded the track again, which dropped the +// person's beat edit and its undo. +it("keeps a music track's beat edits and undo when its file's length arrives", async () => { + savedBeats = serializeBeats([1, 2], [1, 1], "assets/edited.m4a"); + const music = { src: "assets/edited.m4a", timelineRole: "music" as const, duration: 180 }; + const settle = () => + act(async () => { + for (let i = 0; i < 5; i += 1) await Promise.resolve(); + }); + usePlayerStore.setState({ elements: [clip(music)], beatAnalysis: null, beatEdits: null }); + const root = mountReactHarness( + + + , + ); + await settle(); + const edit = { src: "assets/edited.m4a", added: [{ time: 1.5, strength: 1 }], removed: [] }; + act(() => usePlayerStore.getState().commitBeatEdits(edit, "add beat")); + act(() => usePlayerStore.setState({ elements: [clip({ ...music, sourceDuration: 180 })] })); + await settle(); + const { beatEdits, beatUndo, beatAnalysis } = usePlayerStore.getState(); + act(() => root.unmount()); + expect({ beatEdits, undo: beatUndo.length, beats: beatAnalysis?.beatTimes }).toEqual({ + beatEdits: edit, + undo: 1, + beats: [1, 2], + }); +}); diff --git a/packages/studio/src/hooks/useMusicBeatAnalysis.ts b/packages/studio/src/hooks/useMusicBeatAnalysis.ts index e71339ba99..d88a3c008c 100644 --- a/packages/studio/src/hooks/useMusicBeatAnalysis.ts +++ b/packages/studio/src/hooks/useMusicBeatAnalysis.ts @@ -1,6 +1,6 @@ import { useEffect, useMemo, useRef } from "react"; import { usePlayerStore } from "../player/store/playerStore"; -import { resolveBeatSourceTrack } from "../utils/timelineInspector"; +import { fitsBeatAnalysis, resolveBeatSourceTrack } from "../utils/timelineInspector"; import { analyzeMusicFromUrl } from "@hyperframes/core/beats"; import { whenTimelineIdle } from "../player/components/timelineMotion"; import { useFileManagerContextOptional } from "../contexts/FileManagerContext"; @@ -32,67 +32,66 @@ function cacheAnalysis(url: string, promise: ReturnType Promise }; -/** - * Resolve the effective beat list for a track: a saved file with real beats - * wins; otherwise the detected beats are used (and `hasFile` is false so the - * caller seeds a new file). An empty saved file is ignored so detection retries. - */ -async function resolveBeats( - beatPath: string | null, - detected: { times: number[]; strengths: number[] }, - io: ProjectIo, -): Promise<{ times: number[]; strengths: number[]; hasFile: boolean }> { - if (!beatPath) return { ...detected, hasFile: false }; - try { - const content = await io.readOptionalProjectFile(beatPath); - const parsed = content ? parseBeats(content) : null; - if (parsed && parsed.times.length > 0) { - return { times: parsed.times, strengths: parsed.strengths, hasFile: true }; - } - } catch { - /* fall back to detected beats */ - } - return { ...detected, hasFile: false }; -} +type Beats = { times: number[]; strengths: number[] }; -/** True when the beats file for a track exists and holds at least one beat. */ -async function readHasSavedBeats(io: ProjectIo, beatPath: string): Promise { +/** The track's saved beats file, or null when it is missing, unreadable or holds no beat (detection then retries). */ +async function readSavedBeats(io: ProjectIo, beatPath: string): Promise { try { const content = await io.readOptionalProjectFile(beatPath); const parsed = content ? parseBeats(content) : null; - return !!(parsed && parsed.times.length > 0); + return parsed && parsed.times.length > 0 + ? { times: parsed.times, strengths: parsed.strengths } + : null; } catch { - return false; + return null; } } type MusicAnalysis = Awaited>; -/** - * Analyze a track (memoized per URL) and fold in any saved beat edits. Null on - * decode/analysis failure — the caller then clears state and drops the cache - * entry (only when the effect is still live). - */ -async function loadBeatAnalysis( - musicSrc: string, - beatPath: string, - io: ProjectIo, -): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> { +type Loaded = { analysis: MusicAnalysis } & Beats; + +/** A track's analysis (memoized per URL), or null when decode or analysis fails. */ +async function decodeTrack(musicSrc: string): Promise { let promise = analysisCache.get(musicSrc); if (!promise) { promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle }); cacheAnalysis(musicSrc, promise); } - try { - const analysis = await promise; - const detected = { times: analysis.beatTimes, strengths: analysis.beatStrengths }; - const { times, strengths } = await resolveBeats(beatPath, detected, io); - return { analysis, times, strengths }; - } catch { - return null; - } + return promise.catch(() => null); } +/** What a track shows. A fallback track (audio dropped in with no role) is analysed so the Beat tool works at once; a + * marked music track only once the person saved beats for it, so unrelated assets add no beat lines. A file that does + * not fit the decode keeps its saved beats, with no strengths measured from the audio. */ +async function loadTrack( + musicSrc: string, + beatPath: string, + io: ProjectIo, + isFallbackTrack: boolean, + decodable: boolean, +): Promise { + const saved = await readSavedBeats(io, beatPath); + if (!isFallbackTrack && !saved) return null; + if (!decodable) return saved && { analysis: savedOnly(saved), ...saved }; + const analysis = await decodeTrack(musicSrc); + if (!analysis) return null; + return { + analysis, + ...(saved ?? { times: analysis.beatTimes, strengths: analysis.beatStrengths }), + }; +} + +const savedOnly = ({ times, strengths }: Beats): MusicAnalysis => ({ + beatTimes: times, + beatStrengths: strengths, + bpm: null, + bpmConfidence: "uncertain", + channelData: null, + sampleRate: 0, + peak: 0, +}); + /** The current beat analysis, remapped onto the composition's edited beat grid. */ export function useAdjustedBeatAnalysis() { const beatAnalysis = usePlayerStore((s) => s.beatAnalysis); @@ -123,16 +122,19 @@ export function useMusicBeatAnalysis(): void { ? { readOptionalProjectFile, writeProjectFile } : null; - const { musicSrc, isFallbackTrack } = useMemo(() => { + const { musicSrc, isFallbackTrack, decodable } = useMemo(() => { const resolved = resolveBeatSourceTrack(elements); return { musicSrc: resolved?.element.src ?? null, isFallbackTrack: resolved?.isFallback ?? false, + decodable: resolved ? fitsBeatAnalysis(resolved.element) : false, }; }, [elements]); + // Read at load, never a reason to load again: a file length arriving for the same track must keep its edits. + const decodableRef = useRef(decodable); + decodableRef.current = decodable; - // ── Load: decode for strength data, then use the saved beat file if present, - // otherwise seed it from detection. Resets edits + history on track change. ── + // ── Load: the track's beats (loadTrack). Resets edits + history on track change. ── useEffect(() => { if (!musicSrc) { setBeatAnalysis(null); @@ -150,26 +152,9 @@ export function useMusicBeatAnalysis(): void { const beatPath = beatFilePathForSrc(musicSrc); const io = ioRef.current; - // For explicitly tagged/named music tracks: only run expensive audio decode - // + beat analysis when the user has an explicit beats file saved. Without - // one, skip entirely — no surprise green lines on the timeline after - // dragging unrelated assets. - // - // For fallback tracks (audio dropped from Finder with no role/music-id): - // always run analysis so the Beat tool becomes usable immediately. (async () => { if (!beatPath || !io) return; - if (!isFallbackTrack) { - const hasSavedBeats = await readHasSavedBeats(io, beatPath); - if (cancelled) return; - if (!hasSavedBeats) { - setBeatAnalysis(null); - return; - } - } - if (cancelled) return; - - const result = await loadBeatAnalysis(musicSrc, beatPath, io); + const result = await loadTrack(musicSrc, beatPath, io, isFallbackTrack, decodableRef.current); if (cancelled) return; if (!result) { setBeatAnalysis(null); diff --git a/packages/studio/src/utils/timelineInspector.test.ts b/packages/studio/src/utils/timelineInspector.test.ts index 27263980a8..509415ddca 100644 --- a/packages/studio/src/utils/timelineInspector.test.ts +++ b/packages/studio/src/utils/timelineInspector.test.ts @@ -5,7 +5,9 @@ import { isAudioDomElement, isAudioTimelineElement, isMusicSourceElement, + fitsBeatAnalysis, isMusicTrack, + MAX_BEAT_SOURCE_SECONDS, resolveBeatSourceTrack, } from "./timelineInspector"; import type { TimelineElement } from "../player"; @@ -15,12 +17,28 @@ function el( overrides: Partial< Pick< TimelineElement, - "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted" + | "tag" + | "src" + | "id" + | "domId" + | "timelineRole" + | "duration" + | "sourceDuration" + | "hasAudio" + | "muted" > >, ): Pick< TimelineElement, - "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted" + | "tag" + | "src" + | "id" + | "domId" + | "timelineRole" + | "duration" + | "sourceDuration" + | "hasAudio" + | "muted" > { return { tag: "audio", @@ -29,6 +47,7 @@ function el( domId: "el-1", timelineRole: undefined, duration: 10, + sourceDuration: 120, ...overrides, }; } @@ -127,6 +146,22 @@ describe("resolveBeatSourceTrack", () => { expect(resolveBeatSourceTrack([sfx, vo])).toBeNull(); }); + it("falls back to a song over a longer clip whose file's length is not known yet", () => { + const unread = el({ id: "unread", domId: "unread", duration: 300, sourceDuration: undefined }); + const song = el({ id: "song", domId: "song", duration: 180, sourceDuration: 180 }); + expect(resolveBeatSourceTrack([unread, song])!.element).toBe(song); + const looped = el({ id: "loop", domId: "loop", duration: 1200, sourceDuration: 180 }); + expect(resolveBeatSourceTrack([looped])!.element).toBe(looped); + }); + + it("never falls back to a clip, or a clip's file, longer than a song", () => { + const talk = el({ id: "talk", domId: "talk", duration: 3720, sourceDuration: 3720 }); + const excerpt = el({ id: "cut", domId: "cut", duration: 180, sourceDuration: 3720 }); + expect(resolveBeatSourceTrack([talk, excerpt])).toBeNull(); + const song = el({ id: "song", domId: "song", duration: 200, sourceDuration: 200 }); + expect(resolveBeatSourceTrack([talk, song])!.element).toBe(song); + }); + it("returns isFallback=true for an untagged clip with a non-music id", () => { const clip = el({ id: "my_audio_file", domId: "my_audio_file", timelineRole: undefined }); const result = resolveBeatSourceTrack([clip]); @@ -184,3 +219,15 @@ describe("beat source from a video with sound", () => { expect(resolveBeatSourceTrack([short, long, muted])?.element).toBe(long); }); }); + +describe("fitsBeatAnalysis", () => { + it("fits a file whose length is known and no longer than a song, however long its clip runs", () => { + expect(fitsBeatAnalysis({ sourceDuration: 180 })).toBe(true); + expect(fitsBeatAnalysis({ sourceDuration: MAX_BEAT_SOURCE_SECONDS })).toBe(true); + }); + + it("refuses a file past the cap and a file whose length is unknown", () => { + expect(fitsBeatAnalysis({ sourceDuration: 3720 })).toBe(false); + expect(fitsBeatAnalysis({})).toBe(false); + }); +}); diff --git a/packages/studio/src/utils/timelineInspector.ts b/packages/studio/src/utils/timelineInspector.ts index 1f652b123c..2c1fdef408 100644 --- a/packages/studio/src/utils/timelineInspector.ts +++ b/packages/studio/src/utils/timelineInspector.ts @@ -80,6 +80,16 @@ export function isMusicTrack( return MUSIC_ID_RE.test(id); } +/** Past a song's length a track is speech or a mix, and beat analysis decodes its whole file (62 min: 1.4 GB). */ +export const MAX_BEAT_SOURCE_SECONDS = 15 * 60; + +/** The decode reads the whole file, so a source fits once its file's length is known and within the cap. */ +export function fitsBeatAnalysis({ + sourceDuration, +}: Pick): boolean { + return sourceDuration != null && sourceDuration <= MAX_BEAT_SOURCE_SECONDS; +} + /** * Resolve the best audio source for beat analysis. An explicitly tagged or * named music track wins; when none is present (e.g. an audio file dropped @@ -94,18 +104,27 @@ export function isMusicTrack( export function resolveBeatSourceTrack( elements: readonly Pick< TimelineElement, - "tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted" + | "tag" + | "src" + | "id" + | "domId" + | "timelineRole" + | "duration" + | "sourceDuration" + | "hasAudio" + | "muted" >[], ): { element: (typeof elements)[number]; isFallback: boolean } | null { const explicit = elements.find(isMusicTrack); if (explicit) return { element: explicit, isFallback: false }; // Fallback: pick the longest audio clip (skipping explicitly non-music roles - // like "sfx" or "voiceover" to avoid triggering beat analysis on those). + // like "sfx" or "voiceover", and anything longer than a song, so none is beat-analysed). let best: (typeof elements)[number] | null = null; for (const el of elements) { if (!isMusicSourceElement(el)) continue; if (el.timelineRole && el.timelineRole !== "music") continue; + if (!fitsBeatAnalysis(el)) continue; if (!best || el.duration > best.duration) best = el; } return best ? { element: best, isFallback: true } : null;