fix(studio): beat analysis never picks a recording longer than a song (#7944)

* fix(studio): beat analysis never picks a recording longer than a song

* fix(studio): a long music track keeps its saved beats; the cap judges the file alone

* fix(studio): a music track's beat edits survive its file length arriving
This commit is contained in:
Miguel Ángel
2026-11-16 16:10:12 +00:00
committed by GitHub
parent 071a2da0b5
commit 87955b890e
4 changed files with 252 additions and 70 deletions
@@ -0,0 +1,131 @@
// @vitest-environment happy-dom
import React, { act } from "react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { TimelineElement } from "../player";
import { usePlayerStore } from "../player/store/playerStore";
import { FileManagerProvider } from "../contexts/FileManagerContext";
import { mountReactHarness } from "./domSelectionTestHarness";
const analyze = vi.hoisted(() => vi.fn());
vi.mock("@hyperframes/core/beats", async (actual) => ({
...(await actual<typeof import("@hyperframes/core/beats")>()),
analyzeMusicFromUrl: analyze,
}));
const { useMusicBeatAnalysis } = await import("./useMusicBeatAnalysis");
const { serializeBeats } = await import("@hyperframes/core/beats");
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
const clip = (overrides: Partial<TimelineElement>) =>
({
id: "talk",
domId: "talk",
key: "talk",
tag: "audio",
src: "assets/talk.m4a",
start: 0,
duration: 180,
track: 3,
...overrides,
}) as TimelineElement;
let savedBeats = "";
const files = {
readOptionalProjectFile: async () => savedBeats,
writeProjectFile: async () => {},
} as unknown as React.ComponentProps<typeof FileManagerProvider>["value"];
function Probe() {
useMusicBeatAnalysis();
return null;
}
/** Studio's timeline opened on `elements`: which files beat analysis decoded, and the beat times it shows. The hook
* caches an analysis per file, so each case names its own. */
async function opened(elements: TimelineElement[]) {
usePlayerStore.setState({ elements, beatAnalysis: null });
const root = mountReactHarness(
<FileManagerProvider value={files}>
<Probe />
</FileManagerProvider>,
);
await act(async () => {
for (let i = 0; i < 5; i += 1) await Promise.resolve();
});
const beats = usePlayerStore.getState().beatAnalysis?.beatTimes ?? null;
act(() => root.unmount());
return { decoded: analyze.mock.calls.map(([src]) => src), beats };
}
beforeEach(() => {
savedBeats = "";
analyze.mockReset();
analyze.mockResolvedValue({ beatTimes: [1, 2], beatStrengths: [1, 1], bpm: 120 });
});
afterEach(() =>
usePlayerStore.setState({ elements: [], beatEdits: null, beatUndo: [], beatRedo: [] }),
);
// fails without the fix: a 62-minute recording with no role was picked as the music and decoded whole (1.4 GB on
// a Mac, which ran a person's page out of memory while their talk was still being transcribed).
describe("beat analysis decodes only a file no longer than a song", () => {
it("leaves a 62-minute unlabelled recording alone", async () => {
const talk = clip({ src: "assets/talk.m4a", duration: 3720, sourceDuration: 3720 });
expect((await opened([talk])).decoded).toEqual([]);
});
it("leaves a 3-minute clip of a 62-minute file alone, since the decode reads the whole file", async () => {
const cut = clip({ src: "assets/cut.m4a", duration: 180, sourceDuration: 3720 });
expect((await opened([cut])).decoded).toEqual([]);
});
it("leaves a recording alone while its file's length is unknown", async () => {
expect((await opened([clip({ src: "assets/unread.m4a", duration: 180 })])).decoded).toEqual([]);
});
it("shows a 62-minute music track's saved beats without decoding it", async () => {
savedBeats = serializeBeats([1, 2], [1, 1], "assets/mix.m4a");
const mix = clip({
src: "assets/mix.m4a",
timelineRole: "music",
duration: 3720,
sourceDuration: 3720,
});
expect(await opened([mix])).toEqual({ decoded: [], beats: [1, 2] });
});
it("still finds the beats of a 3-minute song", async () => {
const song = clip({ src: "assets/song.mp3", sourceDuration: 180 });
expect(await opened([song])).toEqual({ decoded: ["assets/song.mp3"], beats: [1, 2] });
});
});
// fails without the fix: the length arriving after the saved beats showed loaded the track again, which dropped the
// person's beat edit and its undo.
it("keeps a music track's beat edits and undo when its file's length arrives", async () => {
savedBeats = serializeBeats([1, 2], [1, 1], "assets/edited.m4a");
const music = { src: "assets/edited.m4a", timelineRole: "music" as const, duration: 180 };
const settle = () =>
act(async () => {
for (let i = 0; i < 5; i += 1) await Promise.resolve();
});
usePlayerStore.setState({ elements: [clip(music)], beatAnalysis: null, beatEdits: null });
const root = mountReactHarness(
<FileManagerProvider value={files}>
<Probe />
</FileManagerProvider>,
);
await settle();
const edit = { src: "assets/edited.m4a", added: [{ time: 1.5, strength: 1 }], removed: [] };
act(() => usePlayerStore.getState().commitBeatEdits(edit, "add beat"));
act(() => usePlayerStore.setState({ elements: [clip({ ...music, sourceDuration: 180 })] }));
await settle();
const { beatEdits, beatUndo, beatAnalysis } = usePlayerStore.getState();
act(() => root.unmount());
expect({ beatEdits, undo: beatUndo.length, beats: beatAnalysis?.beatTimes }).toEqual({
beatEdits: edit,
undo: 1,
beats: [1, 2],
});
});
@@ -1,6 +1,6 @@
import { useEffect, useMemo, useRef } from "react";
import { usePlayerStore } from "../player/store/playerStore";
import { resolveBeatSourceTrack } from "../utils/timelineInspector";
import { fitsBeatAnalysis, resolveBeatSourceTrack } from "../utils/timelineInspector";
import { analyzeMusicFromUrl } from "@hyperframes/core/beats";
import { whenTimelineIdle } from "../player/components/timelineMotion";
import { useFileManagerContextOptional } from "../contexts/FileManagerContext";
@@ -32,67 +32,66 @@ function cacheAnalysis(url: string, promise: ReturnType<typeof analyzeMusicFromU
type ProjectIo = { readOptionalProjectFile: (p: string) => Promise<string> };
/**
* Resolve the effective beat list for a track: a saved file with real beats
* wins; otherwise the detected beats are used (and `hasFile` is false so the
* caller seeds a new file). An empty saved file is ignored so detection retries.
*/
async function resolveBeats(
beatPath: string | null,
detected: { times: number[]; strengths: number[] },
io: ProjectIo,
): Promise<{ times: number[]; strengths: number[]; hasFile: boolean }> {
if (!beatPath) return { ...detected, hasFile: false };
try {
const content = await io.readOptionalProjectFile(beatPath);
const parsed = content ? parseBeats(content) : null;
if (parsed && parsed.times.length > 0) {
return { times: parsed.times, strengths: parsed.strengths, hasFile: true };
}
} catch {
/* fall back to detected beats */
}
return { ...detected, hasFile: false };
}
type Beats = { times: number[]; strengths: number[] };
/** True when the beats file for a track exists and holds at least one beat. */
async function readHasSavedBeats(io: ProjectIo, beatPath: string): Promise<boolean> {
/** The track's saved beats file, or null when it is missing, unreadable or holds no beat (detection then retries). */
async function readSavedBeats(io: ProjectIo, beatPath: string): Promise<Beats | null> {
try {
const content = await io.readOptionalProjectFile(beatPath);
const parsed = content ? parseBeats(content) : null;
return !!(parsed && parsed.times.length > 0);
return parsed && parsed.times.length > 0
? { times: parsed.times, strengths: parsed.strengths }
: null;
} catch {
return false;
return null;
}
}
type MusicAnalysis = Awaited<ReturnType<typeof analyzeMusicFromUrl>>;
/**
* Analyze a track (memoized per URL) and fold in any saved beat edits. Null on
* decode/analysis failure — the caller then clears state and drops the cache
* entry (only when the effect is still live).
*/
async function loadBeatAnalysis(
musicSrc: string,
beatPath: string,
io: ProjectIo,
): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> {
type Loaded = { analysis: MusicAnalysis } & Beats;
/** A track's analysis (memoized per URL), or null when decode or analysis fails. */
async function decodeTrack(musicSrc: string): Promise<MusicAnalysis | null> {
let promise = analysisCache.get(musicSrc);
if (!promise) {
promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle });
cacheAnalysis(musicSrc, promise);
}
try {
const analysis = await promise;
const detected = { times: analysis.beatTimes, strengths: analysis.beatStrengths };
const { times, strengths } = await resolveBeats(beatPath, detected, io);
return { analysis, times, strengths };
} catch {
return null;
}
return promise.catch(() => null);
}
/** What a track shows. A fallback track (audio dropped in with no role) is analysed so the Beat tool works at once; a
* marked music track only once the person saved beats for it, so unrelated assets add no beat lines. A file that does
* not fit the decode keeps its saved beats, with no strengths measured from the audio. */
async function loadTrack(
musicSrc: string,
beatPath: string,
io: ProjectIo,
isFallbackTrack: boolean,
decodable: boolean,
): Promise<Loaded | null> {
const saved = await readSavedBeats(io, beatPath);
if (!isFallbackTrack && !saved) return null;
if (!decodable) return saved && { analysis: savedOnly(saved), ...saved };
const analysis = await decodeTrack(musicSrc);
if (!analysis) return null;
return {
analysis,
...(saved ?? { times: analysis.beatTimes, strengths: analysis.beatStrengths }),
};
}
const savedOnly = ({ times, strengths }: Beats): MusicAnalysis => ({
beatTimes: times,
beatStrengths: strengths,
bpm: null,
bpmConfidence: "uncertain",
channelData: null,
sampleRate: 0,
peak: 0,
});
/** The current beat analysis, remapped onto the composition's edited beat grid. */
export function useAdjustedBeatAnalysis() {
const beatAnalysis = usePlayerStore((s) => s.beatAnalysis);
@@ -123,16 +122,19 @@ export function useMusicBeatAnalysis(): void {
? { readOptionalProjectFile, writeProjectFile }
: null;
const { musicSrc, isFallbackTrack } = useMemo(() => {
const { musicSrc, isFallbackTrack, decodable } = useMemo(() => {
const resolved = resolveBeatSourceTrack(elements);
return {
musicSrc: resolved?.element.src ?? null,
isFallbackTrack: resolved?.isFallback ?? false,
decodable: resolved ? fitsBeatAnalysis(resolved.element) : false,
};
}, [elements]);
// Read at load, never a reason to load again: a file length arriving for the same track must keep its edits.
const decodableRef = useRef(decodable);
decodableRef.current = decodable;
// ── Load: decode for strength data, then use the saved beat file if present,
// otherwise seed it from detection. Resets edits + history on track change. ──
// ── Load: the track's beats (loadTrack). Resets edits + history on track change. ──
useEffect(() => {
if (!musicSrc) {
setBeatAnalysis(null);
@@ -150,26 +152,9 @@ export function useMusicBeatAnalysis(): void {
const beatPath = beatFilePathForSrc(musicSrc);
const io = ioRef.current;
// For explicitly tagged/named music tracks: only run expensive audio decode
// + beat analysis when the user has an explicit beats file saved. Without
// one, skip entirely — no surprise green lines on the timeline after
// dragging unrelated assets.
//
// For fallback tracks (audio dropped from Finder with no role/music-id):
// always run analysis so the Beat tool becomes usable immediately.
(async () => {
if (!beatPath || !io) return;
if (!isFallbackTrack) {
const hasSavedBeats = await readHasSavedBeats(io, beatPath);
if (cancelled) return;
if (!hasSavedBeats) {
setBeatAnalysis(null);
return;
}
}
if (cancelled) return;
const result = await loadBeatAnalysis(musicSrc, beatPath, io);
const result = await loadTrack(musicSrc, beatPath, io, isFallbackTrack, decodableRef.current);
if (cancelled) return;
if (!result) {
setBeatAnalysis(null);
@@ -5,7 +5,9 @@ import {
isAudioDomElement,
isAudioTimelineElement,
isMusicSourceElement,
fitsBeatAnalysis,
isMusicTrack,
MAX_BEAT_SOURCE_SECONDS,
resolveBeatSourceTrack,
} from "./timelineInspector";
import type { TimelineElement } from "../player";
@@ -15,12 +17,28 @@ function el(
overrides: Partial<
Pick<
TimelineElement,
"tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
| "tag"
| "src"
| "id"
| "domId"
| "timelineRole"
| "duration"
| "sourceDuration"
| "hasAudio"
| "muted"
>
>,
): Pick<
TimelineElement,
"tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
| "tag"
| "src"
| "id"
| "domId"
| "timelineRole"
| "duration"
| "sourceDuration"
| "hasAudio"
| "muted"
> {
return {
tag: "audio",
@@ -29,6 +47,7 @@ function el(
domId: "el-1",
timelineRole: undefined,
duration: 10,
sourceDuration: 120,
...overrides,
};
}
@@ -127,6 +146,22 @@ describe("resolveBeatSourceTrack", () => {
expect(resolveBeatSourceTrack([sfx, vo])).toBeNull();
});
it("falls back to a song over a longer clip whose file's length is not known yet", () => {
const unread = el({ id: "unread", domId: "unread", duration: 300, sourceDuration: undefined });
const song = el({ id: "song", domId: "song", duration: 180, sourceDuration: 180 });
expect(resolveBeatSourceTrack([unread, song])!.element).toBe(song);
const looped = el({ id: "loop", domId: "loop", duration: 1200, sourceDuration: 180 });
expect(resolveBeatSourceTrack([looped])!.element).toBe(looped);
});
it("never falls back to a clip, or a clip's file, longer than a song", () => {
const talk = el({ id: "talk", domId: "talk", duration: 3720, sourceDuration: 3720 });
const excerpt = el({ id: "cut", domId: "cut", duration: 180, sourceDuration: 3720 });
expect(resolveBeatSourceTrack([talk, excerpt])).toBeNull();
const song = el({ id: "song", domId: "song", duration: 200, sourceDuration: 200 });
expect(resolveBeatSourceTrack([talk, song])!.element).toBe(song);
});
it("returns isFallback=true for an untagged clip with a non-music id", () => {
const clip = el({ id: "my_audio_file", domId: "my_audio_file", timelineRole: undefined });
const result = resolveBeatSourceTrack([clip]);
@@ -184,3 +219,15 @@ describe("beat source from a video with sound", () => {
expect(resolveBeatSourceTrack([short, long, muted])?.element).toBe(long);
});
});
describe("fitsBeatAnalysis", () => {
it("fits a file whose length is known and no longer than a song, however long its clip runs", () => {
expect(fitsBeatAnalysis({ sourceDuration: 180 })).toBe(true);
expect(fitsBeatAnalysis({ sourceDuration: MAX_BEAT_SOURCE_SECONDS })).toBe(true);
});
it("refuses a file past the cap and a file whose length is unknown", () => {
expect(fitsBeatAnalysis({ sourceDuration: 3720 })).toBe(false);
expect(fitsBeatAnalysis({})).toBe(false);
});
});
+21 -2
View File
@@ -80,6 +80,16 @@ export function isMusicTrack(
return MUSIC_ID_RE.test(id);
}
/** Past a song's length a track is speech or a mix, and beat analysis decodes its whole file (62 min: 1.4 GB). */
export const MAX_BEAT_SOURCE_SECONDS = 15 * 60;
/** The decode reads the whole file, so a source fits once its file's length is known and within the cap. */
export function fitsBeatAnalysis({
sourceDuration,
}: Pick<TimelineElement, "sourceDuration">): boolean {
return sourceDuration != null && sourceDuration <= MAX_BEAT_SOURCE_SECONDS;
}
/**
* Resolve the best audio source for beat analysis. An explicitly tagged or
* named music track wins; when none is present (e.g. an audio file dropped
@@ -94,18 +104,27 @@ export function isMusicTrack(
export function resolveBeatSourceTrack(
elements: readonly Pick<
TimelineElement,
"tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
| "tag"
| "src"
| "id"
| "domId"
| "timelineRole"
| "duration"
| "sourceDuration"
| "hasAudio"
| "muted"
>[],
): { element: (typeof elements)[number]; isFallback: boolean } | null {
const explicit = elements.find(isMusicTrack);
if (explicit) return { element: explicit, isFallback: false };
// Fallback: pick the longest audio clip (skipping explicitly non-music roles
// like "sfx" or "voiceover" to avoid triggering beat analysis on those).
// like "sfx" or "voiceover", and anything longer than a song, so none is beat-analysed).
let best: (typeof elements)[number] | null = null;
for (const el of elements) {
if (!isMusicSourceElement(el)) continue;
if (el.timelineRole && el.timelineRole !== "music") continue;
if (!fitsBeatAnalysis(el)) continue;
if (!best || el.duration > best.duration) best = el;
}
return best ? { element: best, isFallback: true } : null;