mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(studio): beat analysis never picks a recording longer than a song (#7944)
* fix(studio): beat analysis never picks a recording longer than a song * fix(studio): a long music track keeps its saved beats; the cap judges the file alone * fix(studio): a music track's beat edits survive its file length arriving
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
// @vitest-environment happy-dom
|
||||
import React, { act } from "react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { TimelineElement } from "../player";
|
||||
import { usePlayerStore } from "../player/store/playerStore";
|
||||
import { FileManagerProvider } from "../contexts/FileManagerContext";
|
||||
import { mountReactHarness } from "./domSelectionTestHarness";
|
||||
|
||||
const analyze = vi.hoisted(() => vi.fn());
|
||||
vi.mock("@hyperframes/core/beats", async (actual) => ({
|
||||
...(await actual<typeof import("@hyperframes/core/beats")>()),
|
||||
analyzeMusicFromUrl: analyze,
|
||||
}));
|
||||
|
||||
const { useMusicBeatAnalysis } = await import("./useMusicBeatAnalysis");
|
||||
const { serializeBeats } = await import("@hyperframes/core/beats");
|
||||
|
||||
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
|
||||
|
||||
const clip = (overrides: Partial<TimelineElement>) =>
|
||||
({
|
||||
id: "talk",
|
||||
domId: "talk",
|
||||
key: "talk",
|
||||
tag: "audio",
|
||||
src: "assets/talk.m4a",
|
||||
start: 0,
|
||||
duration: 180,
|
||||
track: 3,
|
||||
...overrides,
|
||||
}) as TimelineElement;
|
||||
|
||||
let savedBeats = "";
|
||||
const files = {
|
||||
readOptionalProjectFile: async () => savedBeats,
|
||||
writeProjectFile: async () => {},
|
||||
} as unknown as React.ComponentProps<typeof FileManagerProvider>["value"];
|
||||
|
||||
function Probe() {
|
||||
useMusicBeatAnalysis();
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Studio's timeline opened on `elements`: which files beat analysis decoded, and the beat times it shows. The hook
|
||||
* caches an analysis per file, so each case names its own. */
|
||||
async function opened(elements: TimelineElement[]) {
|
||||
usePlayerStore.setState({ elements, beatAnalysis: null });
|
||||
const root = mountReactHarness(
|
||||
<FileManagerProvider value={files}>
|
||||
<Probe />
|
||||
</FileManagerProvider>,
|
||||
);
|
||||
await act(async () => {
|
||||
for (let i = 0; i < 5; i += 1) await Promise.resolve();
|
||||
});
|
||||
const beats = usePlayerStore.getState().beatAnalysis?.beatTimes ?? null;
|
||||
act(() => root.unmount());
|
||||
return { decoded: analyze.mock.calls.map(([src]) => src), beats };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
savedBeats = "";
|
||||
analyze.mockReset();
|
||||
analyze.mockResolvedValue({ beatTimes: [1, 2], beatStrengths: [1, 1], bpm: 120 });
|
||||
});
|
||||
afterEach(() =>
|
||||
usePlayerStore.setState({ elements: [], beatEdits: null, beatUndo: [], beatRedo: [] }),
|
||||
);
|
||||
|
||||
// fails without the fix: a 62-minute recording with no role was picked as the music and decoded whole (1.4 GB on
|
||||
// a Mac, which ran a person's page out of memory while their talk was still being transcribed).
|
||||
describe("beat analysis decodes only a file no longer than a song", () => {
|
||||
it("leaves a 62-minute unlabelled recording alone", async () => {
|
||||
const talk = clip({ src: "assets/talk.m4a", duration: 3720, sourceDuration: 3720 });
|
||||
expect((await opened([talk])).decoded).toEqual([]);
|
||||
});
|
||||
|
||||
it("leaves a 3-minute clip of a 62-minute file alone, since the decode reads the whole file", async () => {
|
||||
const cut = clip({ src: "assets/cut.m4a", duration: 180, sourceDuration: 3720 });
|
||||
expect((await opened([cut])).decoded).toEqual([]);
|
||||
});
|
||||
|
||||
it("leaves a recording alone while its file's length is unknown", async () => {
|
||||
expect((await opened([clip({ src: "assets/unread.m4a", duration: 180 })])).decoded).toEqual([]);
|
||||
});
|
||||
|
||||
it("shows a 62-minute music track's saved beats without decoding it", async () => {
|
||||
savedBeats = serializeBeats([1, 2], [1, 1], "assets/mix.m4a");
|
||||
const mix = clip({
|
||||
src: "assets/mix.m4a",
|
||||
timelineRole: "music",
|
||||
duration: 3720,
|
||||
sourceDuration: 3720,
|
||||
});
|
||||
expect(await opened([mix])).toEqual({ decoded: [], beats: [1, 2] });
|
||||
});
|
||||
|
||||
it("still finds the beats of a 3-minute song", async () => {
|
||||
const song = clip({ src: "assets/song.mp3", sourceDuration: 180 });
|
||||
expect(await opened([song])).toEqual({ decoded: ["assets/song.mp3"], beats: [1, 2] });
|
||||
});
|
||||
});
|
||||
|
||||
// fails without the fix: the length arriving after the saved beats showed loaded the track again, which dropped the
|
||||
// person's beat edit and its undo.
|
||||
it("keeps a music track's beat edits and undo when its file's length arrives", async () => {
|
||||
savedBeats = serializeBeats([1, 2], [1, 1], "assets/edited.m4a");
|
||||
const music = { src: "assets/edited.m4a", timelineRole: "music" as const, duration: 180 };
|
||||
const settle = () =>
|
||||
act(async () => {
|
||||
for (let i = 0; i < 5; i += 1) await Promise.resolve();
|
||||
});
|
||||
usePlayerStore.setState({ elements: [clip(music)], beatAnalysis: null, beatEdits: null });
|
||||
const root = mountReactHarness(
|
||||
<FileManagerProvider value={files}>
|
||||
<Probe />
|
||||
</FileManagerProvider>,
|
||||
);
|
||||
await settle();
|
||||
const edit = { src: "assets/edited.m4a", added: [{ time: 1.5, strength: 1 }], removed: [] };
|
||||
act(() => usePlayerStore.getState().commitBeatEdits(edit, "add beat"));
|
||||
act(() => usePlayerStore.setState({ elements: [clip({ ...music, sourceDuration: 180 })] }));
|
||||
await settle();
|
||||
const { beatEdits, beatUndo, beatAnalysis } = usePlayerStore.getState();
|
||||
act(() => root.unmount());
|
||||
expect({ beatEdits, undo: beatUndo.length, beats: beatAnalysis?.beatTimes }).toEqual({
|
||||
beatEdits: edit,
|
||||
undo: 1,
|
||||
beats: [1, 2],
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useMemo, useRef } from "react";
|
||||
import { usePlayerStore } from "../player/store/playerStore";
|
||||
import { resolveBeatSourceTrack } from "../utils/timelineInspector";
|
||||
import { fitsBeatAnalysis, resolveBeatSourceTrack } from "../utils/timelineInspector";
|
||||
import { analyzeMusicFromUrl } from "@hyperframes/core/beats";
|
||||
import { whenTimelineIdle } from "../player/components/timelineMotion";
|
||||
import { useFileManagerContextOptional } from "../contexts/FileManagerContext";
|
||||
@@ -32,67 +32,66 @@ function cacheAnalysis(url: string, promise: ReturnType<typeof analyzeMusicFromU
|
||||
|
||||
type ProjectIo = { readOptionalProjectFile: (p: string) => Promise<string> };
|
||||
|
||||
/**
|
||||
* Resolve the effective beat list for a track: a saved file with real beats
|
||||
* wins; otherwise the detected beats are used (and `hasFile` is false so the
|
||||
* caller seeds a new file). An empty saved file is ignored so detection retries.
|
||||
*/
|
||||
async function resolveBeats(
|
||||
beatPath: string | null,
|
||||
detected: { times: number[]; strengths: number[] },
|
||||
io: ProjectIo,
|
||||
): Promise<{ times: number[]; strengths: number[]; hasFile: boolean }> {
|
||||
if (!beatPath) return { ...detected, hasFile: false };
|
||||
try {
|
||||
const content = await io.readOptionalProjectFile(beatPath);
|
||||
const parsed = content ? parseBeats(content) : null;
|
||||
if (parsed && parsed.times.length > 0) {
|
||||
return { times: parsed.times, strengths: parsed.strengths, hasFile: true };
|
||||
}
|
||||
} catch {
|
||||
/* fall back to detected beats */
|
||||
}
|
||||
return { ...detected, hasFile: false };
|
||||
}
|
||||
type Beats = { times: number[]; strengths: number[] };
|
||||
|
||||
/** True when the beats file for a track exists and holds at least one beat. */
|
||||
async function readHasSavedBeats(io: ProjectIo, beatPath: string): Promise<boolean> {
|
||||
/** The track's saved beats file, or null when it is missing, unreadable or holds no beat (detection then retries). */
|
||||
async function readSavedBeats(io: ProjectIo, beatPath: string): Promise<Beats | null> {
|
||||
try {
|
||||
const content = await io.readOptionalProjectFile(beatPath);
|
||||
const parsed = content ? parseBeats(content) : null;
|
||||
return !!(parsed && parsed.times.length > 0);
|
||||
return parsed && parsed.times.length > 0
|
||||
? { times: parsed.times, strengths: parsed.strengths }
|
||||
: null;
|
||||
} catch {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
type MusicAnalysis = Awaited<ReturnType<typeof analyzeMusicFromUrl>>;
|
||||
|
||||
/**
|
||||
* Analyze a track (memoized per URL) and fold in any saved beat edits. Null on
|
||||
* decode/analysis failure — the caller then clears state and drops the cache
|
||||
* entry (only when the effect is still live).
|
||||
*/
|
||||
async function loadBeatAnalysis(
|
||||
musicSrc: string,
|
||||
beatPath: string,
|
||||
io: ProjectIo,
|
||||
): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> {
|
||||
type Loaded = { analysis: MusicAnalysis } & Beats;
|
||||
|
||||
/** A track's analysis (memoized per URL), or null when decode or analysis fails. */
|
||||
async function decodeTrack(musicSrc: string): Promise<MusicAnalysis | null> {
|
||||
let promise = analysisCache.get(musicSrc);
|
||||
if (!promise) {
|
||||
promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle });
|
||||
cacheAnalysis(musicSrc, promise);
|
||||
}
|
||||
try {
|
||||
const analysis = await promise;
|
||||
const detected = { times: analysis.beatTimes, strengths: analysis.beatStrengths };
|
||||
const { times, strengths } = await resolveBeats(beatPath, detected, io);
|
||||
return { analysis, times, strengths };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return promise.catch(() => null);
|
||||
}
|
||||
|
||||
/** What a track shows. A fallback track (audio dropped in with no role) is analysed so the Beat tool works at once; a
|
||||
* marked music track only once the person saved beats for it, so unrelated assets add no beat lines. A file that does
|
||||
* not fit the decode keeps its saved beats, with no strengths measured from the audio. */
|
||||
async function loadTrack(
|
||||
musicSrc: string,
|
||||
beatPath: string,
|
||||
io: ProjectIo,
|
||||
isFallbackTrack: boolean,
|
||||
decodable: boolean,
|
||||
): Promise<Loaded | null> {
|
||||
const saved = await readSavedBeats(io, beatPath);
|
||||
if (!isFallbackTrack && !saved) return null;
|
||||
if (!decodable) return saved && { analysis: savedOnly(saved), ...saved };
|
||||
const analysis = await decodeTrack(musicSrc);
|
||||
if (!analysis) return null;
|
||||
return {
|
||||
analysis,
|
||||
...(saved ?? { times: analysis.beatTimes, strengths: analysis.beatStrengths }),
|
||||
};
|
||||
}
|
||||
|
||||
const savedOnly = ({ times, strengths }: Beats): MusicAnalysis => ({
|
||||
beatTimes: times,
|
||||
beatStrengths: strengths,
|
||||
bpm: null,
|
||||
bpmConfidence: "uncertain",
|
||||
channelData: null,
|
||||
sampleRate: 0,
|
||||
peak: 0,
|
||||
});
|
||||
|
||||
/** The current beat analysis, remapped onto the composition's edited beat grid. */
|
||||
export function useAdjustedBeatAnalysis() {
|
||||
const beatAnalysis = usePlayerStore((s) => s.beatAnalysis);
|
||||
@@ -123,16 +122,19 @@ export function useMusicBeatAnalysis(): void {
|
||||
? { readOptionalProjectFile, writeProjectFile }
|
||||
: null;
|
||||
|
||||
const { musicSrc, isFallbackTrack } = useMemo(() => {
|
||||
const { musicSrc, isFallbackTrack, decodable } = useMemo(() => {
|
||||
const resolved = resolveBeatSourceTrack(elements);
|
||||
return {
|
||||
musicSrc: resolved?.element.src ?? null,
|
||||
isFallbackTrack: resolved?.isFallback ?? false,
|
||||
decodable: resolved ? fitsBeatAnalysis(resolved.element) : false,
|
||||
};
|
||||
}, [elements]);
|
||||
// Read at load, never a reason to load again: a file length arriving for the same track must keep its edits.
|
||||
const decodableRef = useRef(decodable);
|
||||
decodableRef.current = decodable;
|
||||
|
||||
// ── Load: decode for strength data, then use the saved beat file if present,
|
||||
// otherwise seed it from detection. Resets edits + history on track change. ──
|
||||
// ── Load: the track's beats (loadTrack). Resets edits + history on track change. ──
|
||||
useEffect(() => {
|
||||
if (!musicSrc) {
|
||||
setBeatAnalysis(null);
|
||||
@@ -150,26 +152,9 @@ export function useMusicBeatAnalysis(): void {
|
||||
const beatPath = beatFilePathForSrc(musicSrc);
|
||||
const io = ioRef.current;
|
||||
|
||||
// For explicitly tagged/named music tracks: only run expensive audio decode
|
||||
// + beat analysis when the user has an explicit beats file saved. Without
|
||||
// one, skip entirely — no surprise green lines on the timeline after
|
||||
// dragging unrelated assets.
|
||||
//
|
||||
// For fallback tracks (audio dropped from Finder with no role/music-id):
|
||||
// always run analysis so the Beat tool becomes usable immediately.
|
||||
(async () => {
|
||||
if (!beatPath || !io) return;
|
||||
if (!isFallbackTrack) {
|
||||
const hasSavedBeats = await readHasSavedBeats(io, beatPath);
|
||||
if (cancelled) return;
|
||||
if (!hasSavedBeats) {
|
||||
setBeatAnalysis(null);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (cancelled) return;
|
||||
|
||||
const result = await loadBeatAnalysis(musicSrc, beatPath, io);
|
||||
const result = await loadTrack(musicSrc, beatPath, io, isFallbackTrack, decodableRef.current);
|
||||
if (cancelled) return;
|
||||
if (!result) {
|
||||
setBeatAnalysis(null);
|
||||
|
||||
@@ -5,7 +5,9 @@ import {
|
||||
isAudioDomElement,
|
||||
isAudioTimelineElement,
|
||||
isMusicSourceElement,
|
||||
fitsBeatAnalysis,
|
||||
isMusicTrack,
|
||||
MAX_BEAT_SOURCE_SECONDS,
|
||||
resolveBeatSourceTrack,
|
||||
} from "./timelineInspector";
|
||||
import type { TimelineElement } from "../player";
|
||||
@@ -15,12 +17,28 @@ function el(
|
||||
overrides: Partial<
|
||||
Pick<
|
||||
TimelineElement,
|
||||
"tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
|
||||
| "tag"
|
||||
| "src"
|
||||
| "id"
|
||||
| "domId"
|
||||
| "timelineRole"
|
||||
| "duration"
|
||||
| "sourceDuration"
|
||||
| "hasAudio"
|
||||
| "muted"
|
||||
>
|
||||
>,
|
||||
): Pick<
|
||||
TimelineElement,
|
||||
"tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
|
||||
| "tag"
|
||||
| "src"
|
||||
| "id"
|
||||
| "domId"
|
||||
| "timelineRole"
|
||||
| "duration"
|
||||
| "sourceDuration"
|
||||
| "hasAudio"
|
||||
| "muted"
|
||||
> {
|
||||
return {
|
||||
tag: "audio",
|
||||
@@ -29,6 +47,7 @@ function el(
|
||||
domId: "el-1",
|
||||
timelineRole: undefined,
|
||||
duration: 10,
|
||||
sourceDuration: 120,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -127,6 +146,22 @@ describe("resolveBeatSourceTrack", () => {
|
||||
expect(resolveBeatSourceTrack([sfx, vo])).toBeNull();
|
||||
});
|
||||
|
||||
it("falls back to a song over a longer clip whose file's length is not known yet", () => {
|
||||
const unread = el({ id: "unread", domId: "unread", duration: 300, sourceDuration: undefined });
|
||||
const song = el({ id: "song", domId: "song", duration: 180, sourceDuration: 180 });
|
||||
expect(resolveBeatSourceTrack([unread, song])!.element).toBe(song);
|
||||
const looped = el({ id: "loop", domId: "loop", duration: 1200, sourceDuration: 180 });
|
||||
expect(resolveBeatSourceTrack([looped])!.element).toBe(looped);
|
||||
});
|
||||
|
||||
it("never falls back to a clip, or a clip's file, longer than a song", () => {
|
||||
const talk = el({ id: "talk", domId: "talk", duration: 3720, sourceDuration: 3720 });
|
||||
const excerpt = el({ id: "cut", domId: "cut", duration: 180, sourceDuration: 3720 });
|
||||
expect(resolveBeatSourceTrack([talk, excerpt])).toBeNull();
|
||||
const song = el({ id: "song", domId: "song", duration: 200, sourceDuration: 200 });
|
||||
expect(resolveBeatSourceTrack([talk, song])!.element).toBe(song);
|
||||
});
|
||||
|
||||
it("returns isFallback=true for an untagged clip with a non-music id", () => {
|
||||
const clip = el({ id: "my_audio_file", domId: "my_audio_file", timelineRole: undefined });
|
||||
const result = resolveBeatSourceTrack([clip]);
|
||||
@@ -184,3 +219,15 @@ describe("beat source from a video with sound", () => {
|
||||
expect(resolveBeatSourceTrack([short, long, muted])?.element).toBe(long);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fitsBeatAnalysis", () => {
|
||||
it("fits a file whose length is known and no longer than a song, however long its clip runs", () => {
|
||||
expect(fitsBeatAnalysis({ sourceDuration: 180 })).toBe(true);
|
||||
expect(fitsBeatAnalysis({ sourceDuration: MAX_BEAT_SOURCE_SECONDS })).toBe(true);
|
||||
});
|
||||
|
||||
it("refuses a file past the cap and a file whose length is unknown", () => {
|
||||
expect(fitsBeatAnalysis({ sourceDuration: 3720 })).toBe(false);
|
||||
expect(fitsBeatAnalysis({})).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -80,6 +80,16 @@ export function isMusicTrack(
|
||||
return MUSIC_ID_RE.test(id);
|
||||
}
|
||||
|
||||
/** Past a song's length a track is speech or a mix, and beat analysis decodes its whole file (62 min: 1.4 GB). */
|
||||
export const MAX_BEAT_SOURCE_SECONDS = 15 * 60;
|
||||
|
||||
/** The decode reads the whole file, so a source fits once its file's length is known and within the cap. */
|
||||
export function fitsBeatAnalysis({
|
||||
sourceDuration,
|
||||
}: Pick<TimelineElement, "sourceDuration">): boolean {
|
||||
return sourceDuration != null && sourceDuration <= MAX_BEAT_SOURCE_SECONDS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the best audio source for beat analysis. An explicitly tagged or
|
||||
* named music track wins; when none is present (e.g. an audio file dropped
|
||||
@@ -94,18 +104,27 @@ export function isMusicTrack(
|
||||
export function resolveBeatSourceTrack(
|
||||
elements: readonly Pick<
|
||||
TimelineElement,
|
||||
"tag" | "src" | "id" | "domId" | "timelineRole" | "duration" | "hasAudio" | "muted"
|
||||
| "tag"
|
||||
| "src"
|
||||
| "id"
|
||||
| "domId"
|
||||
| "timelineRole"
|
||||
| "duration"
|
||||
| "sourceDuration"
|
||||
| "hasAudio"
|
||||
| "muted"
|
||||
>[],
|
||||
): { element: (typeof elements)[number]; isFallback: boolean } | null {
|
||||
const explicit = elements.find(isMusicTrack);
|
||||
if (explicit) return { element: explicit, isFallback: false };
|
||||
|
||||
// Fallback: pick the longest audio clip (skipping explicitly non-music roles
|
||||
// like "sfx" or "voiceover" to avoid triggering beat analysis on those).
|
||||
// like "sfx" or "voiceover", and anything longer than a song, so none is beat-analysed).
|
||||
let best: (typeof elements)[number] | null = null;
|
||||
for (const el of elements) {
|
||||
if (!isMusicSourceElement(el)) continue;
|
||||
if (el.timelineRole && el.timelineRole !== "music") continue;
|
||||
if (!fitsBeatAnalysis(el)) continue;
|
||||
if (!best || el.duration > best.duration) best = el;
|
||||
}
|
||||
return best ? { element: best, isFallback: true } : null;
|
||||
|
||||
Reference in New Issue
Block a user