diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 3f72ed9947..18af9f3ea6 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2088,6 +2088,9 @@ files, the browser download and models are never touched. duration, element counts by type, track count, and total size. `npx hyperframes compositions` lists every composition in the project with its ID, duration, resolution, and element count. Both take `--json`. +`npx hyperframes info --media --json` adds `heard`: every clip the render mixes, scenes included, on the film's clock, +each with `src`, `start`, `end` (where it stops, or where its media runs out if sooner; `0` runs to the media's end), +`mediaStart`, `playbackRate`, `volume` and `type`. ### `update` diff --git a/packages/cli/src/commands/info.test.ts b/packages/cli/src/commands/info.test.ts index dca6a2c1cf..8c34eb593c 100644 --- a/packages/cli/src/commands/info.test.ts +++ b/packages/cli/src/commands/info.test.ts @@ -69,7 +69,7 @@ describe("info", () => { const log = vi.spyOn(console, "log").mockImplementation(() => {}); try { writeFileSync(join(dir, "index.html"), html); - await info.run?.({ args: { _: [], dir, json: true }, rawArgs: [], cmd: info }); + await info.run?.({ args: { _: [], dir, json: true, media: false }, rawArgs: [], cmd: info }); expect(JSON.parse(String(log.mock.calls[0]?.[0]))).toMatchObject({ resolution: "portrait", @@ -93,7 +93,7 @@ describe("info", () => {
`, ); - await info.run?.({ args: { _: [], dir, json: true }, rawArgs: [], cmd: info }); + await info.run?.({ args: { _: [], dir, json: true, media: false }, rawArgs: [], cmd: info }); expect(JSON.parse(String(log.mock.calls[0]?.[0]))).toMatchObject({ resolution: "landscape", diff --git a/packages/cli/src/commands/info.ts b/packages/cli/src/commands/info.ts index 6e86049240..7e4f0a9267 100644 --- a/packages/cli/src/commands/info.ts +++ b/packages/cli/src/commands/info.ts @@ -5,6 +5,7 @@ import { readFileSync, readdirSync, statSync } from "node:fs"; export const examples: Example[] = [ ["Show project metadata", "hyperframes info"], ["Output as JSON", "hyperframes info --json"], + ["List the clips the render hears, on the film's clock", "hyperframes info --media --json"], ]; import { join } from "node:path"; import { parseHtml, CANVAS_DIMENSIONS } from "@hyperframes/core"; @@ -13,6 +14,7 @@ import { readCompositionSize, } from "@hyperframes/core/runtime/composition-length"; import { readDataDurationSeconds } from "@hyperframes/parsers/media-duration"; +import { heardClips } from "../utils/projectMedia.js"; import { c } from "../ui/colors.js"; import { formatBytes, label } from "../ui/format.js"; import { ensureDOMParser } from "../utils/dom.js"; @@ -61,6 +63,11 @@ export default defineCommand({ args: { dir: { type: "positional", description: "Project directory", required: false }, json: { type: "boolean", description: "Output as JSON", default: false }, + media: { + type: "boolean", + description: "Also list every clip the render hears, scenes included, on the film's clock", + default: false, + }, }, async run({ args }) { const project = resolveProject(args.dir); @@ -90,6 +97,8 @@ export default defineCommand({ .map(([t, count]) => `${count} ${t}`) .join(", "); + const heard = args.media ? await heardClips(project.dir) : undefined; + if (args.json) { console.log( JSON.stringify( @@ -103,6 +112,7 @@ export default defineCommand({ tracks: tracks.size, types: typeCounts, size, + ...(heard ? { heard } : {}), }), null, 2, @@ -117,5 +127,9 @@ export default defineCommand({ console.log(label("Elements", `${parsed.elements.length}${typeStr ? ` (${typeStr})` : ""}`)); console.log(label("Tracks", `${tracks.size}`)); console.log(label("Size", formatBytes(size))); + for (const clip of heard ?? []) { + const until = clip.end > 0 ? `${clip.end.toFixed(2)}s` : "its end"; + console.log(label("Heard", `${clip.src} ${clip.start.toFixed(2)}s to ${until}`)); + } }, }); diff --git a/packages/cli/src/utils/projectMedia.test.ts b/packages/cli/src/utils/projectMedia.test.ts new file mode 100644 index 0000000000..54c10ebc8c --- /dev/null +++ b/packages/cli/src/utils/projectMedia.test.ts @@ -0,0 +1,82 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { heardClips } from "./projectMedia.js"; + +let dir = ""; +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +function project(scene: string): string { + dir = mkdtempSync(join(tmpdir(), "hf-heard-")); + mkdirSync(join(dir, "compositions")); + writeFileSync( + join(dir, "index.html"), + ` +
+
+
`, + ); + writeFileSync( + join(dir, "compositions", "talk.html"), + ``, + ); + return dir; +} + +const probe = { seconds: async () => 6 }; + +describe("heardClips", () => { + it("hears a speaking video inside a timed wrapper in a scene, where the render plays it", async () => { + const clips = await heardClips( + project(`
+ +
`), + probe, + ); + expect(clips.map(({ src, start, end }) => ({ src, start, end }))).toEqual([ + { src: "assets/talk.mp4", start: 10, end: 15 }, + ]); + }); + + it("ends a speaking clip with no set length where its media runs out, not where its scene ends", async () => { + const clips = await heardClips( + project( + ``, + ), + probe, + ); + expect(clips.map(({ src, start, end }) => ({ src, start, end }))).toEqual([ + { src: "assets/talk.mp4", start: 12, end: 17 }, + ]); + }); + + it("leaves out a clip that starts past its media's end, and reads each file's length once", async () => { + const probed: string[] = []; + const counting = { seconds: async (path: string) => (probed.push(path), 6) }; + const clips = await heardClips( + project(` +`), + counting, + ); + expect(clips.map(({ id, start, end }) => ({ id, start, end }))).toEqual([ + { id: "talk-audio", start: 18, end: 21 }, + ]); + expect(probed).toHaveLength(1); + }); + + it("hears a video only when it says data-has-audio and is not muted, as the render does", async () => { + const clips = await heardClips( + project(` + +`), + probe, + ); + expect(clips.map(({ src, start, end }) => ({ src, start, end }))).toEqual([ + { src: "assets/talk.mp4", start: 18, end: 21 }, + ]); + }); +}); diff --git a/packages/cli/src/utils/projectMedia.ts b/packages/cli/src/utils/projectMedia.ts index 20990d76cd..73bdf56689 100644 --- a/packages/cli/src/utils/projectMedia.ts +++ b/packages/cli/src/utils/projectMedia.ts @@ -1,4 +1,5 @@ import { + extractAudioMetadata, resolveProjectRelativeSrc, type AudioElement, type VideoElement, @@ -29,3 +30,50 @@ export async function collectProjectMedia(projectDir: string): Promise; + +export interface HeardProbe { + seconds(path: string): Promise; +} + +const PROBE: HeardProbe = { + seconds: async (path) => (await extractAudioMetadata(path)).durationSeconds, +}; + +/** Every clip the render mixes, read as the render reads them (scenes inlined, on the film's clock), each ended where + * its media runs out if sooner, and left out if it starts past that; a ramped rate or a loop keeps its end. */ +export async function heardClips( + projectDir: string, + probe: HeardProbe = PROBE, +): Promise { + const { audios } = await collectProjectMedia(projectDir); + const lengths = new Map>(); + const length = (src: string): Promise => { + const path = mediaPath(src, projectDir); + const known = lengths.get(path) ?? probe.seconds(path).catch(() => null); + lengths.set(path, known); + return known; + }; + const clips = await Promise.all(audios.map((clip) => endedClip(clip, length))); + return clips.filter((clip) => clip !== null); +} + +async function endedClip( + clip: AudioElement, + length: (src: string) => Promise, +): Promise { + const { id, src, start, mediaStart, playbackRate, volume, type } = clip; + const heard = { id, src, start, end: clip.end, mediaStart, playbackRate, volume, type }; + const rate = playbackRate === undefined ? 1 : playbackRate; + if (clip.loop || typeof rate !== "number" || !(rate > 0)) return heard; + const seconds = await length(src); + if (seconds === null) return heard; + if (seconds <= mediaStart) return null; + const runsOut = start + (seconds - mediaStart) / rate; + return { ...heard, end: clip.end > 0 ? Math.min(clip.end, runsOut) : runsOut }; +}