diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 110457f1c..e79a49359 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -262,11 +262,19 @@ jobs: test: name: Test - needs: changes - if: needs.changes.outputs.code == 'true' + needs: [changes, producer-source-tests] + # Keep the existing required `Test` context authoritative for producer + # failures too. The dedicated producer matrix remains parallel and legible, + # while this job fails closed if either lane fails or is cancelled. + if: always() && needs.changes.outputs.code == 'true' runs-on: ubuntu-latest timeout-minutes: 10 steps: + - name: Require producer source tests + if: needs.producer-source-tests.result != 'success' + run: | + echo "::error::Producer unit/integration tests did not succeed." + exit 1 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: lfs: true diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index e9c348143..d8e554062 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -264,9 +264,11 @@ export { extractVideoMetadata, extractFinalVideoFrameTimestamp, extractAudioMetadata, + probeMediaProfile, analyzeKeyframeIntervals, type VideoMetadata, type AudioMetadata, + type MediaProbeProfile, type KeyframeAnalysis, } from "./utils/ffprobe.js"; diff --git a/packages/engine/src/utils/ffprobe.test.ts b/packages/engine/src/utils/ffprobe.test.ts index 256cfa1a9..2030dc105 100644 --- a/packages/engine/src/utils/ffprobe.test.ts +++ b/packages/engine/src/utils/ffprobe.test.ts @@ -1,6 +1,8 @@ // fallow-ignore-file code-duplication import { EventEmitter } from "events"; -import { readFileSync } from "fs"; +import { spawnSync } from "child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; import { basename, resolve } from "path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -113,6 +115,272 @@ describe("extractPngMetadataFromBuffer", () => { ); expect(extractPngMetadataFromBuffer(fixture)?.colorSpace?.colorTransfer).toBe("smpte2084"); }); + + it("keeps metadata fallback independent from full PNG integrity validation", () => { + const ihdr = pngChunk("IHDR", [0, 0, 0, 1, 0, 0, 0, 1, 16, 2, 0, 0, 0]); + expect(extractPngMetadataFromBuffer(buildPngWithChunks([ihdr]))).toMatchObject({ + width: 1, + height: 1, + }); + }); +}); + +describe("probeMediaProfile", () => { + afterEach(() => { + vi.resetModules(); + vi.doUnmock("child_process"); + }); + + it("classifies still, moving, audio-only, and mixed streams from probe data", async () => { + const { spawn } = createSpawnSpy([ + { + kind: "exit", + code: 0, + stdout: JSON.stringify({ + streams: [{ codec_type: "video", codec_name: "png" }], + format: { format_name: "png_pipe" }, + }), + }, + { + kind: "exit", + code: 0, + stdout: JSON.stringify({ + streams: [{ codec_type: "video", codec_name: "h264" }], + format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" }, + }), + }, + { + kind: "exit", + code: 0, + stdout: JSON.stringify({ + streams: [{ codec_type: "audio", codec_name: "mp3" }], + format: { format_name: "mp3" }, + }), + }, + { + kind: "exit", + code: 0, + stdout: JSON.stringify({ + streams: [{ codec_type: "video" }, { codec_type: "audio" }], + format: { format_name: "matroska,webm" }, + }), + }, + ]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { probeMediaProfile } = await import("./ffprobe.js"); + + const validPngPath = resolve( + __dirname, + "../../../producer/tests/hdr-regression/src/hdr-photo-pq.png", + ); + await expect(probeMediaProfile(validPngPath)).resolves.toEqual({ + hasVideoStream: true, + hasAudioStream: false, + visualKind: "still", + }); + await expect(probeMediaProfile("/tmp/extensionless-video")).resolves.toEqual({ + hasVideoStream: true, + hasAudioStream: false, + visualKind: "moving", + }); + await expect(probeMediaProfile("/tmp/extensionless-audio")).resolves.toEqual({ + hasVideoStream: false, + hasAudioStream: true, + visualKind: "none", + }); + await expect(probeMediaProfile("/tmp/mixed-av")).resolves.toEqual({ + hasVideoStream: true, + hasAudioStream: true, + visualKind: "moving", + }); + }); + + it("does not treat attached cover art in an audio container as an image asset", async () => { + const { spawn } = createSpawnSpy([ + { + kind: "exit", + code: 0, + stdout: JSON.stringify({ + streams: [ + { codec_type: "video", disposition: { attached_pic: 1 } }, + { codec_type: "audio" }, + ], + format: { format_name: "mp3" }, + }), + }, + ]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { probeMediaProfile } = await import("./ffprobe.js"); + await expect(probeMediaProfile("/tmp/audio-with-cover")).resolves.toMatchObject({ + hasAudioStream: true, + visualKind: "none", + }); + }); + + it("deduplicates probes within one cancellation scope without sharing across scopes", async () => { + const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-media-probe-cache-")); + const fixturePath = resolve(fixtureDir, "asset"); + writeFileSync(fixturePath, "cache identity only"); + const outcome = { + kind: "exit" as const, + code: 0, + stdout: JSON.stringify({ + streams: [{ codec_type: "video", codec_name: "h264" }], + format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" }, + }), + }; + const { spawn, calls } = createSpawnSpy([outcome, outcome]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { probeMediaProfile } = await import("./ffprobe.js"); + const firstSignal = new AbortController().signal; + const secondSignal = new AbortController().signal; + try { + await Promise.all([ + probeMediaProfile(fixturePath, { signal: firstSignal }), + probeMediaProfile(fixturePath, { signal: firstSignal }), + ]); + expect(calls).toHaveLength(1); + await probeMediaProfile(fixturePath, { signal: secondSignal }); + expect(calls).toHaveLength(2); + } finally { + rmSync(fixtureDir, { recursive: true, force: true }); + } + }); + + it("bounds the process-scoped probe cache", async () => { + const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-media-probe-lru-")); + const fixturePaths = Array.from({ length: 129 }, (_, index) => + resolve(fixtureDir, `asset-${index}`), + ); + for (const fixturePath of fixturePaths) writeFileSync(fixturePath, "probe identity"); + const outcome = { + kind: "exit" as const, + code: 0, + stdout: JSON.stringify({ + streams: [{ codec_type: "audio", codec_name: "aac" }], + format: { format_name: "aac" }, + }), + }; + const { spawn, calls } = createSpawnSpy([outcome]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { probeMediaProfile } = await import("./ffprobe.js"); + try { + for (const fixturePath of fixturePaths) await probeMediaProfile(fixturePath); + await probeMediaProfile(fixturePaths[0]!); + expect(calls).toHaveLength(130); + } finally { + rmSync(fixtureDir, { recursive: true, force: true }); + } + }); + + it("classifies extensionless AVIF from its ISO-BMFF brand instead of the generic mov demuxer", async () => { + const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-avif-profile-")); + const fixturePath = resolve(fixtureDir, "asset"); + const ftyp = Buffer.alloc(24); + ftyp.writeUInt32BE(24, 0); + ftyp.write("ftyp", 4, 4, "ascii"); + ftyp.write("avif", 8, 4, "ascii"); + ftyp.writeUInt32BE(0, 12); + ftyp.write("mif1", 16, 4, "ascii"); + ftyp.write("avif", 20, 4, "ascii"); + writeFileSync(fixturePath, ftyp); + const { spawn } = createSpawnSpy([ + { + kind: "exit", + code: 0, + stdout: JSON.stringify({ + streams: [{ codec_type: "video", codec_name: "av1" }], + format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" }, + }), + }, + ]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { probeMediaProfile } = await import("./ffprobe.js"); + try { + await expect(probeMediaProfile(fixturePath)).resolves.toMatchObject({ + hasVideoStream: true, + hasAudioStream: false, + visualKind: "still", + }); + } finally { + rmSync(fixtureDir, { recursive: true, force: true }); + } + }); + + it("uses any non-attached video stream when cover art precedes moving video", async () => { + const { spawn } = createSpawnSpy([ + { + kind: "exit", + code: 0, + stdout: JSON.stringify({ + streams: [ + { codec_type: "video", disposition: { attached_pic: 1 } }, + { codec_type: "video", codec_name: "h264" }, + { codec_type: "audio" }, + ], + format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" }, + }), + }, + ]); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { probeMediaProfile } = await import("./ffprobe.js"); + await expect(probeMediaProfile("/tmp/video-with-cover")).resolves.toMatchObject({ + visualKind: "moving", + }); + }); + + it.skipIf(spawnSync("ffprobe", ["-version"]).status !== 0)( + "rejects an IHDR-only truncated PNG even when ffprobe accepts png_pipe", + async () => { + vi.resetModules(); + vi.doUnmock("child_process"); + const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-truncated-png-profile-")); + const fixturePath = resolve(fixtureDir, "asset"); + const ihdr = pngChunk("IHDR", [0, 0, 0, 1, 0, 0, 0, 1, 16, 2, 0, 0, 0]); + writeFileSync(fixturePath, buildPngWithChunks([ihdr])); + const { probeMediaProfile } = await import("./ffprobe.js"); + try { + await expect(probeMediaProfile(fixturePath)).rejects.toThrow(); + } finally { + rmSync(fixtureDir, { recursive: true, force: true }); + } + }, + ); + + it("does not turn an aborted PNG probe into a successful metadata fallback", async () => { + type KillableFakeProc = FakeProc & { kill: (signal?: NodeJS.Signals) => boolean }; + const spawn = () => { + const proc = new EventEmitter() as KillableFakeProc; + proc.stdout = new EventEmitter(); + proc.stderr = new EventEmitter(); + proc.kill = vi.fn(() => { + process.nextTick(() => proc.emit("close", null, "SIGTERM")); + return true; + }); + process.nextTick(() => proc.emit("spawn")); + return proc; + }; + const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-aborted-png-profile-")); + const fixturePath = resolve(fixtureDir, "asset"); + writeFileSync(fixturePath, buildMinimalPng()); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { probeMediaProfile } = await import("./ffprobe.js"); + const controller = new AbortController(); + try { + const pending = probeMediaProfile(fixturePath, { signal: controller.signal }); + controller.abort(new Error("render cancelled")); + await expect(pending).rejects.toThrow("render cancelled"); + } finally { + rmSync(fixtureDir, { recursive: true, force: true }); + } + }); }); interface SpawnCall { @@ -222,6 +490,23 @@ describe("ffprobe missing-binary fallback", () => { expect(calls[0]?.args.slice(0, 2)).toEqual(["-v", "error"]); }); + it("does not accept an incomplete PNG through the missing-binary fallback", async () => { + const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-truncated-png-fallback-")); + const fixturePath = resolve(fixtureDir, "asset"); + const ihdr = pngChunk("IHDR", [0, 0, 0, 1, 0, 0, 0, 1, 16, 2, 0, 0, 0]); + writeFileSync(fixturePath, buildPngWithChunks([ihdr])); + const { spawn } = createSpawnSpy([{ kind: "missing" }]); + hidePathBinaries(); + vi.resetModules(); + vi.doMock("child_process", () => ({ spawn })); + const { probeMediaProfile } = await import("./ffprobe.js"); + try { + await expect(probeMediaProfile(fixturePath)).rejects.toThrow(/ffprobe/i); + } finally { + rmSync(fixtureDir, { recursive: true, force: true }); + } + }); + // `profile` matters now: the packet refinement is an allowlist on AAC-LC, // because the 1024-sample formula is wrong for LD/ELD/HE and unverified for // the rest. An unprofiled "aac" stream deliberately keeps its container @@ -715,11 +1000,12 @@ describe("extractPngMetadataFromBuffer cICP ordering", () => { it("does not emit color space until IHDR provides width and height", () => { const ihdr = pngChunk("IHDR", [0, 0, 0, 1, 0, 0, 0, 1, 16, 2, 0, 0, 0]); const cicp = pngChunk("cICP", [9, 16, 0, 1]); + const idat = pngChunk("IDAT", [0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01]); const iend = pngChunk("IEND", []); // cICP before IHDR is invalid PNG ordering; make sure we don't return // zero-sized metadata in that case. - const malformed = buildPngWithChunks([cicp, ihdr, iend]); + const malformed = buildPngWithChunks([cicp, ihdr, idat, iend]); expect(extractPngMetadataFromBuffer(malformed)).toEqual({ width: 1, height: 1, diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index 72a06f5b1..006d6ef08 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -1,9 +1,9 @@ // fallow-ignore-file code-duplication complexity import { spawn } from "child_process"; -import { readFileSync } from "fs"; +import { createReadStream, readFileSync, statSync } from "fs"; import * as zlib from "node:zlib"; import { StringDecoder } from "node:string_decoder"; -import { basename, extname } from "path"; +import { basename } from "path"; import { redactTelemetryString } from "@hyperframes/core"; import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js"; import { ManagedChildProcess } from "./managedChildProcess.js"; @@ -147,6 +147,14 @@ const finalVideoFrameTimestampSignalCaches = new WeakMap< Map> >(); const audioMetadataCache = new Map>(); +interface MediaProbeCacheEntry { + identity: string; + promise: Promise; +} + +const mediaProbeOutputCache = new Map(); +const mediaProbeOutputSignalCaches = new WeakMap>(); +const MEDIA_PROBE_OUTPUT_CACHE_MAX_ENTRIES = 128; // FFmpeg's built-in AAC encoder emits AAC-LC, which has 1024 samples per packet. const AAC_LC_SAMPLES_PER_PACKET = 1024; @@ -213,11 +221,13 @@ interface FFProbeStream { color_primaries?: string; color_space?: string; tags?: Record; + disposition?: { attached_pic?: number }; } interface FFProbeFormat { duration?: string; bit_rate?: string; + format_name?: string; } interface FFProbeOutput { @@ -231,6 +241,226 @@ interface StillImageMetadata { colorSpace: VideoColorSpace | null; } +export interface MediaProbeProfile { + hasVideoStream: boolean; + hasAudioStream: boolean; + visualKind: "none" | "still" | "moving"; +} + +const STILL_IMAGE_DEMUXERS = new Set([ + "apng", + "bmp_pipe", + "dds_pipe", + "dpx_pipe", + "exr_pipe", + "gif", + "ico", + "image2", + "image2pipe", + "jpeg_pipe", + "jxl_pipe", + "png_pipe", + "qdraw_pipe", + "sgi_pipe", + "svg_pipe", + "tiff_pipe", + "webp_pipe", +]); + +async function hasAvifFileBrand(filePath: string): Promise { + try { + const chunks: Buffer[] = []; + for await (const chunk of createReadStream(filePath, { start: 0, end: 4095 })) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const bytes = Buffer.concat(chunks); + if (bytes.length < 16 || bytes.toString("ascii", 4, 8) !== "ftyp") return false; + const boxSize = bytes.readUInt32BE(0); + if (boxSize < 16 || boxSize > bytes.length) return false; + const avifBrands = new Set(["avif", "avis"]); + if (avifBrands.has(bytes.toString("ascii", 8, 12))) return true; + for (let offset = 16; offset + 4 <= boxSize; offset += 4) { + if (avifBrands.has(bytes.toString("ascii", offset, offset + 4))) return true; + } + return false; + } catch { + return false; + } +} + +async function isStillImageVisual(output: FFProbeOutput, filePath: string): Promise { + const formatNames = (output.format.format_name ?? "") + .split(",") + .map((name) => name.trim().toLowerCase()) + .filter(Boolean); + return ( + formatNames.some((name) => STILL_IMAGE_DEMUXERS.has(name)) || (await hasAvifFileBrand(filePath)) + ); +} + +function isPngImageProbe(output: FFProbeOutput): boolean { + const formatNames = (output.format.format_name ?? "") + .split(",") + .map((name) => name.trim().toLowerCase()); + return ( + formatNames.includes("png_pipe") || + output.streams.some( + (stream) => stream.codec_type === "video" && stream.codec_name?.toLowerCase() === "png", + ) + ); +} + +function mediaFileIdentity(filePath: string): string | null { + try { + const stat = statSync(filePath, { bigint: true }); + return [stat.dev, stat.ino, stat.size, stat.mtimeNs, stat.ctimeNs].join(":"); + } catch { + return null; + } +} + +async function probeMediaOutput(filePath: string, signal?: AbortSignal): Promise { + let cache = mediaProbeOutputCache; + if (signal) { + cache = mediaProbeOutputSignalCaches.get(signal) ?? new Map(); + mediaProbeOutputSignalCaches.set(signal, cache); + } + + const identity = mediaFileIdentity(filePath); + const cached = cache.get(filePath); + if (identity !== null && cached?.identity === identity) { + // The no-signal fallback is process-scoped, so touch its entries to make + // the fixed-size map an LRU. Signal-owned maps are released with their + // render and do not need process-lifetime eviction. + if (!signal) { + cache.delete(filePath); + cache.set(filePath, cached); + } + return cached.promise; + } + const promise = runFfprobe( + filePath, + ["-print_format", "json", "-show_format", "-show_streams"], + signal, + ).then(parseProbeJson); + if (identity !== null) { + cache.set(filePath, { identity, promise }); + if (!signal && cache.size > MEDIA_PROBE_OUTPUT_CACHE_MAX_ENTRIES) { + const oldestPath = cache.keys().next().value; + if (oldestPath !== undefined) cache.delete(oldestPath); + } + } + promise.catch(() => { + if (cache.get(filePath)?.promise === promise) { + cache.delete(filePath); + } + }); + return promise; +} + +class StructurallyIncompletePngError extends Error {} + +async function readFileRange( + filePath: string, + start: number, + length: number, + signal?: AbortSignal, +): Promise { + const chunks: Buffer[] = []; + for await (const chunk of createReadStream(filePath, { + start, + end: start + length - 1, + highWaterMark: length, + signal, + })) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +async function hasCompletePngStructure(filePath: string, signal?: AbortSignal): Promise { + try { + signal?.throwIfAborted(); + const fileSize = statSync(filePath).size; + // Range streams are read-only and skip large IDAT payloads without loading + // the entire image into memory. + const signature = await readFileRange(filePath, 0, 8, signal); + signal?.throwIfAborted(); + if ( + signature.length !== 8 || + !signature.equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) + ) { + return false; + } + + let seenHeader = false; + let seenImageData = false; + let offset = 8; + while (offset + 12 <= fileSize) { + signal?.throwIfAborted(); + const chunkHeader = await readFileRange(filePath, offset, 8, signal); + signal?.throwIfAborted(); + if (chunkHeader.length !== 8) return false; + const chunkLength = chunkHeader.readUInt32BE(0); + const chunkEnd = offset + 12 + chunkLength; + if (chunkEnd > fileSize) return false; + const chunkType = chunkHeader.toString("ascii", 4, 8); + if (chunkType === "IHDR") seenHeader = chunkLength === 13 && offset === 8; + if (chunkType === "IDAT") seenImageData = true; + if (chunkType === "IEND") return seenHeader && seenImageData && chunkLength === 0; + offset = chunkEnd; + } + return false; + } catch (error) { + if (signal?.aborted) throw signal.reason ?? error; + return false; + } +} + +/** + * Probe stream capabilities without assuming the caller's element type. + * File extensions and HTTP MIME are deliberately ignored: extensionless + * assets and valid media served through generic CDN content types must work. + */ +export async function probeMediaProfile( + filePath: string, + options?: { signal?: AbortSignal }, +): Promise { + try { + const output = await probeMediaOutput(filePath, options?.signal); + options?.signal?.throwIfAborted(); + const videoStreams = output.streams.filter((stream) => stream.codec_type === "video"); + const hasMovingVideoStream = videoStreams.some( + (stream) => stream.disposition?.attached_pic !== 1, + ); + const isStillImage = videoStreams.length > 0 && (await isStillImageVisual(output, filePath)); + options?.signal?.throwIfAborted(); + if ( + isStillImage && + isPngImageProbe(output) && + !(await hasCompletePngStructure(filePath, options?.signal)) + ) { + throw new StructurallyIncompletePngError("[FFmpeg] PNG input is structurally incomplete"); + } + return { + hasVideoStream: videoStreams.length > 0, + hasAudioStream: output.streams.some((stream) => stream.codec_type === "audio"), + visualKind: isStillImage ? "still" : hasMovingVideoStream ? "moving" : "none", + }; + } catch (error) { + if (options?.signal?.aborted) throw options.signal.reason ?? error; + if (error instanceof StructurallyIncompletePngError) throw error; + // Preserve the PNG parser fallback used by extractMediaMetadata when the + // packaged ffprobe binary is unavailable. Signature parsing (not the file + // extension) keeps extensionless PNGs eligible for preflight. + const stillImage = extractStillImageMetadata(filePath); + if (stillImage && (await hasCompletePngStructure(filePath, options?.signal))) { + return { hasVideoStream: true, hasAudioStream: false, visualKind: "still" }; + } + throw error; + } +} + // node:zlib's crc32 is native and takes a running seed, so the chunk type and // the chunk data can be CRC'd in sequence without concatenating them into a // throwaway buffer: ~210 ms -> ~1.3 ms on a 12 MiB PNG. @@ -365,8 +595,6 @@ export function pixelFormatHasAlpha(pixelFormat: string): boolean { } function extractStillImageMetadata(filePath: string): StillImageMetadata | null { - if (extname(filePath).toLowerCase() !== ".png") return null; - try { return extractPngMetadataFromBuffer(readFileSync(filePath)); } catch { @@ -477,13 +705,7 @@ export async function extractMediaMetadata(filePath: string): Promise => { - const stdout = await runFfprobe( - filePath, - ["-print_format", "json", "-show_format", "-show_streams"], - options?.signal, - ); - const output = parseProbeJson(stdout); + const output = await probeMediaOutput(filePath, options?.signal); const audioStream = output.streams.find((s) => s.codec_type === "audio"); if (!audioStream) throw new Error("[FFmpeg] No audio stream found"); diff --git a/packages/producer/scripts/test-classification.mjs b/packages/producer/scripts/test-classification.mjs index 25d4d6610..20ad22f36 100644 --- a/packages/producer/scripts/test-classification.mjs +++ b/packages/producer/scripts/test-classification.mjs @@ -20,6 +20,9 @@ const INTEGRATION_TEST_FILES = new Set([ "src/services/distributed/renderChunk.test.ts", "src/services/fileServer.test.ts", "src/services/healthWorker.test.ts", + "src/services/assetMediaType.test.ts", + "src/services/htmlCompiler.mediaType.test.ts", + "src/services/render/stages/compileStage.mediaType.test.ts", "src/utils/audioRegression.test.ts", "src/utils/streamDurationParity.test.ts", ]); diff --git a/packages/producer/src/server.errorCode.test.ts b/packages/producer/src/server.errorCode.test.ts index 1479a1b3a..3348ab080 100644 --- a/packages/producer/src/server.errorCode.test.ts +++ b/packages/producer/src/server.errorCode.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; -import { extractSafeRenderErrorCode } from "./server.js"; +import { extractSafeRenderErrorCode, extractSafeRenderErrorMetadata } from "./server.js"; import { VideoExtractionStageError } from "./services/render/stages/extractVideosStage.js"; +import { AssetMediaTypeMismatchError } from "./services/assetMediaType.js"; describe("extractSafeRenderErrorCode", () => { it("preserves allowlisted typed extraction codes", () => { @@ -24,6 +25,21 @@ describe("extractSafeRenderErrorCode", () => { ); }); + it("transports stable ownership and retry policy for media-type mismatches", () => { + const error = new AssetMediaTypeMismatchError([ + { expected: "video", detected: "image", elementFingerprint: "0123456789abcdef" }, + ]); + expect(error.code).toBe("ASSET_MEDIA_TYPE_MISMATCH"); + expect(error.owner).toBe("user"); + expect(error.retryable).toBe(false); + expect(extractSafeRenderErrorCode(error)).toBe("ASSET_MEDIA_TYPE_MISMATCH"); + expect(extractSafeRenderErrorMetadata(error)).toEqual({ + errorCode: "ASSET_MEDIA_TYPE_MISMATCH", + errorOwner: "user", + retryable: false, + }); + }); + it("does not forward arbitrary codes or parse message text", () => { expect(extractSafeRenderErrorCode({ code: "INTERNAL_ERROR" })).toBeUndefined(); expect( diff --git a/packages/producer/src/server.ts b/packages/producer/src/server.ts index 6787f8a96..e840da909 100644 --- a/packages/producer/src/server.ts +++ b/packages/producer/src/server.ts @@ -120,6 +120,7 @@ interface PreparedRenderInput { const DEFAULT_SERVER_FPS = { num: 30, den: 1 } as const; const SAFE_RENDER_ERROR_CODES = new Set([ + "ASSET_MEDIA_TYPE_MISMATCH", "INVALID_VIDEO_METADATA", "VIDEO_SOURCE_UNRENDERABLE", "VIDEO_EXTRACTION_FAILED", @@ -135,6 +136,27 @@ export function extractSafeRenderErrorCode(error: unknown): string | undefined { return typeof code === "string" && SAFE_RENDER_ERROR_CODES.has(code) ? code : undefined; } +export interface SafeRenderErrorMetadata { + errorCode: string; + errorOwner?: "system" | "user"; + retryable?: boolean; +} + +/** Additive bounded metadata for typed producer failures. */ +export function extractSafeRenderErrorMetadata( + error: unknown, +): SafeRenderErrorMetadata | undefined { + const errorCode = extractSafeRenderErrorCode(error); + if (!errorCode || typeof error !== "object" || error === null) return undefined; + const owner = "owner" in error ? error.owner : undefined; + const retryable = "retryable" in error ? error.retryable : undefined; + return { + errorCode, + errorOwner: owner === "user" || owner === "system" ? owner : undefined, + retryable: typeof retryable === "boolean" ? retryable : undefined, + }; +} + function parseServerFps(value: unknown): RenderInput["fps"] { if (typeof value !== "number" && typeof value !== "string") return DEFAULT_SERVER_FPS; const parsed = parseFps(value); @@ -585,7 +607,7 @@ async function writeRenderStreamFailure(input: { return; } const errorMsg = error instanceof Error ? error.message : String(error); - const errorCode = extractSafeRenderErrorCode(error); + const safeError = extractSafeRenderErrorMetadata(error); const elapsedMs = Date.now() - startedAtMs; log.error("render-stream failed", { requestId, @@ -598,7 +620,9 @@ async function writeRenderStreamFailure(input: { type: "error", requestId, error: errorMsg, - errorCode, + errorCode: safeError?.errorCode, + errorOwner: safeError?.errorOwner, + retryable: safeError?.retryable, stage: job.currentStage, elapsedMs, errorDetails: job.errorDetails ?? null, @@ -747,7 +771,7 @@ export function createRenderHandlers(options: HandlerOptions = {}): RenderHandle } catch (error) { const durationMs = Date.now() - t0; const errorMsg = error instanceof Error ? error.message : String(error); - const errorCode = extractSafeRenderErrorCode(error); + const safeError = extractSafeRenderErrorMetadata(error); log.error("render failed", { requestId, durationMs, @@ -759,7 +783,9 @@ export function createRenderHandlers(options: HandlerOptions = {}): RenderHandle success: false, requestId, error: errorMsg, - errorCode, + errorCode: safeError?.errorCode, + errorOwner: safeError?.errorOwner, + retryable: safeError?.retryable, stage: job.currentStage, durationMs, errorDetails: job.errorDetails ?? null, diff --git a/packages/producer/src/services/assetMediaType.test.ts b/packages/producer/src/services/assetMediaType.test.ts new file mode 100644 index 000000000..6cd1a3351 --- /dev/null +++ b/packages/producer/src/services/assetMediaType.test.ts @@ -0,0 +1,206 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + ASSET_MEDIA_TYPE_MISMATCH, + AssetMediaTypeMismatchError, + preflightCompositionAssetMediaTypes, +} from "./assetMediaType.js"; +import { synthesizeMediaFixture } from "./mediaTypeTestFixtures.js"; + +describe("preflightCompositionAssetMediaTypes", () => { + const fixtureDir = mkdtempSync(join(tmpdir(), "hf-media-type-preflight-")); + const projectDir = join(fixtureDir, "project"); + const compiledDir = join(fixtureDir, "compiled"); + const stillPath = join(projectDir, "extensionless-still"); + const videoPath = join(projectDir, "extensionless-video"); + const audioPath = join(projectDir, "extensionless-audio"); + const mixedPath = join(projectDir, "extensionless-mixed"); + + beforeAll(() => { + mkdirSync(projectDir, { recursive: true }); + mkdirSync(compiledDir, { recursive: true }); + synthesizeMediaFixture([ + "-f", + "lavfi", + "-i", + "color=c=red:s=32x32:d=0.1", + "-frames:v", + "1", + "-c:v", + "png", + "-f", + "image2", + stillPath, + ]); + synthesizeMediaFixture([ + "-f", + "lavfi", + "-i", + "testsrc2=s=32x32:d=1:r=30", + "-c:v", + "mpeg4", + "-f", + "mp4", + videoPath, + ]); + synthesizeMediaFixture([ + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=1", + "-c:a", + "pcm_s16le", + "-f", + "wav", + audioPath, + ]); + synthesizeMediaFixture([ + "-f", + "lavfi", + "-i", + "testsrc2=s=32x32:d=1:r=30", + "-f", + "lavfi", + "-i", + "sine=frequency=880:duration=1", + "-shortest", + "-c:v", + "mpeg4", + "-c:a", + "aac", + "-f", + "mp4", + mixedPath, + ]); + }, 30_000); + + afterAll(() => { + if (existsSync(fixtureDir)) rmSync(fixtureDir, { recursive: true, force: true }); + }); + + function composition(input: { videoSrc?: string; audioSrc?: string; imageSrc?: string }) { + return { + videos: input.videoSrc + ? [ + { + id: "video-element", + src: input.videoSrc, + start: 0, + end: 1, + mediaStart: 0, + loop: false, + hasAudio: false, + }, + ] + : [], + audios: input.audioSrc + ? [ + { + id: "audio-element", + src: input.audioSrc, + start: 0, + end: 1, + mediaStart: 0, + layer: 0, + type: "audio" as const, + }, + ] + : [], + images: input.imageSrc + ? [{ id: "image-element", src: input.imageSrc, start: 0, end: 1 }] + : [], + }; + } + + async function run( + input: { videoSrc?: string; audioSrc?: string; imageSrc?: string }, + signal?: AbortSignal, + ) { + return preflightCompositionAssetMediaTypes({ + projectDir, + compiledDir, + composition: composition(input), + signal, + }); + } + + it("accepts valid extensionless image, video, and audio assets", async () => { + await expect( + run({ + imageSrc: "extensionless-still", + videoSrc: "extensionless-video", + audioSrc: "extensionless-audio", + }), + ).resolves.toBeUndefined(); + }); + + it("accepts a mixed audio/video container for an audio element", async () => { + await expect(run({ audioSrc: "extensionless-mixed" })).resolves.toBeUndefined(); + }); + + it.each([ + { name: "image under video", input: { videoSrc: "extensionless-still" }, detected: "image" }, + { name: "audio under video", input: { videoSrc: "extensionless-audio" }, detected: "audio" }, + { name: "video under image", input: { imageSrc: "extensionless-video" }, detected: "video" }, + { name: "audio under image", input: { imageSrc: "extensionless-audio" }, detected: "audio" }, + { name: "image under audio", input: { audioSrc: "extensionless-still" }, detected: "image" }, + { + name: "silent video under audio", + input: { audioSrc: "extensionless-video" }, + detected: "video", + }, + ])("fails deterministically for $name", async ({ input, detected }) => { + let caught: unknown; + try { + await run(input); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(AssetMediaTypeMismatchError); + expect(caught).toMatchObject({ + code: ASSET_MEDIA_TYPE_MISMATCH, + owner: "user", + retryable: false, + mismatches: [expect.objectContaining({ detected })], + }); + const serialized = JSON.stringify(caught); + expect(serialized).not.toContain(fixtureDir); + expect(serialized).not.toContain("extensionless-"); + expect((caught as Error).message).not.toContain("video-element"); + expect((caught as Error).message).not.toContain("audio-element"); + expect((caught as Error).message).not.toContain("image-element"); + }); + + it("catches the zero-video image-probe shape before extraction", async () => { + const mismatched = composition({ imageSrc: "extensionless-audio" }); + expect(mismatched.videos).toHaveLength(0); + await expect( + preflightCompositionAssetMediaTypes({ projectDir, compiledDir, composition: mismatched }), + ).rejects.toMatchObject({ + code: ASSET_MEDIA_TYPE_MISMATCH, + owner: "user", + retryable: false, + }); + }); + + it("does not relabel deterministic missing or corrupt media as a type mismatch", async () => { + writeFileSync(join(projectDir, "corrupt-media"), "not a media container"); + await expect(run({ videoSrc: "missing-media" })).resolves.toBeUndefined(); + await expect(run({ imageSrc: "corrupt-media" })).resolves.toBeUndefined(); + }); + + it("re-probes a path after its media contents are replaced", async () => { + const mutablePath = join(projectDir, "mutable-media"); + const signal = new AbortController().signal; + copyFileSync(audioPath, mutablePath); + await expect(run({ videoSrc: "mutable-media" }, signal)).rejects.toMatchObject({ + code: ASSET_MEDIA_TYPE_MISMATCH, + retryable: false, + }); + + copyFileSync(videoPath, mutablePath); + await expect(run({ videoSrc: "mutable-media" }, signal)).resolves.toBeUndefined(); + }); +}); diff --git a/packages/producer/src/services/assetMediaType.ts b/packages/producer/src/services/assetMediaType.ts new file mode 100644 index 000000000..cc410261a --- /dev/null +++ b/packages/producer/src/services/assetMediaType.ts @@ -0,0 +1,160 @@ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { + probeMediaProfile, + resolveProjectRelativeSrc, + type AudioElement, + type ImageElement, + type MediaProbeProfile, + type VideoElement, +} from "@hyperframes/engine"; +import { withMediaProbeSlot } from "../utils/mediaProbeConcurrency.js"; + +export const ASSET_MEDIA_TYPE_MISMATCH = "ASSET_MEDIA_TYPE_MISMATCH" as const; + +export type AssetElementMediaType = "audio" | "image" | "video"; +export type DetectedAssetMediaType = AssetElementMediaType | "unknown"; + +export interface AssetMediaTypeMismatch { + expected: AssetElementMediaType; + detected: DetectedAssetMediaType; + /** Stable, bounded correlation key. Never the raw element id or source. */ + elementFingerprint: string; +} + +export class AssetMediaTypeMismatchError extends Error { + readonly code = ASSET_MEDIA_TYPE_MISMATCH; + readonly owner = "user" as const; + readonly retryable = false as const; + readonly mismatches: readonly AssetMediaTypeMismatch[]; + + constructor(mismatches: readonly AssetMediaTypeMismatch[]) { + const expectedKinds = [...new Set(mismatches.map((item) => item.expected))].sort().join(","); + super( + `${mismatches.length} composition asset(s) do not match their authored media element type` + + (expectedKinds ? ` (expected: ${expectedKinds})` : ""), + ); + this.name = "AssetMediaTypeMismatchError"; + this.mismatches = mismatches; + } +} + +function fingerprintElementId(elementId: string): string { + return createHash("sha256").update(elementId).digest("hex").slice(0, 16); +} + +function detectedAssetMediaType(profile: MediaProbeProfile): DetectedAssetMediaType { + if (profile.visualKind === "moving") return "video"; + if (profile.visualKind === "still") return "image"; + if (profile.hasAudioStream) return "audio"; + return "unknown"; +} + +function mediaProfileMatchesElementType( + expected: AssetElementMediaType, + profile: MediaProbeProfile, +): boolean { + if (expected === "audio") return profile.hasAudioStream; + if (expected === "image") return profile.visualKind === "still"; + return profile.visualKind === "moving"; +} + +export function assertAssetMediaTypeProfile( + expected: AssetElementMediaType, + profile: MediaProbeProfile, + elementIdentity: string, +): void { + if (mediaProfileMatchesElementType(expected, profile)) return; + throw new AssetMediaTypeMismatchError([ + { + expected, + detected: detectedAssetMediaType(profile), + elementFingerprint: fingerprintElementId(elementIdentity), + }, + ]); +} + +interface CompositionMediaAssets { + videos: readonly VideoElement[]; + audios: readonly AudioElement[]; + images: readonly ImageElement[]; +} + +interface MediaReference { + id: string; + src: string; + expected: AssetElementMediaType; +} + +function isRemoteOrInlineSource(src: string): boolean { + return /^(?:https?:|data:|blob:|about:)/i.test(src.trim()); +} + +/** + * Fail early when a successfully-probed local asset cannot satisfy the HTML + * element that owns it. Missing, corrupt, remote-at-runtime, and unprobeable + * inputs deliberately remain untouched so their existing source/download/ + * invalid-media classification is preserved downstream. + */ +export async function preflightCompositionAssetMediaTypes(input: { + projectDir: string; + compiledDir: string; + composition: CompositionMediaAssets; + signal?: AbortSignal; +}): Promise { + const references: MediaReference[] = [ + ...input.composition.videos.map((asset) => ({ + id: asset.id, + src: asset.src, + expected: "video" as const, + })), + ...input.composition.audios.map((asset) => ({ + id: asset.id, + src: asset.src, + expected: "audio" as const, + })), + ...input.composition.images.map((asset) => ({ + id: asset.id, + src: asset.src, + expected: "image" as const, + })), + ]; + + const byPath = new Map(); + for (const reference of references) { + if (!reference.src || isRemoteOrInlineSource(reference.src)) continue; + const resolvedPath = resolveProjectRelativeSrc( + reference.src, + input.projectDir, + input.compiledDir, + ); + if (!existsSync(resolvedPath)) continue; + byPath.set(resolvedPath, [...(byPath.get(resolvedPath) ?? []), reference]); + } + + const mismatches: AssetMediaTypeMismatch[] = []; + const entries = [...byPath]; + await Promise.all( + entries.map(([resolvedPath, pathReferences]) => + withMediaProbeSlot(async () => { + let profile: MediaProbeProfile; + try { + profile = await probeMediaProfile(resolvedPath, { signal: input.signal }); + } catch (error) { + if (input.signal?.aborted) throw input.signal.reason ?? error; + return; + } + for (const reference of pathReferences) { + if (mediaProfileMatchesElementType(reference.expected, profile)) continue; + mismatches.push({ + expected: reference.expected, + detected: detectedAssetMediaType(profile), + elementFingerprint: fingerprintElementId(reference.id), + }); + } + }), + ), + ); + + if (mismatches.length > 0) throw new AssetMediaTypeMismatchError(mismatches); +} diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 7712e995b..a7d1deb96 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -989,6 +989,7 @@ export async function buildLocalExecutionPlan( forceScreenshot, log, assertNotAborted, + abortSignal, compiled, composition, width, diff --git a/packages/producer/src/services/htmlCompiler.mediaType.test.ts b/packages/producer/src/services/htmlCompiler.mediaType.test.ts new file mode 100644 index 000000000..11a724b52 --- /dev/null +++ b/packages/producer/src/services/htmlCompiler.mediaType.test.ts @@ -0,0 +1,135 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { compileForRender } from "./htmlCompiler.js"; +import { synthesizeMediaFixture } from "./mediaTypeTestFixtures.js"; +import { Semaphore } from "../utils/semaphore.js"; +import { sharedMediaProbeSemaphore } from "../utils/mediaProbeConcurrency.js"; + +describe("compileForRender media-type ownership", () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-compiler-media-type-")); + const downloadDir = join(projectDir, "downloads"); + const stillPath = join(projectDir, "extensionless-still"); + const videoPath = join(projectDir, "extensionless-video"); + + beforeAll(() => { + mkdirSync(downloadDir, { recursive: true }); + synthesizeMediaFixture([ + "-f", + "lavfi", + "-i", + "color=c=blue:s=32x32:d=0.1", + "-frames:v", + "1", + "-c:v", + "png", + "-f", + "image2", + stillPath, + ]); + synthesizeMediaFixture([ + "-f", + "lavfi", + "-i", + "testsrc2=s=32x32:d=1:r=30", + "-c:v", + "mpeg4", + "-f", + "mp4", + videoPath, + ]); + }, 30_000); + + afterAll(() => { + if (existsSync(projectDir)) rmSync(projectDir, { recursive: true, force: true }); + }); + + async function compile(mediaMarkup: string) { + const htmlPath = join(projectDir, "index.html"); + writeFileSync( + htmlPath, + ` +
+ ${mediaMarkup} +
+ `, + ); + return compileForRender(projectDir, htmlPath, downloadDir); + } + + it("does not leak image-under-video through the generic no-video-stream path", async () => { + await expect( + compile(''), + ).rejects.toMatchObject({ + code: "ASSET_MEDIA_TYPE_MISMATCH", + owner: "user", + retryable: false, + }); + }); + + it("does not silently drop a silent video authored as audio", async () => { + await expect( + compile(''), + ).rejects.toMatchObject({ + code: "ASSET_MEDIA_TYPE_MISMATCH", + owner: "user", + retryable: false, + }); + }); + + it("shares a four-wide media-probe limiter across parallel sub-compositions", async () => { + const originalAcquire = Semaphore.prototype.acquire; + const limiterInstances = new Set(); + let maxActive = 0; + const acquireSpy = vi + .spyOn(Semaphore.prototype, "acquire") + .mockImplementation(async function (this: Semaphore) { + const release = await originalAcquire.call(this); + limiterInstances.add(this); + maxActive = Math.max(maxActive, this.activeCount); + return release; + }); + + try { + const subCompositions: string[] = []; + for (let index = 0; index < 8; index += 1) { + const name = `probe-${index}.html`; + writeFileSync( + join(projectDir, name), + ` +
+ +
+ `, + ); + subCompositions.push( + `
`, + ); + } + const htmlPath = join(projectDir, "index.html"); + writeFileSync( + htmlPath, + ` +
+ ${subCompositions.join("\n")} +
+ `, + ); + + await compileForRender(projectDir, htmlPath, downloadDir); + await vi.waitFor( + () => { + expect(sharedMediaProbeSemaphore.activeCount).toBe(0); + expect(sharedMediaProbeSemaphore.waitingCount).toBe(0); + }, + { timeout: 30_000, interval: 5 }, + ); + + expect(limiterInstances.size).toBe(1); + expect(maxActive).toBe(4); + } finally { + acquireSpy.mockRestore(); + } + }); +}); diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts index 8e24f92c9..a69be7e02 100644 --- a/packages/producer/src/services/htmlCompiler.test.ts +++ b/packages/producer/src/services/htmlCompiler.test.ts @@ -14,6 +14,7 @@ import { detectRenderModeHints, detectShaderTransitionUsage, detectThreeDTransformUsage, + discoverMediaFromBrowser, discoverAudioVolumeAutomationFromTimeline, inlineExternalScripts, localizeRemoteMediaSources, @@ -23,6 +24,67 @@ import { } from "./htmlCompiler.js"; import { validateNoSystemFonts } from "./render/planValidation.js"; +describe("discoverMediaFromBrowser", () => { + async function discover(html: string, currentSrcById: Record) { + const { document } = parseHTML(html); + for (const [id, currentSrc] of Object.entries(currentSrcById)) { + const element = document.getElementById(id); + if (element) Object.defineProperty(element, "currentSrc", { value: currentSrc }); + } + const previousDocument = Reflect.get(globalThis, "document"); + Reflect.set(globalThis, "document", document); + try { + return await discoverMediaFromBrowser({ evaluate: async (collect) => collect() } as never); + } finally { + if (previousDocument === undefined) Reflect.deleteProperty(globalThis, "document"); + else Reflect.set(globalThis, "document", previousDocument); + } + } + + it("uses the selected currentSrc from a variable-bound nested source", async () => { + const media = await discover( + ``, + { clip: "https://cdn.example/runtime.webm" }, + ); + + expect(media).toHaveLength(1); + expect(media[0]).toMatchObject({ + id: "clip", + tagName: "video", + src: "https://cdn.example/runtime.webm", + }); + }); + + it("discovers variable-bound images with the same generated id as the static parser", async () => { + const media = await discover( + ``, + {}, + ); + + expect(media).toHaveLength(1); + expect(media[0]).toMatchObject({ id: "hf-img-1", tagName: "image" }); + }); + + it("discovers the owning image for a variable-bound picture source", async () => { + const media = await discover( + ` + + + `, + { hero: "https://cdn.example/runtime.avif" }, + ); + + expect(media).toHaveLength(1); + expect(media[0]).toMatchObject({ + id: "hero", + tagName: "image", + src: "https://cdn.example/runtime.avif", + }); + }); +}); + describe("injectSdkPositionEditsRenderScript", () => { it("injects before when SDK position-edit markers are present", () => { const html = diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index d45dd5d39..e55090bab 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -49,7 +49,9 @@ import { parseAudioElements, type AudioElement, type AudioVolumeKeyframe, + type MediaProbeProfile, analyzeKeyframeIntervals, + probeMediaProfile, } from "@hyperframes/engine"; import { assertPublicHttpsUrl, downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js"; import type { Page } from "puppeteer-core"; @@ -61,6 +63,8 @@ import { prepareAnimatedGifInputs } from "./animatedGifPrep.js"; import { createStudioPositionSeekReapplyScript } from "@hyperframes/studio-server/manual-edits-render-script"; import { getPositionEditsRenderScript } from "@hyperframes/core/runtime/position-edits-render"; import { defaultLogger, type ProducerLogger } from "../logger.js"; +import { assertAssetMediaTypeProfile } from "./assetMediaType.js"; +import { withMediaProbeSlot } from "../utils/mediaProbeConcurrency.js"; export interface CompiledComposition { html: string; @@ -408,6 +412,7 @@ async function resolveMediaDuration( baseDir: string, downloadDir: string, tagName: string, + elementIdentity: string, ): Promise<{ duration: number; resolvedPath: string }> { let filePath = src; @@ -428,25 +433,39 @@ async function resolveMediaDuration( return { duration: 0, resolvedPath: filePath }; } - let metadata: { durationSeconds: number }; - if (tagName === "video") { - metadata = await extractMediaMetadata(filePath); - } else { + return withMediaProbeSlot(async () => { + let profile: MediaProbeProfile; try { - metadata = await extractAudioMetadata(filePath); - } catch { - // Source file has no audio stream (e.g. a silent video used as an audio src). - // Return duration 0 so the element is excluded from the composition gracefully, - // matching how missing files and failed downloads are already handled above. - return { duration: 0, resolvedPath: filePath }; + profile = await probeMediaProfile(filePath); + } catch (error) { + // Preserve the historical split: invalid video sources surface their + // probe failure, while invalid/unreadable audio sources resolve to zero + // duration and are excluded by the compiler. + if (tagName !== "video") return { duration: 0, resolvedPath: filePath }; + throw error; } - } + assertAssetMediaTypeProfile(tagName === "video" ? "video" : "audio", profile, elementIdentity); - const fileDuration = metadata.durationSeconds; - const effectiveDuration = fileDuration - mediaStart; - const duration = effectiveDuration > 0 ? effectiveDuration : fileDuration; + let metadata: { durationSeconds: number }; + if (tagName === "video") { + metadata = await extractMediaMetadata(filePath); + } else { + try { + metadata = await extractAudioMetadata(filePath); + } catch { + // Source file has no audio stream (e.g. a silent video used as an audio src). + // Return duration 0 so the element is excluded from the composition gracefully, + // matching how missing files and failed downloads are already handled above. + return { duration: 0, resolvedPath: filePath }; + } + } - return { duration, resolvedPath: filePath }; + const fileDuration = metadata.durationSeconds; + const effectiveDuration = fileDuration - mediaStart; + const duration = effectiveDuration > 0 ? effectiveDuration : fileDuration; + + return { duration, resolvedPath: filePath }; + }); } /** @@ -470,7 +489,7 @@ async function compileHtmlFile( // Phase 1: Resolve missing durations (parallel ffprobe) const resolvedResults = await Promise.all( mediaUnresolved.map((el) => - resolveMediaDuration(el.src!, el.mediaStart, baseDir, downloadDir, el.tagName).then( + resolveMediaDuration(el.src!, el.mediaStart, baseDir, downloadDir, el.tagName, el.id).then( ({ duration }) => ({ id: el.id, duration }), ), ), @@ -493,6 +512,7 @@ async function compileHtmlFile( baseDir, downloadDir, el.tagName, + el.id, ); return { id: el.id, tagName: el.tagName, duration: el.duration, maxDuration, src: el.src! }; }), @@ -1992,7 +2012,10 @@ export async function compileForRender( if (isHttpUrl(video.src)) continue; const videoPath = resolve(projectDir, video.src); const reencode = `ffmpeg -i "${video.src}" -c:v libx264 -r 30 -g 30 -keyint_min 30 -movflags +faststart -c:a copy output.mp4`; - Promise.all([analyzeKeyframeIntervals(videoPath), extractMediaMetadata(videoPath)]) + Promise.all([ + withMediaProbeSlot(() => analyzeKeyframeIntervals(videoPath)), + withMediaProbeSlot(() => extractMediaMetadata(videoPath)), + ]) .then(([analysis, metadata]) => { if (analysis.isProblematic) { defaultLogger.warn( @@ -2056,7 +2079,7 @@ export async function compileForRender( */ export interface BrowserMediaElement { id: string; - tagName: "video" | "audio"; + tagName: "video" | "audio" | "image"; src: string; start: number; end: number; @@ -2090,13 +2113,29 @@ export async function discoverMediaFromBrowser(page: Page): Promise(); + let autoImageId = 0; + document.querySelectorAll("img[src]").forEach((image) => { + if (!image.id) autoImageIds.set(image, `hf-img-${autoImageId++}`); + }); + + const mediaEls = new Set( + document.querySelectorAll("video[data-start], audio[data-start], img[data-var-src]"), + ); + // A variable-bound changes the owning image's currentSrc; + // the fallback itself does not necessarily carry data-var-src. + document.querySelectorAll("picture source[data-var-src]").forEach((source) => { + const image = source.closest("picture")?.querySelector("img"); + if (image) mediaEls.add(image); + }); mediaEls.forEach((el) => { - const htmlEl = el as HTMLVideoElement | HTMLAudioElement; - const id = htmlEl.id; + const htmlEl = el as HTMLVideoElement | HTMLAudioElement | HTMLImageElement; + const isImage = htmlEl.tagName.toLowerCase() === "img"; + const id = htmlEl.id || (isImage ? autoImageIds.get(htmlEl) : undefined); if (!id) return; - const src = htmlEl.src || htmlEl.getAttribute("src") || ""; + // currentSrc is authoritative for