mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 14:50:02 +00:00
fix(producer): reject asset media type mismatches (#2937)
* fix(producer): reject asset media type mismatches * fix(engine): document read-only AVIF probe * fix(engine): bound read-only AVIF brand probe * fix(producer): make media preflight lifecycle-safe * fix(producer): reconcile runtime media before preflight * fix(engine): avoid writable file-open detection * fix(producer): close runtime media preflight gaps
This commit is contained in:
James Russo
@@ -262,11 +262,19 @@ jobs:
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: changes
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
needs: [changes, producer-source-tests]
|
||||
# Keep the existing required `Test` context authoritative for producer
|
||||
# failures too. The dedicated producer matrix remains parallel and legible,
|
||||
# while this job fails closed if either lane fails or is cancelled.
|
||||
if: always() && needs.changes.outputs.code == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Require producer source tests
|
||||
if: needs.producer-source-tests.result != 'success'
|
||||
run: |
|
||||
echo "::error::Producer unit/integration tests did not succeed."
|
||||
exit 1
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
lfs: true
|
||||
|
||||
@@ -264,9 +264,11 @@ export {
|
||||
extractVideoMetadata,
|
||||
extractFinalVideoFrameTimestamp,
|
||||
extractAudioMetadata,
|
||||
probeMediaProfile,
|
||||
analyzeKeyframeIntervals,
|
||||
type VideoMetadata,
|
||||
type AudioMetadata,
|
||||
type MediaProbeProfile,
|
||||
type KeyframeAnalysis,
|
||||
} from "./utils/ffprobe.js";
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { EventEmitter } from "events";
|
||||
import { readFileSync } from "fs";
|
||||
import { spawnSync } from "child_process";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { basename, resolve } from "path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
@@ -113,6 +115,272 @@ describe("extractPngMetadataFromBuffer", () => {
|
||||
);
|
||||
expect(extractPngMetadataFromBuffer(fixture)?.colorSpace?.colorTransfer).toBe("smpte2084");
|
||||
});
|
||||
|
||||
it("keeps metadata fallback independent from full PNG integrity validation", () => {
|
||||
const ihdr = pngChunk("IHDR", [0, 0, 0, 1, 0, 0, 0, 1, 16, 2, 0, 0, 0]);
|
||||
expect(extractPngMetadataFromBuffer(buildPngWithChunks([ihdr]))).toMatchObject({
|
||||
width: 1,
|
||||
height: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("probeMediaProfile", () => {
|
||||
afterEach(() => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("child_process");
|
||||
});
|
||||
|
||||
it("classifies still, moving, audio-only, and mixed streams from probe data", async () => {
|
||||
const { spawn } = createSpawnSpy([
|
||||
{
|
||||
kind: "exit",
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [{ codec_type: "video", codec_name: "png" }],
|
||||
format: { format_name: "png_pipe" },
|
||||
}),
|
||||
},
|
||||
{
|
||||
kind: "exit",
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [{ codec_type: "video", codec_name: "h264" }],
|
||||
format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
|
||||
}),
|
||||
},
|
||||
{
|
||||
kind: "exit",
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [{ codec_type: "audio", codec_name: "mp3" }],
|
||||
format: { format_name: "mp3" },
|
||||
}),
|
||||
},
|
||||
{
|
||||
kind: "exit",
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [{ codec_type: "video" }, { codec_type: "audio" }],
|
||||
format: { format_name: "matroska,webm" },
|
||||
}),
|
||||
},
|
||||
]);
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
|
||||
const validPngPath = resolve(
|
||||
__dirname,
|
||||
"../../../producer/tests/hdr-regression/src/hdr-photo-pq.png",
|
||||
);
|
||||
await expect(probeMediaProfile(validPngPath)).resolves.toEqual({
|
||||
hasVideoStream: true,
|
||||
hasAudioStream: false,
|
||||
visualKind: "still",
|
||||
});
|
||||
await expect(probeMediaProfile("/tmp/extensionless-video")).resolves.toEqual({
|
||||
hasVideoStream: true,
|
||||
hasAudioStream: false,
|
||||
visualKind: "moving",
|
||||
});
|
||||
await expect(probeMediaProfile("/tmp/extensionless-audio")).resolves.toEqual({
|
||||
hasVideoStream: false,
|
||||
hasAudioStream: true,
|
||||
visualKind: "none",
|
||||
});
|
||||
await expect(probeMediaProfile("/tmp/mixed-av")).resolves.toEqual({
|
||||
hasVideoStream: true,
|
||||
hasAudioStream: true,
|
||||
visualKind: "moving",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat attached cover art in an audio container as an image asset", async () => {
|
||||
const { spawn } = createSpawnSpy([
|
||||
{
|
||||
kind: "exit",
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [
|
||||
{ codec_type: "video", disposition: { attached_pic: 1 } },
|
||||
{ codec_type: "audio" },
|
||||
],
|
||||
format: { format_name: "mp3" },
|
||||
}),
|
||||
},
|
||||
]);
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
await expect(probeMediaProfile("/tmp/audio-with-cover")).resolves.toMatchObject({
|
||||
hasAudioStream: true,
|
||||
visualKind: "none",
|
||||
});
|
||||
});
|
||||
|
||||
it("deduplicates probes within one cancellation scope without sharing across scopes", async () => {
|
||||
const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-media-probe-cache-"));
|
||||
const fixturePath = resolve(fixtureDir, "asset");
|
||||
writeFileSync(fixturePath, "cache identity only");
|
||||
const outcome = {
|
||||
kind: "exit" as const,
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [{ codec_type: "video", codec_name: "h264" }],
|
||||
format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
|
||||
}),
|
||||
};
|
||||
const { spawn, calls } = createSpawnSpy([outcome, outcome]);
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
const firstSignal = new AbortController().signal;
|
||||
const secondSignal = new AbortController().signal;
|
||||
try {
|
||||
await Promise.all([
|
||||
probeMediaProfile(fixturePath, { signal: firstSignal }),
|
||||
probeMediaProfile(fixturePath, { signal: firstSignal }),
|
||||
]);
|
||||
expect(calls).toHaveLength(1);
|
||||
await probeMediaProfile(fixturePath, { signal: secondSignal });
|
||||
expect(calls).toHaveLength(2);
|
||||
} finally {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds the process-scoped probe cache", async () => {
|
||||
const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-media-probe-lru-"));
|
||||
const fixturePaths = Array.from({ length: 129 }, (_, index) =>
|
||||
resolve(fixtureDir, `asset-${index}`),
|
||||
);
|
||||
for (const fixturePath of fixturePaths) writeFileSync(fixturePath, "probe identity");
|
||||
const outcome = {
|
||||
kind: "exit" as const,
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [{ codec_type: "audio", codec_name: "aac" }],
|
||||
format: { format_name: "aac" },
|
||||
}),
|
||||
};
|
||||
const { spawn, calls } = createSpawnSpy([outcome]);
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
try {
|
||||
for (const fixturePath of fixturePaths) await probeMediaProfile(fixturePath);
|
||||
await probeMediaProfile(fixturePaths[0]!);
|
||||
expect(calls).toHaveLength(130);
|
||||
} finally {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("classifies extensionless AVIF from its ISO-BMFF brand instead of the generic mov demuxer", async () => {
|
||||
const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-avif-profile-"));
|
||||
const fixturePath = resolve(fixtureDir, "asset");
|
||||
const ftyp = Buffer.alloc(24);
|
||||
ftyp.writeUInt32BE(24, 0);
|
||||
ftyp.write("ftyp", 4, 4, "ascii");
|
||||
ftyp.write("avif", 8, 4, "ascii");
|
||||
ftyp.writeUInt32BE(0, 12);
|
||||
ftyp.write("mif1", 16, 4, "ascii");
|
||||
ftyp.write("avif", 20, 4, "ascii");
|
||||
writeFileSync(fixturePath, ftyp);
|
||||
const { spawn } = createSpawnSpy([
|
||||
{
|
||||
kind: "exit",
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [{ codec_type: "video", codec_name: "av1" }],
|
||||
format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
|
||||
}),
|
||||
},
|
||||
]);
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
try {
|
||||
await expect(probeMediaProfile(fixturePath)).resolves.toMatchObject({
|
||||
hasVideoStream: true,
|
||||
hasAudioStream: false,
|
||||
visualKind: "still",
|
||||
});
|
||||
} finally {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("uses any non-attached video stream when cover art precedes moving video", async () => {
|
||||
const { spawn } = createSpawnSpy([
|
||||
{
|
||||
kind: "exit",
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [
|
||||
{ codec_type: "video", disposition: { attached_pic: 1 } },
|
||||
{ codec_type: "video", codec_name: "h264" },
|
||||
{ codec_type: "audio" },
|
||||
],
|
||||
format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
|
||||
}),
|
||||
},
|
||||
]);
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
await expect(probeMediaProfile("/tmp/video-with-cover")).resolves.toMatchObject({
|
||||
visualKind: "moving",
|
||||
});
|
||||
});
|
||||
|
||||
it.skipIf(spawnSync("ffprobe", ["-version"]).status !== 0)(
|
||||
"rejects an IHDR-only truncated PNG even when ffprobe accepts png_pipe",
|
||||
async () => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("child_process");
|
||||
const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-truncated-png-profile-"));
|
||||
const fixturePath = resolve(fixtureDir, "asset");
|
||||
const ihdr = pngChunk("IHDR", [0, 0, 0, 1, 0, 0, 0, 1, 16, 2, 0, 0, 0]);
|
||||
writeFileSync(fixturePath, buildPngWithChunks([ihdr]));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
try {
|
||||
await expect(probeMediaProfile(fixturePath)).rejects.toThrow();
|
||||
} finally {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("does not turn an aborted PNG probe into a successful metadata fallback", async () => {
|
||||
type KillableFakeProc = FakeProc & { kill: (signal?: NodeJS.Signals) => boolean };
|
||||
const spawn = () => {
|
||||
const proc = new EventEmitter() as KillableFakeProc;
|
||||
proc.stdout = new EventEmitter();
|
||||
proc.stderr = new EventEmitter();
|
||||
proc.kill = vi.fn(() => {
|
||||
process.nextTick(() => proc.emit("close", null, "SIGTERM"));
|
||||
return true;
|
||||
});
|
||||
process.nextTick(() => proc.emit("spawn"));
|
||||
return proc;
|
||||
};
|
||||
const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-aborted-png-profile-"));
|
||||
const fixturePath = resolve(fixtureDir, "asset");
|
||||
writeFileSync(fixturePath, buildMinimalPng());
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
const controller = new AbortController();
|
||||
try {
|
||||
const pending = probeMediaProfile(fixturePath, { signal: controller.signal });
|
||||
controller.abort(new Error("render cancelled"));
|
||||
await expect(pending).rejects.toThrow("render cancelled");
|
||||
} finally {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
interface SpawnCall {
|
||||
@@ -222,6 +490,23 @@ describe("ffprobe missing-binary fallback", () => {
|
||||
expect(calls[0]?.args.slice(0, 2)).toEqual(["-v", "error"]);
|
||||
});
|
||||
|
||||
it("does not accept an incomplete PNG through the missing-binary fallback", async () => {
|
||||
const fixtureDir = mkdtempSync(resolve(tmpdir(), "hf-truncated-png-fallback-"));
|
||||
const fixturePath = resolve(fixtureDir, "asset");
|
||||
const ihdr = pngChunk("IHDR", [0, 0, 0, 1, 0, 0, 0, 1, 16, 2, 0, 0, 0]);
|
||||
writeFileSync(fixturePath, buildPngWithChunks([ihdr]));
|
||||
const { spawn } = createSpawnSpy([{ kind: "missing" }]);
|
||||
hidePathBinaries();
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
const { probeMediaProfile } = await import("./ffprobe.js");
|
||||
try {
|
||||
await expect(probeMediaProfile(fixturePath)).rejects.toThrow(/ffprobe/i);
|
||||
} finally {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// `profile` matters now: the packet refinement is an allowlist on AAC-LC,
|
||||
// because the 1024-sample formula is wrong for LD/ELD/HE and unverified for
|
||||
// the rest. An unprofiled "aac" stream deliberately keeps its container
|
||||
@@ -715,11 +1000,12 @@ describe("extractPngMetadataFromBuffer cICP ordering", () => {
|
||||
it("does not emit color space until IHDR provides width and height", () => {
|
||||
const ihdr = pngChunk("IHDR", [0, 0, 0, 1, 0, 0, 0, 1, 16, 2, 0, 0, 0]);
|
||||
const cicp = pngChunk("cICP", [9, 16, 0, 1]);
|
||||
const idat = pngChunk("IDAT", [0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01]);
|
||||
const iend = pngChunk("IEND", []);
|
||||
|
||||
// cICP before IHDR is invalid PNG ordering; make sure we don't return
|
||||
// zero-sized metadata in that case.
|
||||
const malformed = buildPngWithChunks([cicp, ihdr, iend]);
|
||||
const malformed = buildPngWithChunks([cicp, ihdr, idat, iend]);
|
||||
expect(extractPngMetadataFromBuffer(malformed)).toEqual({
|
||||
width: 1,
|
||||
height: 1,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
// fallow-ignore-file code-duplication complexity
|
||||
import { spawn } from "child_process";
|
||||
import { readFileSync } from "fs";
|
||||
import { createReadStream, readFileSync, statSync } from "fs";
|
||||
import * as zlib from "node:zlib";
|
||||
import { StringDecoder } from "node:string_decoder";
|
||||
import { basename, extname } from "path";
|
||||
import { basename } from "path";
|
||||
import { redactTelemetryString } from "@hyperframes/core";
|
||||
import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js";
|
||||
import { ManagedChildProcess } from "./managedChildProcess.js";
|
||||
@@ -147,6 +147,14 @@ const finalVideoFrameTimestampSignalCaches = new WeakMap<
|
||||
Map<string, Promise<number>>
|
||||
>();
|
||||
const audioMetadataCache = new Map<string, Promise<AudioMetadata>>();
|
||||
interface MediaProbeCacheEntry {
|
||||
identity: string;
|
||||
promise: Promise<FFProbeOutput>;
|
||||
}
|
||||
|
||||
const mediaProbeOutputCache = new Map<string, MediaProbeCacheEntry>();
|
||||
const mediaProbeOutputSignalCaches = new WeakMap<AbortSignal, Map<string, MediaProbeCacheEntry>>();
|
||||
const MEDIA_PROBE_OUTPUT_CACHE_MAX_ENTRIES = 128;
|
||||
// FFmpeg's built-in AAC encoder emits AAC-LC, which has 1024 samples per packet.
|
||||
const AAC_LC_SAMPLES_PER_PACKET = 1024;
|
||||
|
||||
@@ -213,11 +221,13 @@ interface FFProbeStream {
|
||||
color_primaries?: string;
|
||||
color_space?: string;
|
||||
tags?: Record<string, string>;
|
||||
disposition?: { attached_pic?: number };
|
||||
}
|
||||
|
||||
interface FFProbeFormat {
|
||||
duration?: string;
|
||||
bit_rate?: string;
|
||||
format_name?: string;
|
||||
}
|
||||
|
||||
interface FFProbeOutput {
|
||||
@@ -231,6 +241,226 @@ interface StillImageMetadata {
|
||||
colorSpace: VideoColorSpace | null;
|
||||
}
|
||||
|
||||
export interface MediaProbeProfile {
|
||||
hasVideoStream: boolean;
|
||||
hasAudioStream: boolean;
|
||||
visualKind: "none" | "still" | "moving";
|
||||
}
|
||||
|
||||
const STILL_IMAGE_DEMUXERS = new Set([
|
||||
"apng",
|
||||
"bmp_pipe",
|
||||
"dds_pipe",
|
||||
"dpx_pipe",
|
||||
"exr_pipe",
|
||||
"gif",
|
||||
"ico",
|
||||
"image2",
|
||||
"image2pipe",
|
||||
"jpeg_pipe",
|
||||
"jxl_pipe",
|
||||
"png_pipe",
|
||||
"qdraw_pipe",
|
||||
"sgi_pipe",
|
||||
"svg_pipe",
|
||||
"tiff_pipe",
|
||||
"webp_pipe",
|
||||
]);
|
||||
|
||||
async function hasAvifFileBrand(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of createReadStream(filePath, { start: 0, end: 4095 })) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
const bytes = Buffer.concat(chunks);
|
||||
if (bytes.length < 16 || bytes.toString("ascii", 4, 8) !== "ftyp") return false;
|
||||
const boxSize = bytes.readUInt32BE(0);
|
||||
if (boxSize < 16 || boxSize > bytes.length) return false;
|
||||
const avifBrands = new Set(["avif", "avis"]);
|
||||
if (avifBrands.has(bytes.toString("ascii", 8, 12))) return true;
|
||||
for (let offset = 16; offset + 4 <= boxSize; offset += 4) {
|
||||
if (avifBrands.has(bytes.toString("ascii", offset, offset + 4))) return true;
|
||||
}
|
||||
return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function isStillImageVisual(output: FFProbeOutput, filePath: string): Promise<boolean> {
|
||||
const formatNames = (output.format.format_name ?? "")
|
||||
.split(",")
|
||||
.map((name) => name.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
return (
|
||||
formatNames.some((name) => STILL_IMAGE_DEMUXERS.has(name)) || (await hasAvifFileBrand(filePath))
|
||||
);
|
||||
}
|
||||
|
||||
function isPngImageProbe(output: FFProbeOutput): boolean {
|
||||
const formatNames = (output.format.format_name ?? "")
|
||||
.split(",")
|
||||
.map((name) => name.trim().toLowerCase());
|
||||
return (
|
||||
formatNames.includes("png_pipe") ||
|
||||
output.streams.some(
|
||||
(stream) => stream.codec_type === "video" && stream.codec_name?.toLowerCase() === "png",
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function mediaFileIdentity(filePath: string): string | null {
|
||||
try {
|
||||
const stat = statSync(filePath, { bigint: true });
|
||||
return [stat.dev, stat.ino, stat.size, stat.mtimeNs, stat.ctimeNs].join(":");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function probeMediaOutput(filePath: string, signal?: AbortSignal): Promise<FFProbeOutput> {
|
||||
let cache = mediaProbeOutputCache;
|
||||
if (signal) {
|
||||
cache = mediaProbeOutputSignalCaches.get(signal) ?? new Map<string, MediaProbeCacheEntry>();
|
||||
mediaProbeOutputSignalCaches.set(signal, cache);
|
||||
}
|
||||
|
||||
const identity = mediaFileIdentity(filePath);
|
||||
const cached = cache.get(filePath);
|
||||
if (identity !== null && cached?.identity === identity) {
|
||||
// The no-signal fallback is process-scoped, so touch its entries to make
|
||||
// the fixed-size map an LRU. Signal-owned maps are released with their
|
||||
// render and do not need process-lifetime eviction.
|
||||
if (!signal) {
|
||||
cache.delete(filePath);
|
||||
cache.set(filePath, cached);
|
||||
}
|
||||
return cached.promise;
|
||||
}
|
||||
const promise = runFfprobe(
|
||||
filePath,
|
||||
["-print_format", "json", "-show_format", "-show_streams"],
|
||||
signal,
|
||||
).then(parseProbeJson);
|
||||
if (identity !== null) {
|
||||
cache.set(filePath, { identity, promise });
|
||||
if (!signal && cache.size > MEDIA_PROBE_OUTPUT_CACHE_MAX_ENTRIES) {
|
||||
const oldestPath = cache.keys().next().value;
|
||||
if (oldestPath !== undefined) cache.delete(oldestPath);
|
||||
}
|
||||
}
|
||||
promise.catch(() => {
|
||||
if (cache.get(filePath)?.promise === promise) {
|
||||
cache.delete(filePath);
|
||||
}
|
||||
});
|
||||
return promise;
|
||||
}
|
||||
|
||||
class StructurallyIncompletePngError extends Error {}
|
||||
|
||||
async function readFileRange(
|
||||
filePath: string,
|
||||
start: number,
|
||||
length: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of createReadStream(filePath, {
|
||||
start,
|
||||
end: start + length - 1,
|
||||
highWaterMark: length,
|
||||
signal,
|
||||
})) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async function hasCompletePngStructure(filePath: string, signal?: AbortSignal): Promise<boolean> {
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
const fileSize = statSync(filePath).size;
|
||||
// Range streams are read-only and skip large IDAT payloads without loading
|
||||
// the entire image into memory.
|
||||
const signature = await readFileRange(filePath, 0, 8, signal);
|
||||
signal?.throwIfAborted();
|
||||
if (
|
||||
signature.length !== 8 ||
|
||||
!signature.equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let seenHeader = false;
|
||||
let seenImageData = false;
|
||||
let offset = 8;
|
||||
while (offset + 12 <= fileSize) {
|
||||
signal?.throwIfAborted();
|
||||
const chunkHeader = await readFileRange(filePath, offset, 8, signal);
|
||||
signal?.throwIfAborted();
|
||||
if (chunkHeader.length !== 8) return false;
|
||||
const chunkLength = chunkHeader.readUInt32BE(0);
|
||||
const chunkEnd = offset + 12 + chunkLength;
|
||||
if (chunkEnd > fileSize) return false;
|
||||
const chunkType = chunkHeader.toString("ascii", 4, 8);
|
||||
if (chunkType === "IHDR") seenHeader = chunkLength === 13 && offset === 8;
|
||||
if (chunkType === "IDAT") seenImageData = true;
|
||||
if (chunkType === "IEND") return seenHeader && seenImageData && chunkLength === 0;
|
||||
offset = chunkEnd;
|
||||
}
|
||||
return false;
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw signal.reason ?? error;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe stream capabilities without assuming the caller's element type.
|
||||
* File extensions and HTTP MIME are deliberately ignored: extensionless
|
||||
* assets and valid media served through generic CDN content types must work.
|
||||
*/
|
||||
export async function probeMediaProfile(
|
||||
filePath: string,
|
||||
options?: { signal?: AbortSignal },
|
||||
): Promise<MediaProbeProfile> {
|
||||
try {
|
||||
const output = await probeMediaOutput(filePath, options?.signal);
|
||||
options?.signal?.throwIfAborted();
|
||||
const videoStreams = output.streams.filter((stream) => stream.codec_type === "video");
|
||||
const hasMovingVideoStream = videoStreams.some(
|
||||
(stream) => stream.disposition?.attached_pic !== 1,
|
||||
);
|
||||
const isStillImage = videoStreams.length > 0 && (await isStillImageVisual(output, filePath));
|
||||
options?.signal?.throwIfAborted();
|
||||
if (
|
||||
isStillImage &&
|
||||
isPngImageProbe(output) &&
|
||||
!(await hasCompletePngStructure(filePath, options?.signal))
|
||||
) {
|
||||
throw new StructurallyIncompletePngError("[FFmpeg] PNG input is structurally incomplete");
|
||||
}
|
||||
return {
|
||||
hasVideoStream: videoStreams.length > 0,
|
||||
hasAudioStream: output.streams.some((stream) => stream.codec_type === "audio"),
|
||||
visualKind: isStillImage ? "still" : hasMovingVideoStream ? "moving" : "none",
|
||||
};
|
||||
} catch (error) {
|
||||
if (options?.signal?.aborted) throw options.signal.reason ?? error;
|
||||
if (error instanceof StructurallyIncompletePngError) throw error;
|
||||
// Preserve the PNG parser fallback used by extractMediaMetadata when the
|
||||
// packaged ffprobe binary is unavailable. Signature parsing (not the file
|
||||
// extension) keeps extensionless PNGs eligible for preflight.
|
||||
const stillImage = extractStillImageMetadata(filePath);
|
||||
if (stillImage && (await hasCompletePngStructure(filePath, options?.signal))) {
|
||||
return { hasVideoStream: true, hasAudioStream: false, visualKind: "still" };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// node:zlib's crc32 is native and takes a running seed, so the chunk type and
|
||||
// the chunk data can be CRC'd in sequence without concatenating them into a
|
||||
// throwaway buffer: ~210 ms -> ~1.3 ms on a 12 MiB PNG.
|
||||
@@ -365,8 +595,6 @@ export function pixelFormatHasAlpha(pixelFormat: string): boolean {
|
||||
}
|
||||
|
||||
function extractStillImageMetadata(filePath: string): StillImageMetadata | null {
|
||||
if (extname(filePath).toLowerCase() !== ".png") return null;
|
||||
|
||||
try {
|
||||
return extractPngMetadataFromBuffer(readFileSync(filePath));
|
||||
} catch {
|
||||
@@ -477,13 +705,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
|
||||
|
||||
let output: FFProbeOutput | null = null;
|
||||
try {
|
||||
const stdout = await runFfprobe(filePath, [
|
||||
"-print_format",
|
||||
"json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
]);
|
||||
output = parseProbeJson(stdout);
|
||||
output = await probeMediaOutput(filePath);
|
||||
} catch (error) {
|
||||
if (!stillImage()) throw error;
|
||||
}
|
||||
@@ -679,12 +901,7 @@ export async function extractAudioMetadata(
|
||||
if (cached) return cached;
|
||||
|
||||
const probePromise = (async (): Promise<AudioMetadata> => {
|
||||
const stdout = await runFfprobe(
|
||||
filePath,
|
||||
["-print_format", "json", "-show_format", "-show_streams"],
|
||||
options?.signal,
|
||||
);
|
||||
const output = parseProbeJson(stdout);
|
||||
const output = await probeMediaOutput(filePath, options?.signal);
|
||||
const audioStream = output.streams.find((s) => s.codec_type === "audio");
|
||||
if (!audioStream) throw new Error("[FFmpeg] No audio stream found");
|
||||
|
||||
|
||||
@@ -20,6 +20,9 @@ const INTEGRATION_TEST_FILES = new Set([
|
||||
"src/services/distributed/renderChunk.test.ts",
|
||||
"src/services/fileServer.test.ts",
|
||||
"src/services/healthWorker.test.ts",
|
||||
"src/services/assetMediaType.test.ts",
|
||||
"src/services/htmlCompiler.mediaType.test.ts",
|
||||
"src/services/render/stages/compileStage.mediaType.test.ts",
|
||||
"src/utils/audioRegression.test.ts",
|
||||
"src/utils/streamDurationParity.test.ts",
|
||||
]);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { extractSafeRenderErrorCode } from "./server.js";
|
||||
import { extractSafeRenderErrorCode, extractSafeRenderErrorMetadata } from "./server.js";
|
||||
import { VideoExtractionStageError } from "./services/render/stages/extractVideosStage.js";
|
||||
import { AssetMediaTypeMismatchError } from "./services/assetMediaType.js";
|
||||
|
||||
describe("extractSafeRenderErrorCode", () => {
|
||||
it("preserves allowlisted typed extraction codes", () => {
|
||||
@@ -24,6 +25,21 @@ describe("extractSafeRenderErrorCode", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("transports stable ownership and retry policy for media-type mismatches", () => {
|
||||
const error = new AssetMediaTypeMismatchError([
|
||||
{ expected: "video", detected: "image", elementFingerprint: "0123456789abcdef" },
|
||||
]);
|
||||
expect(error.code).toBe("ASSET_MEDIA_TYPE_MISMATCH");
|
||||
expect(error.owner).toBe("user");
|
||||
expect(error.retryable).toBe(false);
|
||||
expect(extractSafeRenderErrorCode(error)).toBe("ASSET_MEDIA_TYPE_MISMATCH");
|
||||
expect(extractSafeRenderErrorMetadata(error)).toEqual({
|
||||
errorCode: "ASSET_MEDIA_TYPE_MISMATCH",
|
||||
errorOwner: "user",
|
||||
retryable: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not forward arbitrary codes or parse message text", () => {
|
||||
expect(extractSafeRenderErrorCode({ code: "INTERNAL_ERROR" })).toBeUndefined();
|
||||
expect(
|
||||
|
||||
@@ -120,6 +120,7 @@ interface PreparedRenderInput {
|
||||
|
||||
const DEFAULT_SERVER_FPS = { num: 30, den: 1 } as const;
|
||||
const SAFE_RENDER_ERROR_CODES = new Set<string>([
|
||||
"ASSET_MEDIA_TYPE_MISMATCH",
|
||||
"INVALID_VIDEO_METADATA",
|
||||
"VIDEO_SOURCE_UNRENDERABLE",
|
||||
"VIDEO_EXTRACTION_FAILED",
|
||||
@@ -135,6 +136,27 @@ export function extractSafeRenderErrorCode(error: unknown): string | undefined {
|
||||
return typeof code === "string" && SAFE_RENDER_ERROR_CODES.has(code) ? code : undefined;
|
||||
}
|
||||
|
||||
export interface SafeRenderErrorMetadata {
|
||||
errorCode: string;
|
||||
errorOwner?: "system" | "user";
|
||||
retryable?: boolean;
|
||||
}
|
||||
|
||||
/** Additive bounded metadata for typed producer failures. */
|
||||
export function extractSafeRenderErrorMetadata(
|
||||
error: unknown,
|
||||
): SafeRenderErrorMetadata | undefined {
|
||||
const errorCode = extractSafeRenderErrorCode(error);
|
||||
if (!errorCode || typeof error !== "object" || error === null) return undefined;
|
||||
const owner = "owner" in error ? error.owner : undefined;
|
||||
const retryable = "retryable" in error ? error.retryable : undefined;
|
||||
return {
|
||||
errorCode,
|
||||
errorOwner: owner === "user" || owner === "system" ? owner : undefined,
|
||||
retryable: typeof retryable === "boolean" ? retryable : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function parseServerFps(value: unknown): RenderInput["fps"] {
|
||||
if (typeof value !== "number" && typeof value !== "string") return DEFAULT_SERVER_FPS;
|
||||
const parsed = parseFps(value);
|
||||
@@ -585,7 +607,7 @@ async function writeRenderStreamFailure(input: {
|
||||
return;
|
||||
}
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
const errorCode = extractSafeRenderErrorCode(error);
|
||||
const safeError = extractSafeRenderErrorMetadata(error);
|
||||
const elapsedMs = Date.now() - startedAtMs;
|
||||
log.error("render-stream failed", {
|
||||
requestId,
|
||||
@@ -598,7 +620,9 @@ async function writeRenderStreamFailure(input: {
|
||||
type: "error",
|
||||
requestId,
|
||||
error: errorMsg,
|
||||
errorCode,
|
||||
errorCode: safeError?.errorCode,
|
||||
errorOwner: safeError?.errorOwner,
|
||||
retryable: safeError?.retryable,
|
||||
stage: job.currentStage,
|
||||
elapsedMs,
|
||||
errorDetails: job.errorDetails ?? null,
|
||||
@@ -747,7 +771,7 @@ export function createRenderHandlers(options: HandlerOptions = {}): RenderHandle
|
||||
} catch (error) {
|
||||
const durationMs = Date.now() - t0;
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
const errorCode = extractSafeRenderErrorCode(error);
|
||||
const safeError = extractSafeRenderErrorMetadata(error);
|
||||
log.error("render failed", {
|
||||
requestId,
|
||||
durationMs,
|
||||
@@ -759,7 +783,9 @@ export function createRenderHandlers(options: HandlerOptions = {}): RenderHandle
|
||||
success: false,
|
||||
requestId,
|
||||
error: errorMsg,
|
||||
errorCode,
|
||||
errorCode: safeError?.errorCode,
|
||||
errorOwner: safeError?.errorOwner,
|
||||
retryable: safeError?.retryable,
|
||||
stage: job.currentStage,
|
||||
durationMs,
|
||||
errorDetails: job.errorDetails ?? null,
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
ASSET_MEDIA_TYPE_MISMATCH,
|
||||
AssetMediaTypeMismatchError,
|
||||
preflightCompositionAssetMediaTypes,
|
||||
} from "./assetMediaType.js";
|
||||
import { synthesizeMediaFixture } from "./mediaTypeTestFixtures.js";
|
||||
|
||||
describe("preflightCompositionAssetMediaTypes", () => {
|
||||
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-media-type-preflight-"));
|
||||
const projectDir = join(fixtureDir, "project");
|
||||
const compiledDir = join(fixtureDir, "compiled");
|
||||
const stillPath = join(projectDir, "extensionless-still");
|
||||
const videoPath = join(projectDir, "extensionless-video");
|
||||
const audioPath = join(projectDir, "extensionless-audio");
|
||||
const mixedPath = join(projectDir, "extensionless-mixed");
|
||||
|
||||
beforeAll(() => {
|
||||
mkdirSync(projectDir, { recursive: true });
|
||||
mkdirSync(compiledDir, { recursive: true });
|
||||
synthesizeMediaFixture([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=red:s=32x32:d=0.1",
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-c:v",
|
||||
"png",
|
||||
"-f",
|
||||
"image2",
|
||||
stillPath,
|
||||
]);
|
||||
synthesizeMediaFixture([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=32x32:d=1:r=30",
|
||||
"-c:v",
|
||||
"mpeg4",
|
||||
"-f",
|
||||
"mp4",
|
||||
videoPath,
|
||||
]);
|
||||
synthesizeMediaFixture([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:duration=1",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
"-f",
|
||||
"wav",
|
||||
audioPath,
|
||||
]);
|
||||
synthesizeMediaFixture([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=32x32:d=1:r=30",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=880:duration=1",
|
||||
"-shortest",
|
||||
"-c:v",
|
||||
"mpeg4",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-f",
|
||||
"mp4",
|
||||
mixedPath,
|
||||
]);
|
||||
}, 30_000);
|
||||
|
||||
afterAll(() => {
|
||||
if (existsSync(fixtureDir)) rmSync(fixtureDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function composition(input: { videoSrc?: string; audioSrc?: string; imageSrc?: string }) {
|
||||
return {
|
||||
videos: input.videoSrc
|
||||
? [
|
||||
{
|
||||
id: "video-element",
|
||||
src: input.videoSrc,
|
||||
start: 0,
|
||||
end: 1,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
audios: input.audioSrc
|
||||
? [
|
||||
{
|
||||
id: "audio-element",
|
||||
src: input.audioSrc,
|
||||
start: 0,
|
||||
end: 1,
|
||||
mediaStart: 0,
|
||||
layer: 0,
|
||||
type: "audio" as const,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
images: input.imageSrc
|
||||
? [{ id: "image-element", src: input.imageSrc, start: 0, end: 1 }]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
async function run(
|
||||
input: { videoSrc?: string; audioSrc?: string; imageSrc?: string },
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
return preflightCompositionAssetMediaTypes({
|
||||
projectDir,
|
||||
compiledDir,
|
||||
composition: composition(input),
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
it("accepts valid extensionless image, video, and audio assets", async () => {
|
||||
await expect(
|
||||
run({
|
||||
imageSrc: "extensionless-still",
|
||||
videoSrc: "extensionless-video",
|
||||
audioSrc: "extensionless-audio",
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("accepts a mixed audio/video container for an audio element", async () => {
|
||||
await expect(run({ audioSrc: "extensionless-mixed" })).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "image under video", input: { videoSrc: "extensionless-still" }, detected: "image" },
|
||||
{ name: "audio under video", input: { videoSrc: "extensionless-audio" }, detected: "audio" },
|
||||
{ name: "video under image", input: { imageSrc: "extensionless-video" }, detected: "video" },
|
||||
{ name: "audio under image", input: { imageSrc: "extensionless-audio" }, detected: "audio" },
|
||||
{ name: "image under audio", input: { audioSrc: "extensionless-still" }, detected: "image" },
|
||||
{
|
||||
name: "silent video under audio",
|
||||
input: { audioSrc: "extensionless-video" },
|
||||
detected: "video",
|
||||
},
|
||||
])("fails deterministically for $name", async ({ input, detected }) => {
|
||||
let caught: unknown;
|
||||
try {
|
||||
await run(input);
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AssetMediaTypeMismatchError);
|
||||
expect(caught).toMatchObject({
|
||||
code: ASSET_MEDIA_TYPE_MISMATCH,
|
||||
owner: "user",
|
||||
retryable: false,
|
||||
mismatches: [expect.objectContaining({ detected })],
|
||||
});
|
||||
const serialized = JSON.stringify(caught);
|
||||
expect(serialized).not.toContain(fixtureDir);
|
||||
expect(serialized).not.toContain("extensionless-");
|
||||
expect((caught as Error).message).not.toContain("video-element");
|
||||
expect((caught as Error).message).not.toContain("audio-element");
|
||||
expect((caught as Error).message).not.toContain("image-element");
|
||||
});
|
||||
|
||||
it("catches the zero-video image-probe shape before extraction", async () => {
|
||||
const mismatched = composition({ imageSrc: "extensionless-audio" });
|
||||
expect(mismatched.videos).toHaveLength(0);
|
||||
await expect(
|
||||
preflightCompositionAssetMediaTypes({ projectDir, compiledDir, composition: mismatched }),
|
||||
).rejects.toMatchObject({
|
||||
code: ASSET_MEDIA_TYPE_MISMATCH,
|
||||
owner: "user",
|
||||
retryable: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not relabel deterministic missing or corrupt media as a type mismatch", async () => {
|
||||
writeFileSync(join(projectDir, "corrupt-media"), "not a media container");
|
||||
await expect(run({ videoSrc: "missing-media" })).resolves.toBeUndefined();
|
||||
await expect(run({ imageSrc: "corrupt-media" })).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("re-probes a path after its media contents are replaced", async () => {
|
||||
const mutablePath = join(projectDir, "mutable-media");
|
||||
const signal = new AbortController().signal;
|
||||
copyFileSync(audioPath, mutablePath);
|
||||
await expect(run({ videoSrc: "mutable-media" }, signal)).rejects.toMatchObject({
|
||||
code: ASSET_MEDIA_TYPE_MISMATCH,
|
||||
retryable: false,
|
||||
});
|
||||
|
||||
copyFileSync(videoPath, mutablePath);
|
||||
await expect(run({ videoSrc: "mutable-media" }, signal)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import {
|
||||
probeMediaProfile,
|
||||
resolveProjectRelativeSrc,
|
||||
type AudioElement,
|
||||
type ImageElement,
|
||||
type MediaProbeProfile,
|
||||
type VideoElement,
|
||||
} from "@hyperframes/engine";
|
||||
import { withMediaProbeSlot } from "../utils/mediaProbeConcurrency.js";
|
||||
|
||||
export const ASSET_MEDIA_TYPE_MISMATCH = "ASSET_MEDIA_TYPE_MISMATCH" as const;
|
||||
|
||||
export type AssetElementMediaType = "audio" | "image" | "video";
|
||||
export type DetectedAssetMediaType = AssetElementMediaType | "unknown";
|
||||
|
||||
export interface AssetMediaTypeMismatch {
|
||||
expected: AssetElementMediaType;
|
||||
detected: DetectedAssetMediaType;
|
||||
/** Stable, bounded correlation key. Never the raw element id or source. */
|
||||
elementFingerprint: string;
|
||||
}
|
||||
|
||||
export class AssetMediaTypeMismatchError extends Error {
|
||||
readonly code = ASSET_MEDIA_TYPE_MISMATCH;
|
||||
readonly owner = "user" as const;
|
||||
readonly retryable = false as const;
|
||||
readonly mismatches: readonly AssetMediaTypeMismatch[];
|
||||
|
||||
constructor(mismatches: readonly AssetMediaTypeMismatch[]) {
|
||||
const expectedKinds = [...new Set(mismatches.map((item) => item.expected))].sort().join(",");
|
||||
super(
|
||||
`${mismatches.length} composition asset(s) do not match their authored media element type` +
|
||||
(expectedKinds ? ` (expected: ${expectedKinds})` : ""),
|
||||
);
|
||||
this.name = "AssetMediaTypeMismatchError";
|
||||
this.mismatches = mismatches;
|
||||
}
|
||||
}
|
||||
|
||||
function fingerprintElementId(elementId: string): string {
|
||||
return createHash("sha256").update(elementId).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
function detectedAssetMediaType(profile: MediaProbeProfile): DetectedAssetMediaType {
|
||||
if (profile.visualKind === "moving") return "video";
|
||||
if (profile.visualKind === "still") return "image";
|
||||
if (profile.hasAudioStream) return "audio";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function mediaProfileMatchesElementType(
|
||||
expected: AssetElementMediaType,
|
||||
profile: MediaProbeProfile,
|
||||
): boolean {
|
||||
if (expected === "audio") return profile.hasAudioStream;
|
||||
if (expected === "image") return profile.visualKind === "still";
|
||||
return profile.visualKind === "moving";
|
||||
}
|
||||
|
||||
export function assertAssetMediaTypeProfile(
|
||||
expected: AssetElementMediaType,
|
||||
profile: MediaProbeProfile,
|
||||
elementIdentity: string,
|
||||
): void {
|
||||
if (mediaProfileMatchesElementType(expected, profile)) return;
|
||||
throw new AssetMediaTypeMismatchError([
|
||||
{
|
||||
expected,
|
||||
detected: detectedAssetMediaType(profile),
|
||||
elementFingerprint: fingerprintElementId(elementIdentity),
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
interface CompositionMediaAssets {
|
||||
videos: readonly VideoElement[];
|
||||
audios: readonly AudioElement[];
|
||||
images: readonly ImageElement[];
|
||||
}
|
||||
|
||||
interface MediaReference {
|
||||
id: string;
|
||||
src: string;
|
||||
expected: AssetElementMediaType;
|
||||
}
|
||||
|
||||
function isRemoteOrInlineSource(src: string): boolean {
|
||||
return /^(?:https?:|data:|blob:|about:)/i.test(src.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail early when a successfully-probed local asset cannot satisfy the HTML
|
||||
* element that owns it. Missing, corrupt, remote-at-runtime, and unprobeable
|
||||
* inputs deliberately remain untouched so their existing source/download/
|
||||
* invalid-media classification is preserved downstream.
|
||||
*/
|
||||
export async function preflightCompositionAssetMediaTypes(input: {
|
||||
projectDir: string;
|
||||
compiledDir: string;
|
||||
composition: CompositionMediaAssets;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<void> {
|
||||
const references: MediaReference[] = [
|
||||
...input.composition.videos.map((asset) => ({
|
||||
id: asset.id,
|
||||
src: asset.src,
|
||||
expected: "video" as const,
|
||||
})),
|
||||
...input.composition.audios.map((asset) => ({
|
||||
id: asset.id,
|
||||
src: asset.src,
|
||||
expected: "audio" as const,
|
||||
})),
|
||||
...input.composition.images.map((asset) => ({
|
||||
id: asset.id,
|
||||
src: asset.src,
|
||||
expected: "image" as const,
|
||||
})),
|
||||
];
|
||||
|
||||
const byPath = new Map<string, MediaReference[]>();
|
||||
for (const reference of references) {
|
||||
if (!reference.src || isRemoteOrInlineSource(reference.src)) continue;
|
||||
const resolvedPath = resolveProjectRelativeSrc(
|
||||
reference.src,
|
||||
input.projectDir,
|
||||
input.compiledDir,
|
||||
);
|
||||
if (!existsSync(resolvedPath)) continue;
|
||||
byPath.set(resolvedPath, [...(byPath.get(resolvedPath) ?? []), reference]);
|
||||
}
|
||||
|
||||
const mismatches: AssetMediaTypeMismatch[] = [];
|
||||
const entries = [...byPath];
|
||||
await Promise.all(
|
||||
entries.map(([resolvedPath, pathReferences]) =>
|
||||
withMediaProbeSlot(async () => {
|
||||
let profile: MediaProbeProfile;
|
||||
try {
|
||||
profile = await probeMediaProfile(resolvedPath, { signal: input.signal });
|
||||
} catch (error) {
|
||||
if (input.signal?.aborted) throw input.signal.reason ?? error;
|
||||
return;
|
||||
}
|
||||
for (const reference of pathReferences) {
|
||||
if (mediaProfileMatchesElementType(reference.expected, profile)) continue;
|
||||
mismatches.push({
|
||||
expected: reference.expected,
|
||||
detected: detectedAssetMediaType(profile),
|
||||
elementFingerprint: fingerprintElementId(reference.id),
|
||||
});
|
||||
}
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
if (mismatches.length > 0) throw new AssetMediaTypeMismatchError(mismatches);
|
||||
}
|
||||
@@ -989,6 +989,7 @@ export async function buildLocalExecutionPlan(
|
||||
forceScreenshot,
|
||||
log,
|
||||
assertNotAborted,
|
||||
abortSignal,
|
||||
compiled,
|
||||
composition,
|
||||
width,
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { compileForRender } from "./htmlCompiler.js";
|
||||
import { synthesizeMediaFixture } from "./mediaTypeTestFixtures.js";
|
||||
import { Semaphore } from "../utils/semaphore.js";
|
||||
import { sharedMediaProbeSemaphore } from "../utils/mediaProbeConcurrency.js";
|
||||
|
||||
describe("compileForRender media-type ownership", () => {
|
||||
const projectDir = mkdtempSync(join(tmpdir(), "hf-compiler-media-type-"));
|
||||
const downloadDir = join(projectDir, "downloads");
|
||||
const stillPath = join(projectDir, "extensionless-still");
|
||||
const videoPath = join(projectDir, "extensionless-video");
|
||||
|
||||
beforeAll(() => {
|
||||
mkdirSync(downloadDir, { recursive: true });
|
||||
synthesizeMediaFixture([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=blue:s=32x32:d=0.1",
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-c:v",
|
||||
"png",
|
||||
"-f",
|
||||
"image2",
|
||||
stillPath,
|
||||
]);
|
||||
synthesizeMediaFixture([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=32x32:d=1:r=30",
|
||||
"-c:v",
|
||||
"mpeg4",
|
||||
"-f",
|
||||
"mp4",
|
||||
videoPath,
|
||||
]);
|
||||
}, 30_000);
|
||||
|
||||
afterAll(() => {
|
||||
if (existsSync(projectDir)) rmSync(projectDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function compile(mediaMarkup: string) {
|
||||
const htmlPath = join(projectDir, "index.html");
|
||||
writeFileSync(
|
||||
htmlPath,
|
||||
`<!doctype html><html><body>
|
||||
<div data-composition-id="root" data-width="320" data-height="180" data-duration="1">
|
||||
${mediaMarkup}
|
||||
</div>
|
||||
</body></html>`,
|
||||
);
|
||||
return compileForRender(projectDir, htmlPath, downloadDir);
|
||||
}
|
||||
|
||||
it("does not leak image-under-video through the generic no-video-stream path", async () => {
|
||||
await expect(
|
||||
compile('<video id="clip" src="extensionless-still" data-start="0"></video>'),
|
||||
).rejects.toMatchObject({
|
||||
code: "ASSET_MEDIA_TYPE_MISMATCH",
|
||||
owner: "user",
|
||||
retryable: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not silently drop a silent video authored as audio", async () => {
|
||||
await expect(
|
||||
compile('<audio id="voice" src="extensionless-video" data-start="0"></audio>'),
|
||||
).rejects.toMatchObject({
|
||||
code: "ASSET_MEDIA_TYPE_MISMATCH",
|
||||
owner: "user",
|
||||
retryable: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("shares a four-wide media-probe limiter across parallel sub-compositions", async () => {
|
||||
const originalAcquire = Semaphore.prototype.acquire;
|
||||
const limiterInstances = new Set<Semaphore>();
|
||||
let maxActive = 0;
|
||||
const acquireSpy = vi
|
||||
.spyOn(Semaphore.prototype, "acquire")
|
||||
.mockImplementation(async function (this: Semaphore) {
|
||||
const release = await originalAcquire.call(this);
|
||||
limiterInstances.add(this);
|
||||
maxActive = Math.max(maxActive, this.activeCount);
|
||||
return release;
|
||||
});
|
||||
|
||||
try {
|
||||
const subCompositions: string[] = [];
|
||||
for (let index = 0; index < 8; index += 1) {
|
||||
const name = `probe-${index}.html`;
|
||||
writeFileSync(
|
||||
join(projectDir, name),
|
||||
`<!doctype html><html><body>
|
||||
<div data-composition-id="sub-${index}" data-width="32" data-height="32" data-duration="1">
|
||||
<video id="video-${index}" src="extensionless-video" data-start="0"></video>
|
||||
</div>
|
||||
</body></html>`,
|
||||
);
|
||||
subCompositions.push(
|
||||
`<div data-composition-id="sub-${index}" data-composition-src="${name}" data-start="0" data-duration="1"></div>`,
|
||||
);
|
||||
}
|
||||
const htmlPath = join(projectDir, "index.html");
|
||||
writeFileSync(
|
||||
htmlPath,
|
||||
`<!doctype html><html><body>
|
||||
<div data-composition-id="root" data-width="320" data-height="180" data-duration="1">
|
||||
${subCompositions.join("\n")}
|
||||
</div>
|
||||
</body></html>`,
|
||||
);
|
||||
|
||||
await compileForRender(projectDir, htmlPath, downloadDir);
|
||||
await vi.waitFor(
|
||||
() => {
|
||||
expect(sharedMediaProbeSemaphore.activeCount).toBe(0);
|
||||
expect(sharedMediaProbeSemaphore.waitingCount).toBe(0);
|
||||
},
|
||||
{ timeout: 30_000, interval: 5 },
|
||||
);
|
||||
|
||||
expect(limiterInstances.size).toBe(1);
|
||||
expect(maxActive).toBe(4);
|
||||
} finally {
|
||||
acquireSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
detectRenderModeHints,
|
||||
detectShaderTransitionUsage,
|
||||
detectThreeDTransformUsage,
|
||||
discoverMediaFromBrowser,
|
||||
discoverAudioVolumeAutomationFromTimeline,
|
||||
inlineExternalScripts,
|
||||
localizeRemoteMediaSources,
|
||||
@@ -23,6 +24,67 @@ import {
|
||||
} from "./htmlCompiler.js";
|
||||
import { validateNoSystemFonts } from "./render/planValidation.js";
|
||||
|
||||
describe("discoverMediaFromBrowser", () => {
|
||||
async function discover(html: string, currentSrcById: Record<string, string>) {
|
||||
const { document } = parseHTML(html);
|
||||
for (const [id, currentSrc] of Object.entries(currentSrcById)) {
|
||||
const element = document.getElementById(id);
|
||||
if (element) Object.defineProperty(element, "currentSrc", { value: currentSrc });
|
||||
}
|
||||
const previousDocument = Reflect.get(globalThis, "document");
|
||||
Reflect.set(globalThis, "document", document);
|
||||
try {
|
||||
return await discoverMediaFromBrowser({ evaluate: async (collect) => collect() } as never);
|
||||
} finally {
|
||||
if (previousDocument === undefined) Reflect.deleteProperty(globalThis, "document");
|
||||
else Reflect.set(globalThis, "document", previousDocument);
|
||||
}
|
||||
}
|
||||
|
||||
it("uses the selected currentSrc from a variable-bound nested source", async () => {
|
||||
const media = await discover(
|
||||
`<video id="clip" data-start="0" data-end="1">
|
||||
<source src="fallback.mp4" data-var-src="clip_src" />
|
||||
</video>`,
|
||||
{ clip: "https://cdn.example/runtime.webm" },
|
||||
);
|
||||
|
||||
expect(media).toHaveLength(1);
|
||||
expect(media[0]).toMatchObject({
|
||||
id: "clip",
|
||||
tagName: "video",
|
||||
src: "https://cdn.example/runtime.webm",
|
||||
});
|
||||
});
|
||||
|
||||
it("discovers variable-bound images with the same generated id as the static parser", async () => {
|
||||
const media = await discover(
|
||||
`<img src="first.png" /><img src="fallback.png" data-var-src="hero_src" />`,
|
||||
{},
|
||||
);
|
||||
|
||||
expect(media).toHaveLength(1);
|
||||
expect(media[0]).toMatchObject({ id: "hf-img-1", tagName: "image" });
|
||||
});
|
||||
|
||||
it("discovers the owning image for a variable-bound picture source", async () => {
|
||||
const media = await discover(
|
||||
`<picture>
|
||||
<source src="fallback.webp" data-var-src="hero_src" />
|
||||
<img id="hero" src="fallback.png" />
|
||||
</picture>`,
|
||||
{ hero: "https://cdn.example/runtime.avif" },
|
||||
);
|
||||
|
||||
expect(media).toHaveLength(1);
|
||||
expect(media[0]).toMatchObject({
|
||||
id: "hero",
|
||||
tagName: "image",
|
||||
src: "https://cdn.example/runtime.avif",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("injectSdkPositionEditsRenderScript", () => {
|
||||
it("injects before </body> when SDK position-edit markers are present", () => {
|
||||
const html =
|
||||
|
||||
@@ -49,7 +49,9 @@ import {
|
||||
parseAudioElements,
|
||||
type AudioElement,
|
||||
type AudioVolumeKeyframe,
|
||||
type MediaProbeProfile,
|
||||
analyzeKeyframeIntervals,
|
||||
probeMediaProfile,
|
||||
} from "@hyperframes/engine";
|
||||
import { assertPublicHttpsUrl, downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js";
|
||||
import type { Page } from "puppeteer-core";
|
||||
@@ -61,6 +63,8 @@ import { prepareAnimatedGifInputs } from "./animatedGifPrep.js";
|
||||
import { createStudioPositionSeekReapplyScript } from "@hyperframes/studio-server/manual-edits-render-script";
|
||||
import { getPositionEditsRenderScript } from "@hyperframes/core/runtime/position-edits-render";
|
||||
import { defaultLogger, type ProducerLogger } from "../logger.js";
|
||||
import { assertAssetMediaTypeProfile } from "./assetMediaType.js";
|
||||
import { withMediaProbeSlot } from "../utils/mediaProbeConcurrency.js";
|
||||
|
||||
export interface CompiledComposition {
|
||||
html: string;
|
||||
@@ -408,6 +412,7 @@ async function resolveMediaDuration(
|
||||
baseDir: string,
|
||||
downloadDir: string,
|
||||
tagName: string,
|
||||
elementIdentity: string,
|
||||
): Promise<{ duration: number; resolvedPath: string }> {
|
||||
let filePath = src;
|
||||
|
||||
@@ -428,25 +433,39 @@ async function resolveMediaDuration(
|
||||
return { duration: 0, resolvedPath: filePath };
|
||||
}
|
||||
|
||||
let metadata: { durationSeconds: number };
|
||||
if (tagName === "video") {
|
||||
metadata = await extractMediaMetadata(filePath);
|
||||
} else {
|
||||
return withMediaProbeSlot(async () => {
|
||||
let profile: MediaProbeProfile;
|
||||
try {
|
||||
metadata = await extractAudioMetadata(filePath);
|
||||
} catch {
|
||||
// Source file has no audio stream (e.g. a silent video used as an audio src).
|
||||
// Return duration 0 so the element is excluded from the composition gracefully,
|
||||
// matching how missing files and failed downloads are already handled above.
|
||||
return { duration: 0, resolvedPath: filePath };
|
||||
profile = await probeMediaProfile(filePath);
|
||||
} catch (error) {
|
||||
// Preserve the historical split: invalid video sources surface their
|
||||
// probe failure, while invalid/unreadable audio sources resolve to zero
|
||||
// duration and are excluded by the compiler.
|
||||
if (tagName !== "video") return { duration: 0, resolvedPath: filePath };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
assertAssetMediaTypeProfile(tagName === "video" ? "video" : "audio", profile, elementIdentity);
|
||||
|
||||
const fileDuration = metadata.durationSeconds;
|
||||
const effectiveDuration = fileDuration - mediaStart;
|
||||
const duration = effectiveDuration > 0 ? effectiveDuration : fileDuration;
|
||||
let metadata: { durationSeconds: number };
|
||||
if (tagName === "video") {
|
||||
metadata = await extractMediaMetadata(filePath);
|
||||
} else {
|
||||
try {
|
||||
metadata = await extractAudioMetadata(filePath);
|
||||
} catch {
|
||||
// Source file has no audio stream (e.g. a silent video used as an audio src).
|
||||
// Return duration 0 so the element is excluded from the composition gracefully,
|
||||
// matching how missing files and failed downloads are already handled above.
|
||||
return { duration: 0, resolvedPath: filePath };
|
||||
}
|
||||
}
|
||||
|
||||
return { duration, resolvedPath: filePath };
|
||||
const fileDuration = metadata.durationSeconds;
|
||||
const effectiveDuration = fileDuration - mediaStart;
|
||||
const duration = effectiveDuration > 0 ? effectiveDuration : fileDuration;
|
||||
|
||||
return { duration, resolvedPath: filePath };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -470,7 +489,7 @@ async function compileHtmlFile(
|
||||
// Phase 1: Resolve missing durations (parallel ffprobe)
|
||||
const resolvedResults = await Promise.all(
|
||||
mediaUnresolved.map((el) =>
|
||||
resolveMediaDuration(el.src!, el.mediaStart, baseDir, downloadDir, el.tagName).then(
|
||||
resolveMediaDuration(el.src!, el.mediaStart, baseDir, downloadDir, el.tagName, el.id).then(
|
||||
({ duration }) => ({ id: el.id, duration }),
|
||||
),
|
||||
),
|
||||
@@ -493,6 +512,7 @@ async function compileHtmlFile(
|
||||
baseDir,
|
||||
downloadDir,
|
||||
el.tagName,
|
||||
el.id,
|
||||
);
|
||||
return { id: el.id, tagName: el.tagName, duration: el.duration, maxDuration, src: el.src! };
|
||||
}),
|
||||
@@ -1992,7 +2012,10 @@ export async function compileForRender(
|
||||
if (isHttpUrl(video.src)) continue;
|
||||
const videoPath = resolve(projectDir, video.src);
|
||||
const reencode = `ffmpeg -i "${video.src}" -c:v libx264 -r 30 -g 30 -keyint_min 30 -movflags +faststart -c:a copy output.mp4`;
|
||||
Promise.all([analyzeKeyframeIntervals(videoPath), extractMediaMetadata(videoPath)])
|
||||
Promise.all([
|
||||
withMediaProbeSlot(() => analyzeKeyframeIntervals(videoPath)),
|
||||
withMediaProbeSlot(() => extractMediaMetadata(videoPath)),
|
||||
])
|
||||
.then(([analysis, metadata]) => {
|
||||
if (analysis.isProblematic) {
|
||||
defaultLogger.warn(
|
||||
@@ -2056,7 +2079,7 @@ export async function compileForRender(
|
||||
*/
|
||||
export interface BrowserMediaElement {
|
||||
id: string;
|
||||
tagName: "video" | "audio";
|
||||
tagName: "video" | "audio" | "image";
|
||||
src: string;
|
||||
start: number;
|
||||
end: number;
|
||||
@@ -2090,13 +2113,29 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
|
||||
muted: boolean;
|
||||
}[] = [];
|
||||
|
||||
const mediaEls = document.querySelectorAll("video[data-start], audio[data-start]");
|
||||
const autoImageIds = new Map<Element, string>();
|
||||
let autoImageId = 0;
|
||||
document.querySelectorAll("img[src]").forEach((image) => {
|
||||
if (!image.id) autoImageIds.set(image, `hf-img-${autoImageId++}`);
|
||||
});
|
||||
|
||||
const mediaEls = new Set<Element>(
|
||||
document.querySelectorAll("video[data-start], audio[data-start], img[data-var-src]"),
|
||||
);
|
||||
// A variable-bound <picture><source> changes the owning image's currentSrc;
|
||||
// the <img> fallback itself does not necessarily carry data-var-src.
|
||||
document.querySelectorAll("picture source[data-var-src]").forEach((source) => {
|
||||
const image = source.closest("picture")?.querySelector("img");
|
||||
if (image) mediaEls.add(image);
|
||||
});
|
||||
mediaEls.forEach((el) => {
|
||||
const htmlEl = el as HTMLVideoElement | HTMLAudioElement;
|
||||
const id = htmlEl.id;
|
||||
const htmlEl = el as HTMLVideoElement | HTMLAudioElement | HTMLImageElement;
|
||||
const isImage = htmlEl.tagName.toLowerCase() === "img";
|
||||
const id = htmlEl.id || (isImage ? autoImageIds.get(htmlEl) : undefined);
|
||||
if (!id) return;
|
||||
|
||||
const src = htmlEl.src || htmlEl.getAttribute("src") || "";
|
||||
// currentSrc is authoritative for <video>/<audio><source> and responsive images.
|
||||
const src = htmlEl.currentSrc || htmlEl.src || htmlEl.getAttribute("src") || "";
|
||||
const start = parseFloat(htmlEl.getAttribute("data-start") || "0");
|
||||
const end = parseFloat(htmlEl.getAttribute("data-end") || "0");
|
||||
const duration = parseFloat(htmlEl.getAttribute("data-duration") || "0");
|
||||
@@ -2104,11 +2143,13 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
|
||||
const loop = htmlEl.hasAttribute("loop");
|
||||
const hasAudio = htmlEl.getAttribute("data-has-audio") === "true";
|
||||
const volume = parseFloat(htmlEl.getAttribute("data-volume") || "1");
|
||||
const muted = htmlEl.hasAttribute("muted") || htmlEl.muted;
|
||||
const muted =
|
||||
!isImage &&
|
||||
(htmlEl.hasAttribute("muted") || (htmlEl as HTMLVideoElement | HTMLAudioElement).muted);
|
||||
|
||||
results.push({
|
||||
id,
|
||||
tagName: htmlEl.tagName.toLowerCase(),
|
||||
tagName: isImage ? "image" : htmlEl.tagName.toLowerCase(),
|
||||
src,
|
||||
start,
|
||||
end,
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const probeTracker = vi.hoisted(() => {
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
let started = 0;
|
||||
let releaseGate = () => {};
|
||||
let gate: Promise<void>;
|
||||
|
||||
const reset = () => {
|
||||
active = 0;
|
||||
maxActive = 0;
|
||||
started = 0;
|
||||
gate = new Promise<void>((resolve) => {
|
||||
releaseGate = resolve;
|
||||
});
|
||||
};
|
||||
reset();
|
||||
|
||||
return {
|
||||
reset,
|
||||
release: () => releaseGate(),
|
||||
run: async <T>(value: T): Promise<T> => {
|
||||
active += 1;
|
||||
started += 1;
|
||||
maxActive = Math.max(maxActive, active);
|
||||
await gate;
|
||||
active -= 1;
|
||||
return value;
|
||||
},
|
||||
get maxActive() {
|
||||
return maxActive;
|
||||
},
|
||||
get started() {
|
||||
return started;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@hyperframes/engine", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@hyperframes/engine")>();
|
||||
return {
|
||||
...actual,
|
||||
analyzeKeyframeIntervals: async () =>
|
||||
probeTracker.run({ isProblematic: false, maxIntervalSeconds: 0 }),
|
||||
probeMediaProfile: async () =>
|
||||
probeTracker.run({ hasVideoStream: true, hasAudioStream: true, visualKind: "moving" }),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../utils/ffprobe.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../utils/ffprobe.js")>();
|
||||
return {
|
||||
...actual,
|
||||
extractMediaMetadata: async () => probeTracker.run({ durationSeconds: 1, isVFR: false }),
|
||||
};
|
||||
});
|
||||
|
||||
import { compileForRender } from "./htmlCompiler.js";
|
||||
import { preflightCompositionAssetMediaTypes } from "./assetMediaType.js";
|
||||
|
||||
describe("aggregate media-probe concurrency", () => {
|
||||
let projectDir: string | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
probeTracker.release();
|
||||
if (projectDir) rmSync(projectDir, { recursive: true, force: true });
|
||||
projectDir = undefined;
|
||||
});
|
||||
|
||||
it("caps overlapping compiler advisories and media-type preflight at four probes", async () => {
|
||||
probeTracker.reset();
|
||||
projectDir = mkdtempSync(join(tmpdir(), "hf-media-probe-concurrency-"));
|
||||
const mediaMarkup: string[] = [];
|
||||
for (let index = 0; index < 6; index += 1) {
|
||||
const src = `video-${index}.asset`;
|
||||
writeFileSync(join(projectDir, src), "fixture");
|
||||
mediaMarkup.push(
|
||||
`<video id="video-${index}" src="${src}" loop data-start="0" data-duration="1"></video>`,
|
||||
);
|
||||
}
|
||||
const htmlPath = join(projectDir, "index.html");
|
||||
writeFileSync(
|
||||
htmlPath,
|
||||
`<!doctype html><html><body>
|
||||
<div data-composition-id="root" data-width="320" data-height="180" data-duration="1">
|
||||
${mediaMarkup.join("\n")}
|
||||
</div>
|
||||
</body></html>`,
|
||||
);
|
||||
|
||||
const compiled = await compileForRender(projectDir, htmlPath, join(projectDir, "downloads"));
|
||||
const preflight = preflightCompositionAssetMediaTypes({
|
||||
projectDir,
|
||||
compiledDir: join(projectDir, "compiled"),
|
||||
composition: compiled,
|
||||
});
|
||||
|
||||
// Let all fire-and-forget advisory calls and the preflight contend for the
|
||||
// shared limiter while their mocked probe bodies remain blocked.
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
expect(probeTracker.maxActive).toBe(4);
|
||||
|
||||
probeTracker.release();
|
||||
await preflight;
|
||||
expect(probeTracker.started).toBe(18);
|
||||
expect(probeTracker.maxActive).toBe(4);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
export function synthesizeMediaFixture(args: string[]): void {
|
||||
const result = spawnSync("ffmpeg", ["-y", "-hide_banner", "-loglevel", "error", ...args]);
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`ffmpeg fixture synthesis failed: ${result.stderr.toString().slice(-400)}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import type { EngineConfig } from "@hyperframes/engine";
|
||||
import { runCompileStage } from "./compileStage.js";
|
||||
|
||||
const noopLog = {
|
||||
error: () => {},
|
||||
warn: () => {},
|
||||
info: () => {},
|
||||
debug: () => {},
|
||||
};
|
||||
|
||||
function createCfg(): EngineConfig {
|
||||
return {
|
||||
chromeArgs: [],
|
||||
chromePath: undefined,
|
||||
captureCostMultiplier: 1,
|
||||
format: "jpeg",
|
||||
jpegQuality: 80,
|
||||
concurrency: "auto",
|
||||
coresPerWorker: 2.5,
|
||||
minParallelFrames: 120,
|
||||
largeRenderThreshold: 1000,
|
||||
disableGpu: false,
|
||||
browserGpuMode: "software",
|
||||
enableBrowserPool: false,
|
||||
browserTimeout: 120_000,
|
||||
protocolTimeout: 300_000,
|
||||
forceScreenshot: false,
|
||||
enableChunkedEncode: false,
|
||||
chunkSizeFrames: 360,
|
||||
enableStreamingEncode: false,
|
||||
streamingEncodeMaxDurationSeconds: 240,
|
||||
ffmpegEncodeTimeout: 600_000,
|
||||
ffmpegProcessTimeout: 300_000,
|
||||
ffmpegStreamingTimeout: 600_000,
|
||||
hdr: false,
|
||||
hdrAutoDetect: true,
|
||||
audioGain: 1,
|
||||
frameDataUriCacheLimit: 256,
|
||||
frameDataUriCacheBytesLimitMb: 1500,
|
||||
playerReadyTimeout: 45_000,
|
||||
renderReadyTimeout: 15_000,
|
||||
verifyRuntime: true,
|
||||
debug: false,
|
||||
};
|
||||
}
|
||||
|
||||
describe("runCompileStage — asset media-type preflight", () => {
|
||||
let workDir: string | null = null;
|
||||
|
||||
afterEach(() => {
|
||||
if (workDir) rmSync(workDir, { recursive: true, force: true });
|
||||
workDir = null;
|
||||
});
|
||||
|
||||
it("rejects the zero-video audio-under-image shape before extraction", async () => {
|
||||
workDir = mkdtempSync(join(tmpdir(), "compile-stage-media-type-"));
|
||||
const projectDir = join(workDir, "project");
|
||||
mkdirSync(projectDir);
|
||||
const htmlPath = join(projectDir, "index.html");
|
||||
writeFileSync(
|
||||
htmlPath,
|
||||
`<!doctype html><html><body>
|
||||
<div data-composition-id="root" data-width="320" data-height="180" data-duration="1">
|
||||
<img id="hero" src="voice.asset" data-start="0" data-end="1" />
|
||||
</div>
|
||||
</body></html>`,
|
||||
);
|
||||
const audioPath = join(projectDir, "voice.asset");
|
||||
const synth = spawnSync("ffmpeg", [
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:duration=1",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
"-f",
|
||||
"wav",
|
||||
audioPath,
|
||||
]);
|
||||
expect(synth.status).toBe(0);
|
||||
|
||||
await expect(
|
||||
runCompileStage({
|
||||
projectDir,
|
||||
workDir,
|
||||
htmlPath,
|
||||
entryFile: "index.html",
|
||||
job: {
|
||||
id: "media-type-test",
|
||||
config: { fps: { num: 30, den: 1 }, quality: "standard" },
|
||||
status: "queued",
|
||||
progress: 0,
|
||||
currentStage: "Queued",
|
||||
createdAt: new Date(0),
|
||||
},
|
||||
cfg: createCfg(),
|
||||
needsAlpha: false,
|
||||
log: noopLog,
|
||||
assertNotAborted: () => {},
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: "ASSET_MEDIA_TYPE_MISMATCH",
|
||||
owner: "user",
|
||||
retryable: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -48,6 +48,7 @@ import {
|
||||
type CompositionMetadata,
|
||||
} from "../shared.js";
|
||||
import type { RenderJob } from "../../renderOrchestrator.js";
|
||||
import { preflightCompositionAssetMediaTypes } from "../../assetMediaType.js";
|
||||
|
||||
export interface CompileStageInput {
|
||||
projectDir: string;
|
||||
@@ -312,6 +313,13 @@ export async function runCompileStage(input: CompileStageInput): Promise<Compile
|
||||
width: compiled.width,
|
||||
height: compiled.height,
|
||||
};
|
||||
await preflightCompositionAssetMediaTypes({
|
||||
projectDir,
|
||||
compiledDir: join(workDir, "compiled"),
|
||||
composition,
|
||||
signal: abortSignal,
|
||||
});
|
||||
assertNotAborted();
|
||||
const { width, height } = composition;
|
||||
const effectiveResolution = adaptAspectAgnosticResolution(
|
||||
job.config.outputResolution,
|
||||
|
||||
@@ -11,6 +11,13 @@ import {
|
||||
// in beginframe mode even when lowMemoryMode demanded screenshot capture).
|
||||
const capturedCfgs: unknown[] = [];
|
||||
const capturedOptions: unknown[] = [];
|
||||
let mediaPreflightCallCount = 0;
|
||||
let mediaPreflightError: Error | null = null;
|
||||
let mediaPreflightSignal: AbortSignal | undefined;
|
||||
let mediaPreflightComposition: unknown;
|
||||
let afterMediaPreflight: (() => void) | null = null;
|
||||
let fileServerCloseCallCount = 0;
|
||||
let browserMediaResults: unknown[] = [];
|
||||
|
||||
type MockSession = {
|
||||
id: number;
|
||||
@@ -55,8 +62,28 @@ function resetRetryMocks() {
|
||||
createdSessions.length = 0;
|
||||
closedSessions.length = 0;
|
||||
durationProbeSessions.length = 0;
|
||||
mediaPreflightCallCount = 0;
|
||||
mediaPreflightError = null;
|
||||
mediaPreflightSignal = undefined;
|
||||
mediaPreflightComposition = undefined;
|
||||
afterMediaPreflight = null;
|
||||
fileServerCloseCallCount = 0;
|
||||
browserMediaResults = [];
|
||||
}
|
||||
|
||||
mock.module("../../assetMediaType.js", () => ({
|
||||
preflightCompositionAssetMediaTypes: async (input: {
|
||||
signal?: AbortSignal;
|
||||
composition?: unknown;
|
||||
}) => {
|
||||
mediaPreflightCallCount += 1;
|
||||
mediaPreflightSignal = input.signal;
|
||||
mediaPreflightComposition = input.composition;
|
||||
if (mediaPreflightError) throw mediaPreflightError;
|
||||
afterMediaPreflight?.();
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module("@hyperframes/engine", () => ({
|
||||
createCaptureSession: async (
|
||||
_url: string,
|
||||
@@ -126,14 +153,17 @@ mock.module("../../fileServer.js", () => ({
|
||||
createFileServer: async () => ({
|
||||
url: "http://127.0.0.1:0",
|
||||
port: 0,
|
||||
close: () => {},
|
||||
close: () => {
|
||||
fileServerCloseCallCount += 1;
|
||||
},
|
||||
addPreHeadScript: () => {},
|
||||
}),
|
||||
closeFileServerSafely: (fileServer: { close: () => void }) => fileServer.close(),
|
||||
VIRTUAL_TIME_SHIM: "",
|
||||
}));
|
||||
|
||||
mock.module("../../htmlCompiler.js", () => ({
|
||||
discoverMediaFromBrowser: async () => [],
|
||||
discoverMediaFromBrowser: async () => browserMediaResults,
|
||||
discoverAudioVolumeAutomationFromTimeline: async () => [],
|
||||
discoverVideoVisibilityFromTimeline: async () => [],
|
||||
recompileWithResolutions: async (c: unknown) => c,
|
||||
@@ -142,7 +172,11 @@ mock.module("../../htmlCompiler.js", () => ({
|
||||
|
||||
mock.module("../shared.js", () => ({
|
||||
BROWSER_MEDIA_EPSILON: 0.0001,
|
||||
projectBrowserEndToCompositionTimeline: () => 0,
|
||||
projectBrowserEndToCompositionTimeline: (
|
||||
existingStart: number,
|
||||
browserStart: number,
|
||||
browserEnd: number,
|
||||
) => browserEnd + (existingStart - browserStart),
|
||||
resolveBrowserMediaEnd: (_start: number, end: number, duration: number) =>
|
||||
Number.isFinite(duration) && duration > 0 ? _start + duration : end,
|
||||
writeCompiledArtifacts: () => {},
|
||||
@@ -238,6 +272,7 @@ function makeProbeInput(overrides: {
|
||||
debug: () => {},
|
||||
},
|
||||
assertNotAborted: () => {},
|
||||
abortSignal: undefined as AbortSignal | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -299,16 +334,132 @@ describe("hasVariableBoundMedia", () => {
|
||||
expect(hasVariableBoundMedia(html, { voice_src: "row-02.wav" })).toBe(true);
|
||||
});
|
||||
|
||||
it("does not probe unrelated overrides or image-only bindings", () => {
|
||||
it("ignores unrelated overrides and probes image-bound sources", () => {
|
||||
const audio = `<audio src="fallback.wav" data-var-src="voice_src"></audio>`;
|
||||
const image = `<img src="fallback.png" data-var-src="hero_src" />`;
|
||||
|
||||
expect(hasVariableBoundMedia(audio, { title: "Row 02" })).toBe(false);
|
||||
expect(hasVariableBoundMedia(image, { hero_src: "row-02.png" })).toBe(false);
|
||||
expect(hasVariableBoundMedia(image, { hero_src: "row-02.png" })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runProbeStage — forceScreenshot threading", () => {
|
||||
it("runs media-type preflight after the browser-reconciliation phase", async () => {
|
||||
mediaPreflightCallCount = 0;
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({});
|
||||
input.composition.duration = 5;
|
||||
input.compiled.html = "<div>static</div>";
|
||||
|
||||
await runProbeStage(input);
|
||||
|
||||
expect(mediaPreflightCallCount).toBe(1);
|
||||
});
|
||||
|
||||
it("reconciles a sub-composition image without losing its parent timeline offset", async () => {
|
||||
resetRetryMocks();
|
||||
browserMediaResults = [
|
||||
{
|
||||
id: "hero",
|
||||
tagName: "image",
|
||||
src: "runtime-video.asset",
|
||||
start: 0,
|
||||
end: 2,
|
||||
duration: 2,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
volume: 1,
|
||||
muted: false,
|
||||
},
|
||||
];
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({});
|
||||
input.composition.duration = 5;
|
||||
input.composition.images.push({ id: "hero", src: "fallback.png", start: 4, end: 6 });
|
||||
input.compiled.html =
|
||||
'<img id="hero" src="fallback.png" data-var-src="hero_src" data-start="0" data-end="2">';
|
||||
input.job.config.variables = { hero_src: "runtime-video.asset" };
|
||||
|
||||
await runProbeStage(input);
|
||||
|
||||
expect(input.composition.images[0]?.src).toBe("runtime-video.asset");
|
||||
expect(input.composition.images[0]?.start).toBe(4);
|
||||
expect(input.composition.images[0]?.end).toBe(6);
|
||||
expect(mediaPreflightComposition).toBe(input.composition);
|
||||
});
|
||||
|
||||
it("reconciles a nested source's selected runtime URL before media-type preflight", async () => {
|
||||
resetRetryMocks();
|
||||
browserMediaResults = [
|
||||
{
|
||||
id: "clip",
|
||||
tagName: "video",
|
||||
src: "runtime-still.asset",
|
||||
start: 0,
|
||||
end: 5,
|
||||
duration: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
volume: 1,
|
||||
muted: false,
|
||||
},
|
||||
];
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({});
|
||||
input.composition.duration = 5;
|
||||
input.composition.videos.push({
|
||||
id: "clip",
|
||||
src: "fallback.mp4",
|
||||
start: 0,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
});
|
||||
input.compiled.html = `<video id="clip" data-start="0" data-end="5">
|
||||
<source src="fallback.mp4" data-var-src="clip_src">
|
||||
</video>`;
|
||||
input.job.config.variables = { clip_src: "runtime-still.asset" };
|
||||
|
||||
await runProbeStage(input);
|
||||
|
||||
expect(input.composition.videos[0]?.src).toBe("runtime-still.asset");
|
||||
expect(mediaPreflightComposition).toBe(input.composition);
|
||||
});
|
||||
|
||||
it("passes cancellation through and closes probe-owned resources when preflight rejects", async () => {
|
||||
resetRetryMocks();
|
||||
mediaPreflightError = new Error("ASSET_MEDIA_TYPE_MISMATCH");
|
||||
const controller = new AbortController();
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({});
|
||||
input.abortSignal = controller.signal;
|
||||
|
||||
await expect(runProbeStage(input)).rejects.toThrow("ASSET_MEDIA_TYPE_MISMATCH");
|
||||
|
||||
expect(mediaPreflightSignal).toBe(controller.signal);
|
||||
expect(closeCaptureSessionCallCount).toBe(1);
|
||||
expect(fileServerCloseCallCount).toBe(1);
|
||||
mediaPreflightError = null;
|
||||
});
|
||||
|
||||
it("closes probe-owned resources when cancellation lands after an empty preflight", async () => {
|
||||
resetRetryMocks();
|
||||
const controller = new AbortController();
|
||||
afterMediaPreflight = () => controller.abort(new Error("render cancelled"));
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({});
|
||||
input.abortSignal = controller.signal;
|
||||
input.assertNotAborted = () => controller.signal.throwIfAborted();
|
||||
|
||||
await expect(runProbeStage(input)).rejects.toThrow("render cancelled");
|
||||
|
||||
expect(closeCaptureSessionCallCount).toBe(1);
|
||||
expect(fileServerCloseCallCount).toBe(1);
|
||||
});
|
||||
|
||||
it("launches a probe when a static-duration composition inserts video at runtime", async () => {
|
||||
capturedCfgs.length = 0;
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
@@ -480,6 +631,40 @@ describe("runProbeStage — decimal duration frame count", () => {
|
||||
});
|
||||
|
||||
describe("runProbeStage — transient browser error retry (#1687)", () => {
|
||||
async function runWithTransientInitializeError(message: string) {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
initializeSessionError = new Error(message);
|
||||
initializeSessionFailUntilAttempt = 1;
|
||||
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({ cfgForceScreenshot: false, stageForceScreenshot: false });
|
||||
const result = await runProbeStage(input);
|
||||
|
||||
expect(initializeSessionCallCount).toBe(2);
|
||||
expect(closeCaptureSessionCallCount).toBe(1);
|
||||
expect(result.duration).toBe(5);
|
||||
expect(result.probeSession).not.toBeNull();
|
||||
}
|
||||
|
||||
async function expectInitializeFailure(input: {
|
||||
message: string;
|
||||
expectedMessage: string;
|
||||
expectedAttempts: number;
|
||||
}) {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
initializeSessionError = new Error(input.message);
|
||||
initializeSessionFailUntilAttempt = 999;
|
||||
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const probeInput = makeProbeInput({ cfgForceScreenshot: false, stageForceScreenshot: false });
|
||||
|
||||
await expect(runProbeStage(probeInput)).rejects.toThrow(input.expectedMessage);
|
||||
expect(initializeSessionCallCount).toBe(input.expectedAttempts);
|
||||
expect(closeCaptureSessionCallCount).toBe(input.expectedAttempts);
|
||||
}
|
||||
|
||||
it("uses the replacement session after a BeginFrame liveness fallback", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
@@ -499,124 +684,42 @@ describe("runProbeStage — transient browser error retry (#1687)", () => {
|
||||
});
|
||||
|
||||
it("retries once on a transient 'Navigating frame was detached' error and succeeds", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
initializeSessionError = new Error("Navigating frame was detached");
|
||||
initializeSessionFailUntilAttempt = 1;
|
||||
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({ cfgForceScreenshot: false, stageForceScreenshot: false });
|
||||
|
||||
const result = await runProbeStage(input);
|
||||
|
||||
expect(initializeSessionCallCount).toBe(2);
|
||||
expect(closeCaptureSessionCallCount).toBe(1);
|
||||
expect(result.duration).toBe(5);
|
||||
expect(result.probeSession).not.toBeNull();
|
||||
await runWithTransientInitializeError("Navigating frame was detached");
|
||||
});
|
||||
|
||||
it("retries once on a browser-probe navigation timeout and succeeds", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
initializeSessionError = new Error("Navigation timeout of 60000 ms exceeded");
|
||||
initializeSessionFailUntilAttempt = 1;
|
||||
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({ cfgForceScreenshot: false, stageForceScreenshot: false });
|
||||
|
||||
const result = await runProbeStage(input);
|
||||
|
||||
expect(initializeSessionCallCount).toBe(2);
|
||||
expect(closeCaptureSessionCallCount).toBe(1);
|
||||
expect(result.duration).toBe(5);
|
||||
expect(result.probeSession).not.toBeNull();
|
||||
await runWithTransientInitializeError("Navigation timeout of 60000 ms exceeded");
|
||||
});
|
||||
|
||||
it("throws immediately on a non-transient error without retrying", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
initializeSessionError = new Error("FONT_FETCH_FAILED: Inter");
|
||||
initializeSessionFailUntilAttempt = 999;
|
||||
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({ cfgForceScreenshot: false, stageForceScreenshot: false });
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await runProbeStage(input);
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
|
||||
expect(caught).toBeInstanceOf(Error);
|
||||
expect((caught as Error).message).toContain("FONT_FETCH_FAILED");
|
||||
expect(initializeSessionCallCount).toBe(1);
|
||||
expect(closeCaptureSessionCallCount).toBe(1);
|
||||
await expectInitializeFailure({
|
||||
message: "FONT_FETCH_FAILED: Inter",
|
||||
expectedMessage: "FONT_FETCH_FAILED",
|
||||
expectedAttempts: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("throws after exhausting retry attempts on persistent transient errors", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
initializeSessionError = new Error("Target closed");
|
||||
initializeSessionFailUntilAttempt = 999;
|
||||
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({ cfgForceScreenshot: false, stageForceScreenshot: false });
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await runProbeStage(input);
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
|
||||
expect(caught).toBeInstanceOf(Error);
|
||||
expect((caught as Error).message).toContain("Target closed");
|
||||
expect(initializeSessionCallCount).toBe(2);
|
||||
expect(closeCaptureSessionCallCount).toBe(2);
|
||||
await expectInitializeFailure({
|
||||
message: "Target closed",
|
||||
expectedMessage: "Target closed",
|
||||
expectedAttempts: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("retries once on a pollHfReady zero-duration timeout (renderReady: false) and succeeds", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
initializeSessionError = new Error(
|
||||
await runWithTransientInitializeError(
|
||||
"[FrameCapture] Composition has zero duration.\n Runtime ready: false, __player: true, __hf.seek: true, GSAP timeline: true, data-duration: 53.3s",
|
||||
);
|
||||
initializeSessionFailUntilAttempt = 1;
|
||||
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({ cfgForceScreenshot: false, stageForceScreenshot: false });
|
||||
|
||||
const result = await runProbeStage(input);
|
||||
|
||||
expect(initializeSessionCallCount).toBe(2);
|
||||
expect(closeCaptureSessionCallCount).toBe(1);
|
||||
expect(result.duration).toBe(5);
|
||||
expect(result.probeSession).not.toBeNull();
|
||||
});
|
||||
|
||||
it("throws immediately on a permanent zero-duration error (renderReady: true — genuine authoring bug)", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
initializeSessionError = new Error(
|
||||
"[FrameCapture] Composition has zero duration.\n Runtime ready: true, __player: true, __hf.seek: true, GSAP timeline: false, data-duration: not set",
|
||||
);
|
||||
initializeSessionFailUntilAttempt = 999;
|
||||
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({ cfgForceScreenshot: false, stageForceScreenshot: false });
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await runProbeStage(input);
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
|
||||
expect(caught).toBeInstanceOf(Error);
|
||||
expect((caught as Error).message).toContain("Runtime ready: true");
|
||||
expect(initializeSessionCallCount).toBe(1);
|
||||
expect(closeCaptureSessionCallCount).toBe(1);
|
||||
await expectInitializeFailure({
|
||||
message:
|
||||
"[FrameCapture] Composition has zero duration.\n Runtime ready: true, __player: true, __hf.seek: true, GSAP timeline: false, data-duration: not set",
|
||||
expectedMessage: "Runtime ready: true",
|
||||
expectedAttempts: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("retries on a transient browser LAUNCH failure (createCaptureSession throws)", async () => {
|
||||
|
||||
@@ -50,7 +50,12 @@ import {
|
||||
recompileWithResolutions,
|
||||
resolveCompositionDurations,
|
||||
} from "../../htmlCompiler.js";
|
||||
import { createFileServer, type FileServerHandle, VIRTUAL_TIME_SHIM } from "../../fileServer.js";
|
||||
import {
|
||||
closeFileServerSafely,
|
||||
createFileServer,
|
||||
type FileServerHandle,
|
||||
VIRTUAL_TIME_SHIM,
|
||||
} from "../../fileServer.js";
|
||||
import type { ProducerLogger } from "../../../logger.js";
|
||||
import {
|
||||
BROWSER_MEDIA_EPSILON,
|
||||
@@ -61,6 +66,7 @@ import {
|
||||
} from "../shared.js";
|
||||
import type { RenderJob } from "../../renderOrchestrator.js";
|
||||
import { isActionableProbeFailure } from "./probeFailures.js";
|
||||
import { preflightCompositionAssetMediaTypes } from "../../assetMediaType.js";
|
||||
|
||||
export interface ProbeStageInput {
|
||||
projectDir: string;
|
||||
@@ -75,6 +81,7 @@ export interface ProbeStageInput {
|
||||
forceScreenshot: boolean;
|
||||
log: ProducerLogger;
|
||||
assertNotAborted: () => void;
|
||||
abortSignal?: AbortSignal;
|
||||
/** From compileStage. May be replaced via `recompileWithResolutions`. */
|
||||
compiled: CompiledComposition;
|
||||
/** From compileStage. Mutated in place (videos/audios pushed, duration set). */
|
||||
@@ -148,7 +155,7 @@ export function hasAutoStartVideos(html: string): boolean {
|
||||
}
|
||||
|
||||
/**
|
||||
* Variable-bound audio/video sources are resolved by the browser runtime, not
|
||||
* Variable-bound image/audio/video sources are resolved by the browser runtime, not
|
||||
* the static compiler. Probe them whenever the current render overrides the
|
||||
* referenced variable so media extraction follows the resolved row value.
|
||||
*/
|
||||
@@ -159,7 +166,9 @@ export function hasVariableBoundMedia(
|
||||
if (!variables || Object.keys(variables).length === 0) return false;
|
||||
const { document } = parseHTML(html);
|
||||
return Array.from(
|
||||
document.querySelectorAll("audio[data-var-src], video[data-var-src], source[data-var-src]"),
|
||||
document.querySelectorAll(
|
||||
"img[data-var-src], audio[data-var-src], video[data-var-src], source[data-var-src]",
|
||||
),
|
||||
).some((element) => {
|
||||
const variableId = element.getAttribute("data-var-src")?.trim();
|
||||
return Boolean(variableId && Object.hasOwn(variables, variableId));
|
||||
@@ -224,6 +233,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
forceScreenshot,
|
||||
log,
|
||||
assertNotAborted,
|
||||
abortSignal,
|
||||
composition,
|
||||
width,
|
||||
height,
|
||||
@@ -469,6 +479,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
if (browserMedia.length > 0) {
|
||||
const existingVideoIds = new Set(composition.videos.map((v) => v.id));
|
||||
const existingAudioIds = new Set(composition.audios.map((a) => a.id));
|
||||
const existingImageIds = new Set(composition.images.map((image) => image.id));
|
||||
|
||||
pruneMutedBrowserMedia(composition, browserMedia, existingAudioIds);
|
||||
|
||||
@@ -575,6 +586,33 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
});
|
||||
existingAudioIds.add(el.id);
|
||||
}
|
||||
} else if (el.tagName === "image") {
|
||||
if (existingImageIds.has(el.id)) {
|
||||
const existing = composition.images.find((image) => image.id === el.id);
|
||||
if (existing) {
|
||||
existing.src = src;
|
||||
const runtimeEnd = resolveBrowserMediaEnd(el.start, el.end, el.duration);
|
||||
const projectedEnd = projectBrowserEndToCompositionTimeline(
|
||||
existing.start,
|
||||
el.start,
|
||||
runtimeEnd,
|
||||
);
|
||||
if (
|
||||
projectedEnd > existing.start &&
|
||||
Math.abs(existing.end - projectedEnd) > BROWSER_MEDIA_EPSILON
|
||||
) {
|
||||
existing.end = projectedEnd;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
composition.images.push({
|
||||
id: el.id,
|
||||
src,
|
||||
start: el.start,
|
||||
end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
|
||||
});
|
||||
existingImageIds.add(el.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -628,6 +666,38 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await preflightCompositionAssetMediaTypes({
|
||||
projectDir,
|
||||
compiledDir: join(workDir, "compiled"),
|
||||
composition,
|
||||
signal: abortSignal,
|
||||
});
|
||||
// Keep the final cancellation check inside the ownership guard: an abort
|
||||
// after the last probe resolves must still release the stage-owned browser
|
||||
// session and file server before propagating.
|
||||
assertNotAborted();
|
||||
} catch (error) {
|
||||
// The orchestrator only takes ownership after this stage returns. Until
|
||||
// then, any post-browser validation failure must release both resources
|
||||
// here or a deterministic user error strands Chrome and its file server.
|
||||
if (probeSession) {
|
||||
try {
|
||||
await closeCaptureSession(probeSession);
|
||||
} catch (closeError) {
|
||||
log.warn("Failed to close probe session after media preflight failure", {
|
||||
error: closeError instanceof Error ? closeError.message : String(closeError),
|
||||
});
|
||||
}
|
||||
probeSession = null;
|
||||
}
|
||||
if (fileServer) {
|
||||
closeFileServerSafely(fileServer, "probe media preflight", log);
|
||||
fileServer = null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const browserProbeMs = Date.now() - probeStart;
|
||||
|
||||
const duration = composition.duration;
|
||||
|
||||
@@ -2272,6 +2272,7 @@ async function executeRenderPipeline(input: {
|
||||
forceScreenshot: captureForceScreenshot,
|
||||
log,
|
||||
assertNotAborted,
|
||||
abortSignal: executionSignal,
|
||||
compiled,
|
||||
composition,
|
||||
width,
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { Semaphore } from "./semaphore.js";
|
||||
|
||||
const MEDIA_PROBE_CONCURRENCY = 4;
|
||||
|
||||
/**
|
||||
* Process-wide because compiler advisories can outlive compileForRender and
|
||||
* overlap the later media-type preflight. A per-phase limiter would still let
|
||||
* those independently scheduled probe pools multiply subprocess load.
|
||||
*/
|
||||
export const sharedMediaProbeSemaphore = new Semaphore(MEDIA_PROBE_CONCURRENCY);
|
||||
|
||||
export async function withMediaProbeSlot<T>(operation: () => Promise<T>): Promise<T> {
|
||||
const release = await sharedMediaProbeSemaphore.acquire();
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user