mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
Merge pull request #753 from heygen-com/refactor/producer-stages-1.5.2-thin-sequencer
refactor(producer): finish thinning executeRenderJob
This commit is contained in:
James Russo
@@ -0,0 +1,410 @@
|
||||
/**
|
||||
* Capture-cost calibration and worker-count resolution.
|
||||
*
|
||||
* The "calibration" flow renders a handful of representative frames in
|
||||
* a throwaway `CaptureSession` and uses p95 capture time to scale the
|
||||
* auto-worker budget. The calibration ceiling
|
||||
* (`MAX_MEASURED_CAPTURE_COST_MULTIPLIER`) and target
|
||||
* (`CAPTURE_CALIBRATION_TARGET_MS`) are tunable knobs — they pin the
|
||||
* relationship between observed capture time and worker count.
|
||||
*/
|
||||
|
||||
import { join } from "node:path";
|
||||
import { fpsToNumber } from "@hyperframes/core";
|
||||
import {
|
||||
type BeforeCaptureHook,
|
||||
type CaptureOptions,
|
||||
type CaptureSession,
|
||||
type EngineConfig,
|
||||
calculateOptimalWorkers,
|
||||
captureFrameToBuffer,
|
||||
closeCaptureSession,
|
||||
createCaptureSession,
|
||||
initializeSession,
|
||||
} from "@hyperframes/engine";
|
||||
import type { CompiledComposition } from "../htmlCompiler.js";
|
||||
import type { FileServerHandle } from "../fileServer.js";
|
||||
import { defaultLogger, type ProducerLogger } from "../../logger.js";
|
||||
import type { RenderJob } from "../renderOrchestrator.js";
|
||||
|
||||
export interface CaptureCostEstimate {
|
||||
multiplier: number;
|
||||
reasons: string[];
|
||||
p95Ms?: number;
|
||||
}
|
||||
|
||||
export interface CaptureCalibrationSample {
|
||||
frameIndex: number;
|
||||
captureTimeMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Target p95 capture time used to scale the auto-worker budget. If the
|
||||
* measured p95 exceeds this, the multiplier ratchets up. Empirically
|
||||
* tuned against the producer's regression-harness fixtures.
|
||||
*/
|
||||
export const CAPTURE_CALIBRATION_TARGET_MS = 600;
|
||||
|
||||
/**
|
||||
* Ceiling on the measured cost multiplier. Without this, a pathological
|
||||
* 30-second capture would push the auto-worker budget arbitrarily high.
|
||||
*/
|
||||
export const MAX_MEASURED_CAPTURE_COST_MULTIPLIER = 8;
|
||||
|
||||
/**
|
||||
* CDP protocol timeout used while running calibration. Bounded below
|
||||
* the normal `cfg.protocolTimeout` so a wedged BeginFrame calibration
|
||||
* times out fast and falls back to screenshot mode (see the
|
||||
* `shouldFallbackToScreenshotAfterCalibrationError` path in the
|
||||
* sequencer).
|
||||
*/
|
||||
export const CAPTURE_CALIBRATION_PROTOCOL_TIMEOUT_MS = 30_000;
|
||||
|
||||
export function estimateCaptureCostMultiplier(
|
||||
compiled: Pick<CompiledComposition, "hasShaderTransitions" | "renderModeHints">,
|
||||
): CaptureCostEstimate {
|
||||
let multiplier = 1;
|
||||
const reasons: string[] = [];
|
||||
|
||||
if (compiled.hasShaderTransitions) {
|
||||
multiplier += 2;
|
||||
reasons.push("shader-transitions");
|
||||
}
|
||||
|
||||
const reasonCodes = new Set(compiled.renderModeHints.reasons.map((reason) => reason.code));
|
||||
if (reasonCodes.has("requestAnimationFrame")) {
|
||||
multiplier += 1;
|
||||
reasons.push("requestAnimationFrame");
|
||||
}
|
||||
if (reasonCodes.has("iframe")) {
|
||||
multiplier += 0.5;
|
||||
reasons.push("iframe");
|
||||
}
|
||||
|
||||
return {
|
||||
multiplier: Math.round(multiplier * 100) / 100,
|
||||
reasons,
|
||||
};
|
||||
}
|
||||
|
||||
function combineCaptureCostEstimates(
|
||||
staticCost: CaptureCostEstimate,
|
||||
measuredCost?: CaptureCostEstimate,
|
||||
): CaptureCostEstimate {
|
||||
if (!measuredCost || measuredCost.multiplier <= 1) return staticCost;
|
||||
if (staticCost.multiplier >= measuredCost.multiplier) {
|
||||
return {
|
||||
multiplier: staticCost.multiplier,
|
||||
reasons: [...staticCost.reasons, ...measuredCost.reasons],
|
||||
p95Ms: measuredCost.p95Ms,
|
||||
};
|
||||
}
|
||||
return {
|
||||
multiplier: measuredCost.multiplier,
|
||||
reasons: [...measuredCost.reasons, ...staticCost.reasons],
|
||||
p95Ms: measuredCost.p95Ms,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveRenderWorkerCount(
|
||||
totalFrames: number,
|
||||
requestedWorkers: number | undefined,
|
||||
cfg: EngineConfig,
|
||||
compiled: Pick<CompiledComposition, "hasShaderTransitions" | "renderModeHints">,
|
||||
log: ProducerLogger = defaultLogger,
|
||||
measuredCaptureCost?: CaptureCostEstimate,
|
||||
): number {
|
||||
const captureCost = combineCaptureCostEstimates(
|
||||
estimateCaptureCostMultiplier(compiled),
|
||||
measuredCaptureCost,
|
||||
);
|
||||
const workerCount = calculateOptimalWorkers(totalFrames, requestedWorkers, {
|
||||
...cfg,
|
||||
captureCostMultiplier: captureCost.multiplier,
|
||||
});
|
||||
|
||||
if (requestedWorkers !== undefined || captureCost.multiplier <= 1) {
|
||||
return workerCount;
|
||||
}
|
||||
|
||||
const baselineWorkers = calculateOptimalWorkers(totalFrames, undefined, cfg);
|
||||
if (workerCount < baselineWorkers) {
|
||||
log.warn(
|
||||
"[Render] Reduced auto worker count for high-cost capture workload to avoid Chrome compositor starvation.",
|
||||
{
|
||||
from: baselineWorkers,
|
||||
to: workerCount,
|
||||
costMultiplier: captureCost.multiplier,
|
||||
reasons: captureCost.reasons,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return workerCount;
|
||||
}
|
||||
|
||||
export function createCaptureCalibrationConfig(cfg: EngineConfig): EngineConfig {
|
||||
return {
|
||||
...cfg,
|
||||
protocolTimeout: Math.min(cfg.protocolTimeout, CAPTURE_CALIBRATION_PROTOCOL_TIMEOUT_MS),
|
||||
};
|
||||
}
|
||||
|
||||
export function estimateMeasuredCaptureCostMultiplier(
|
||||
samples: CaptureCalibrationSample[],
|
||||
): CaptureCostEstimate {
|
||||
if (samples.length === 0) {
|
||||
return { multiplier: 1, reasons: [] };
|
||||
}
|
||||
|
||||
const sorted = [...samples].sort((a, b) => a.captureTimeMs - b.captureTimeMs);
|
||||
const p95Index = Math.max(0, Math.ceil(sorted.length * 0.95) - 1);
|
||||
const p95Sample = sorted[p95Index] ?? sorted[sorted.length - 1];
|
||||
if (!p95Sample) {
|
||||
return { multiplier: 1, reasons: [] };
|
||||
}
|
||||
const p95Ms = Math.round(p95Sample.captureTimeMs);
|
||||
const multiplier = Math.min(
|
||||
MAX_MEASURED_CAPTURE_COST_MULTIPLIER,
|
||||
Math.max(1, Math.round((p95Ms / CAPTURE_CALIBRATION_TARGET_MS) * 100) / 100),
|
||||
);
|
||||
|
||||
return {
|
||||
multiplier,
|
||||
reasons: multiplier > 1 ? [`calibration-p95=${p95Ms}ms`] : [],
|
||||
p95Ms,
|
||||
};
|
||||
}
|
||||
|
||||
export function selectCaptureCalibrationFrames(totalFrames: number): number[] {
|
||||
if (totalFrames <= 0) return [];
|
||||
const lastFrame = totalFrames - 1;
|
||||
const candidates = [
|
||||
0,
|
||||
Math.floor(totalFrames * 0.25),
|
||||
Math.floor(totalFrames * 0.5),
|
||||
Math.floor(totalFrames * 0.75),
|
||||
lastFrame,
|
||||
];
|
||||
return Array.from(
|
||||
new Set(candidates.map((frame) => Math.max(0, Math.min(lastFrame, frame)))),
|
||||
).sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
export async function measureCaptureCostFromSession(
|
||||
session: CaptureSession,
|
||||
totalFrames: number,
|
||||
fps: number,
|
||||
): Promise<{ estimate: CaptureCostEstimate; samples: CaptureCalibrationSample[] }> {
|
||||
const sampledFrames = selectCaptureCalibrationFrames(totalFrames);
|
||||
const samples: CaptureCalibrationSample[] = [];
|
||||
|
||||
for (const frameIndex of sampledFrames) {
|
||||
const time = frameIndex / fps;
|
||||
const startedAt = Date.now();
|
||||
const result = await captureFrameToBuffer(session, frameIndex, time);
|
||||
samples.push({
|
||||
frameIndex,
|
||||
captureTimeMs: result.captureTimeMs || Date.now() - startedAt,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
estimate: estimateMeasuredCaptureCostMultiplier(samples),
|
||||
samples,
|
||||
};
|
||||
}
|
||||
|
||||
export function logCaptureCalibrationResult(
|
||||
calibration: { estimate: CaptureCostEstimate; samples: CaptureCalibrationSample[] },
|
||||
log: ProducerLogger,
|
||||
): void {
|
||||
if (calibration.estimate.multiplier > 1) {
|
||||
log.warn("[Render] Measured slow frame capture during auto-worker calibration.", {
|
||||
multiplier: calibration.estimate.multiplier,
|
||||
p95Ms: calibration.estimate.p95Ms,
|
||||
sampledFrames: calibration.samples.map((sample) => sample.frameIndex),
|
||||
});
|
||||
} else {
|
||||
log.debug("[Render] Auto-worker calibration kept baseline capture cost.", {
|
||||
p95Ms: calibration.estimate.p95Ms,
|
||||
sampledFrames: calibration.samples.map((sample) => sample.frameIndex),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export type CaptureCalibrationFailureReason =
|
||||
| "calibration-failed"
|
||||
| "calibration-screenshot-failed";
|
||||
|
||||
export function createFailedCaptureCalibrationEstimate(reason: CaptureCalibrationFailureReason): {
|
||||
estimate: CaptureCostEstimate;
|
||||
samples: CaptureCalibrationSample[];
|
||||
} {
|
||||
return {
|
||||
estimate: {
|
||||
multiplier: MAX_MEASURED_CAPTURE_COST_MULTIPLIER,
|
||||
reasons: [reason],
|
||||
},
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
|
||||
export interface CaptureCalibrationOutcome {
|
||||
calibration: { estimate: CaptureCostEstimate; samples: CaptureCalibrationSample[] } | undefined;
|
||||
/** Flipped to `true` if BeginFrame calibration timed out and the screenshot retry fired. */
|
||||
forceScreenshot: boolean;
|
||||
/** Closed and nulled when the screenshot fallback fires; passthrough otherwise. */
|
||||
probeSession: CaptureSession | null;
|
||||
/** Buffer of whichever session was active last; the sequencer uses it for the error-path tail. */
|
||||
lastBrowserConsole: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the auto-worker capture-cost calibration, including the
|
||||
* BeginFrame → screenshot fallback on timeout. Owns the calibration
|
||||
* session lifecycle and may close the caller-owned `probeSession` when
|
||||
* the fallback fires (BeginFrame is no longer the active capture mode,
|
||||
* so the probe session is no longer reusable).
|
||||
*/
|
||||
export async function runCaptureCalibration(input: {
|
||||
cfg: EngineConfig;
|
||||
fileServer: FileServerHandle;
|
||||
workDir: string;
|
||||
log: ProducerLogger;
|
||||
job: RenderJob;
|
||||
totalFrames: number;
|
||||
forceScreenshot: boolean;
|
||||
probeSession: CaptureSession | null;
|
||||
buildCaptureOptions: () => CaptureOptions;
|
||||
createRenderVideoFrameInjector: () => BeforeCaptureHook | null;
|
||||
/** Throws `RenderCancelledError` when the caller's abort signal fires. */
|
||||
assertNotAborted: () => void;
|
||||
}): Promise<CaptureCalibrationOutcome> {
|
||||
const {
|
||||
cfg,
|
||||
fileServer,
|
||||
workDir,
|
||||
log,
|
||||
job,
|
||||
totalFrames,
|
||||
buildCaptureOptions,
|
||||
createRenderVideoFrameInjector,
|
||||
assertNotAborted,
|
||||
} = input;
|
||||
let probeSession = input.probeSession;
|
||||
let forceScreenshot = input.forceScreenshot;
|
||||
let lastBrowserConsole: string[] = [];
|
||||
|
||||
const fps = fpsToNumber(job.config.fps);
|
||||
// Holds whichever calibration session is currently open. The closure
|
||||
// writes into the outer `sessionRef` (an object) rather than a `let`
|
||||
// so the `finally` and the fallback branch read the latest value
|
||||
// without TS narrowing it back to the initial `null`.
|
||||
const sessionRef: { current: CaptureSession | null } = { current: null };
|
||||
|
||||
const runOneCalibration = async (
|
||||
sessionDir: string,
|
||||
sessionCfg: EngineConfig,
|
||||
): Promise<{ estimate: CaptureCostEstimate; samples: CaptureCalibrationSample[] }> => {
|
||||
const session = await createCaptureSession(
|
||||
fileServer.url,
|
||||
sessionDir,
|
||||
buildCaptureOptions(),
|
||||
createRenderVideoFrameInjector(),
|
||||
sessionCfg,
|
||||
);
|
||||
sessionRef.current = session;
|
||||
if (!session.isInitialized) {
|
||||
await initializeSession(session);
|
||||
}
|
||||
assertNotAborted();
|
||||
const result = await measureCaptureCostFromSession(session, totalFrames, fps);
|
||||
logCaptureCalibrationResult(result, log);
|
||||
return result;
|
||||
};
|
||||
|
||||
const calibrationCfg = createCaptureCalibrationConfig({ ...cfg, forceScreenshot });
|
||||
let calibration:
|
||||
| { estimate: CaptureCostEstimate; samples: CaptureCalibrationSample[] }
|
||||
| undefined;
|
||||
|
||||
try {
|
||||
calibration = await runOneCalibration(join(workDir, "capture-calibration"), calibrationCfg);
|
||||
} catch (error) {
|
||||
const shouldFallback =
|
||||
!forceScreenshot && shouldFallbackToScreenshotAfterCalibrationError(error);
|
||||
if (!shouldFallback) {
|
||||
calibration = createFailedCaptureCalibrationEstimate("calibration-failed");
|
||||
log.warn("[Render] Auto-worker calibration failed; using conservative worker budget.", {
|
||||
protocolTimeout: calibrationCfg.protocolTimeout,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
} else {
|
||||
// BeginFrame failed on this host's Chrome build; switch the rest
|
||||
// of the pipeline to screenshot capture. Flip only the local
|
||||
// boolean — `cfg` stays the compile-time view; downstream stages
|
||||
// receive the new value via the explicit `forceScreenshot` param.
|
||||
forceScreenshot = true;
|
||||
if (probeSession) {
|
||||
// Snapshot the probe buffer before closing — if the screenshot
|
||||
// session create that follows also fails, this is the only place
|
||||
// the BeginFrame-era diagnostic survives for the caller's
|
||||
// error-path browser-console tail.
|
||||
lastBrowserConsole = probeSession.browserConsoleBuffer;
|
||||
await closeCaptureSession(probeSession).catch(() => {});
|
||||
probeSession = null;
|
||||
}
|
||||
if (sessionRef.current) {
|
||||
lastBrowserConsole = sessionRef.current.browserConsoleBuffer;
|
||||
await closeCaptureSession(sessionRef.current).catch(() => {});
|
||||
sessionRef.current = null;
|
||||
}
|
||||
|
||||
log.warn(
|
||||
"[Render] BeginFrame auto-worker calibration timed out; retrying calibration in screenshot capture mode.",
|
||||
{
|
||||
protocolTimeout: calibrationCfg.protocolTimeout,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
);
|
||||
|
||||
const screenshotCfg = createCaptureCalibrationConfig({ ...cfg, forceScreenshot: true });
|
||||
try {
|
||||
calibration = await runOneCalibration(
|
||||
join(workDir, "capture-calibration-screenshot"),
|
||||
screenshotCfg,
|
||||
);
|
||||
} catch (fallbackError) {
|
||||
calibration = createFailedCaptureCalibrationEstimate("calibration-screenshot-failed");
|
||||
log.warn(
|
||||
"[Render] Screenshot auto-worker calibration failed after BeginFrame fallback; using conservative worker budget.",
|
||||
{
|
||||
protocolTimeout: screenshotCfg.protocolTimeout,
|
||||
error: fallbackError instanceof Error ? fallbackError.message : String(fallbackError),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (sessionRef.current) {
|
||||
lastBrowserConsole = sessionRef.current.browserConsoleBuffer;
|
||||
await closeCaptureSession(sessionRef.current).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
return { calibration, forceScreenshot, probeSession, lastBrowserConsole };
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as `runCaptureCalibration`'s error-classification check, but
|
||||
* exported separately because the sequencer also calls it from the
|
||||
* disk-capture retry loop. Returns `true` for the BeginFrame-specific
|
||||
* protocol errors that recover cleanly under screenshot mode.
|
||||
*/
|
||||
export function shouldFallbackToScreenshotAfterCalibrationError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return /HeadlessExperimental\.beginFrame timed out|beginFrame probe timeout|Another frame is pending|Frame still pending|Protocol error.*HeadlessExperimental\.beginFrame|Runtime\.callFunctionOn timed out|Runtime\.evaluate timed out/i.test(
|
||||
message,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* Tests for the cancel/error-path helpers in `./cleanup.ts`.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { CaptureSession } from "@hyperframes/engine";
|
||||
import type { FileServerHandle } from "../fileServer.js";
|
||||
import { buildRenderErrorDetails, cleanupRenderResources, safeCleanup } from "./cleanup.js";
|
||||
|
||||
function makeLog() {
|
||||
return { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() };
|
||||
}
|
||||
|
||||
describe("safeCleanup", () => {
|
||||
it("returns normally when the operation succeeds", async () => {
|
||||
const log = makeLog();
|
||||
const op = vi.fn().mockResolvedValue(undefined);
|
||||
await safeCleanup("close x", op, log);
|
||||
expect(op).toHaveBeenCalledOnce();
|
||||
expect(log.debug).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("swallows thrown errors and logs them at debug", async () => {
|
||||
const log = makeLog();
|
||||
await safeCleanup(
|
||||
"close x",
|
||||
() => {
|
||||
throw new Error("boom");
|
||||
},
|
||||
log,
|
||||
);
|
||||
expect(log.debug).toHaveBeenCalledWith("Cleanup failed (close x)", { error: "boom" });
|
||||
});
|
||||
|
||||
it("swallows async rejections", async () => {
|
||||
const log = makeLog();
|
||||
await safeCleanup("close x", async () => Promise.reject(new Error("async boom")), log);
|
||||
expect(log.debug).toHaveBeenCalledWith("Cleanup failed (close x)", { error: "async boom" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("cleanupRenderResources", () => {
|
||||
it("closes fileServer, probeSession, then removes workDir (non-debug)", async () => {
|
||||
const log = makeLog();
|
||||
const workDir = mkdtempSync(join(tmpdir(), "cleanup-test-"));
|
||||
writeFileSync(join(workDir, "marker.txt"), "x");
|
||||
|
||||
const order: string[] = [];
|
||||
const fileServer = {
|
||||
close: () => {
|
||||
order.push("fileServer.close");
|
||||
},
|
||||
} as unknown as FileServerHandle;
|
||||
const probeSession = {
|
||||
_markClosed: () => {
|
||||
order.push("probeSession.close");
|
||||
},
|
||||
} as unknown as CaptureSession;
|
||||
|
||||
// closeCaptureSession is the engine helper; the helper itself isn't
|
||||
// mockable per-call without intercepting the module import. Instead
|
||||
// we verify the higher-level invariants: fileServer.close was called,
|
||||
// and the workDir was rmSync'd.
|
||||
await cleanupRenderResources({
|
||||
fileServer,
|
||||
probeSession: null, // skip probe to keep this test focused on the workDir invariant
|
||||
workDir,
|
||||
debug: false,
|
||||
log,
|
||||
label: "cancel",
|
||||
});
|
||||
|
||||
expect(order).toEqual(["fileServer.close"]);
|
||||
expect(existsSync(workDir)).toBe(false);
|
||||
void probeSession; // suppress unused-var (kept as a doc of the surface)
|
||||
});
|
||||
|
||||
it("keeps workDir when debug=true", async () => {
|
||||
const log = makeLog();
|
||||
const workDir = mkdtempSync(join(tmpdir(), "cleanup-debug-"));
|
||||
writeFileSync(join(workDir, "marker.txt"), "x");
|
||||
|
||||
await cleanupRenderResources({
|
||||
fileServer: null,
|
||||
probeSession: null,
|
||||
workDir,
|
||||
debug: true,
|
||||
log,
|
||||
label: "error",
|
||||
});
|
||||
|
||||
expect(existsSync(workDir)).toBe(true);
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("is a no-op for missing workDir thanks to rmSync force:true", async () => {
|
||||
const log = makeLog();
|
||||
const workDir = join(tmpdir(), `cleanup-missing-${Date.now()}`);
|
||||
expect(existsSync(workDir)).toBe(false);
|
||||
|
||||
await cleanupRenderResources({
|
||||
fileServer: null,
|
||||
probeSession: null,
|
||||
workDir,
|
||||
debug: false,
|
||||
log,
|
||||
label: "error",
|
||||
});
|
||||
|
||||
// No throw; nothing logged at debug for the rmSync step.
|
||||
expect(log.debug).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs (and continues past) a fileServer.close that throws", async () => {
|
||||
const log = makeLog();
|
||||
const workDir = mkdtempSync(join(tmpdir(), "cleanup-throw-"));
|
||||
|
||||
const fileServer = {
|
||||
close: () => {
|
||||
throw new Error("server stuck");
|
||||
},
|
||||
} as unknown as FileServerHandle;
|
||||
|
||||
await cleanupRenderResources({
|
||||
fileServer,
|
||||
probeSession: null,
|
||||
workDir,
|
||||
debug: false,
|
||||
log,
|
||||
label: "error",
|
||||
});
|
||||
|
||||
expect(log.debug).toHaveBeenCalledWith("Cleanup failed (close file server (error))", {
|
||||
error: "server stuck",
|
||||
});
|
||||
expect(existsSync(workDir)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildRenderErrorDetails", () => {
|
||||
const baseDiagnostics = { videoExtractionFailures: 0, imageDecodeFailures: 0 };
|
||||
|
||||
it("extracts message + stack from Error instances", () => {
|
||||
const err = new Error("nope");
|
||||
const result = buildRenderErrorDetails({
|
||||
error: err,
|
||||
pipelineStartMs: Date.now() - 5000,
|
||||
lastBrowserConsole: [],
|
||||
perfStages: {},
|
||||
hdrDiagnostics: baseDiagnostics,
|
||||
});
|
||||
expect(result.message).toBe("nope");
|
||||
expect(result.stack).toBeDefined();
|
||||
expect(result.elapsedMs).toBeGreaterThanOrEqual(5000);
|
||||
expect(typeof result.freeMemoryMB).toBe("number");
|
||||
});
|
||||
|
||||
it("stringifies non-Error rejections", () => {
|
||||
const result = buildRenderErrorDetails({
|
||||
error: "raw string failure",
|
||||
pipelineStartMs: Date.now(),
|
||||
lastBrowserConsole: [],
|
||||
perfStages: {},
|
||||
hdrDiagnostics: baseDiagnostics,
|
||||
});
|
||||
expect(result.message).toBe("raw string failure");
|
||||
expect(result.stack).toBeUndefined();
|
||||
});
|
||||
|
||||
it("includes browserConsoleTail only when buffer is non-empty (last 30 lines)", () => {
|
||||
const lines = Array.from({ length: 50 }, (_, i) => `line ${i}`);
|
||||
const result = buildRenderErrorDetails({
|
||||
error: new Error("x"),
|
||||
pipelineStartMs: Date.now(),
|
||||
lastBrowserConsole: lines,
|
||||
perfStages: {},
|
||||
hdrDiagnostics: baseDiagnostics,
|
||||
});
|
||||
expect(result.browserConsoleTail).toHaveLength(30);
|
||||
expect(result.browserConsoleTail?.[0]).toBe("line 20");
|
||||
expect(result.browserConsoleTail?.[29]).toBe("line 49");
|
||||
});
|
||||
|
||||
it("omits browserConsoleTail when buffer is empty", () => {
|
||||
const result = buildRenderErrorDetails({
|
||||
error: new Error("x"),
|
||||
pipelineStartMs: Date.now(),
|
||||
lastBrowserConsole: [],
|
||||
perfStages: {},
|
||||
hdrDiagnostics: baseDiagnostics,
|
||||
});
|
||||
expect(result.browserConsoleTail).toBeUndefined();
|
||||
});
|
||||
|
||||
it("includes perfStages snapshot only when non-empty", () => {
|
||||
const empty = buildRenderErrorDetails({
|
||||
error: new Error("x"),
|
||||
pipelineStartMs: Date.now(),
|
||||
lastBrowserConsole: [],
|
||||
perfStages: {},
|
||||
hdrDiagnostics: baseDiagnostics,
|
||||
});
|
||||
expect(empty.perfStages).toBeUndefined();
|
||||
|
||||
const populated = buildRenderErrorDetails({
|
||||
error: new Error("x"),
|
||||
pipelineStartMs: Date.now(),
|
||||
lastBrowserConsole: [],
|
||||
perfStages: { compileMs: 12, captureMs: 340 },
|
||||
hdrDiagnostics: baseDiagnostics,
|
||||
});
|
||||
expect(populated.perfStages).toEqual({ compileMs: 12, captureMs: 340 });
|
||||
});
|
||||
|
||||
it("includes hdrDiagnostics only when at least one failure counter > 0", () => {
|
||||
const clean = buildRenderErrorDetails({
|
||||
error: new Error("x"),
|
||||
pipelineStartMs: Date.now(),
|
||||
lastBrowserConsole: [],
|
||||
perfStages: {},
|
||||
hdrDiagnostics: baseDiagnostics,
|
||||
});
|
||||
expect(clean.hdrDiagnostics).toBeUndefined();
|
||||
|
||||
const failed = buildRenderErrorDetails({
|
||||
error: new Error("x"),
|
||||
pipelineStartMs: Date.now(),
|
||||
lastBrowserConsole: [],
|
||||
perfStages: {},
|
||||
hdrDiagnostics: { videoExtractionFailures: 2, imageDecodeFailures: 0 },
|
||||
});
|
||||
expect(failed.hdrDiagnostics).toEqual({ videoExtractionFailures: 2, imageDecodeFailures: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
// Quiet unused-import warning — these are referenced via type-only paths.
|
||||
void mkdirSync;
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Sequencer cleanup + error-details helpers shared by the cancel and
|
||||
* error paths in `executeRenderJob`.
|
||||
*/
|
||||
|
||||
import { rmSync } from "node:fs";
|
||||
import { freemem } from "node:os";
|
||||
import { type CaptureSession, closeCaptureSession } from "@hyperframes/engine";
|
||||
import type { FileServerHandle } from "../fileServer.js";
|
||||
import { defaultLogger, type ProducerLogger } from "../../logger.js";
|
||||
import type { HdrDiagnostics, RenderJob } from "../renderOrchestrator.js";
|
||||
|
||||
/**
|
||||
* Wrap a cleanup operation so it never throws, but logs any failure.
|
||||
* The sequencer needs to keep tearing down resources even when one of
|
||||
* them is stuck (e.g. a `fileServer.close()` hitting a TCP race); a
|
||||
* thrown cleanup error would mask the original render failure.
|
||||
*/
|
||||
export async function safeCleanup(
|
||||
label: string,
|
||||
fn: () => Promise<void> | void,
|
||||
log: ProducerLogger = defaultLogger,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await fn();
|
||||
} catch (err) {
|
||||
log.debug(`Cleanup failed (${label})`, {
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the file server, close the probe session, and remove the
|
||||
* working directory. Each step runs through `safeCleanup` so a stuck
|
||||
* resource doesn't mask the original render error.
|
||||
*/
|
||||
export async function cleanupRenderResources(input: {
|
||||
fileServer: FileServerHandle | null;
|
||||
probeSession: CaptureSession | null;
|
||||
workDir: string;
|
||||
debug: boolean;
|
||||
log: ProducerLogger;
|
||||
/** Suffix appended to safeCleanup labels. Pinned to the existing diagnostic payloads. */
|
||||
label: "cancel" | "error";
|
||||
}): Promise<void> {
|
||||
const { fileServer, probeSession, workDir, debug, log, label } = input;
|
||||
if (fileServer) {
|
||||
const fs = fileServer;
|
||||
await safeCleanup(
|
||||
`close file server (${label})`,
|
||||
() => {
|
||||
fs.close();
|
||||
},
|
||||
log,
|
||||
);
|
||||
}
|
||||
if (probeSession) {
|
||||
const session = probeSession;
|
||||
await safeCleanup(`close probe session (${label})`, () => closeCaptureSession(session), log);
|
||||
}
|
||||
if (!debug) {
|
||||
// `force: true` swallows ENOENT, so no need to existsSync first.
|
||||
await safeCleanup(
|
||||
`remove workDir (${label})`,
|
||||
() => rmSync(workDir, { recursive: true, force: true }),
|
||||
log,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the `RenderJob.errorDetails` shape downstream consumers (SSE,
|
||||
* sync `/render` response, queue introspection) read on failure.
|
||||
*/
|
||||
export function buildRenderErrorDetails(input: {
|
||||
error: unknown;
|
||||
pipelineStartMs: number;
|
||||
lastBrowserConsole: string[];
|
||||
perfStages: Record<string, number>;
|
||||
hdrDiagnostics: HdrDiagnostics;
|
||||
}): NonNullable<RenderJob["errorDetails"]> {
|
||||
const errorMessage = input.error instanceof Error ? input.error.message : String(input.error);
|
||||
const errorStack = input.error instanceof Error ? input.error.stack : undefined;
|
||||
return {
|
||||
message: errorMessage,
|
||||
stack: errorStack,
|
||||
elapsedMs: Date.now() - input.pipelineStartMs,
|
||||
freeMemoryMB: Math.round(freemem() / (1024 * 1024)),
|
||||
browserConsoleTail:
|
||||
input.lastBrowserConsole.length > 0 ? input.lastBrowserConsole.slice(-30) : undefined,
|
||||
perfStages: Object.keys(input.perfStages).length > 0 ? { ...input.perfStages } : undefined,
|
||||
hdrDiagnostics:
|
||||
input.hdrDiagnostics.videoExtractionFailures > 0 ||
|
||||
input.hdrDiagnostics.imageDecodeFailures > 0
|
||||
? { ...input.hdrDiagnostics }
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* Tests for `resolveEffectiveHdrMode` — pins the four-signal fold
|
||||
* (caller hdrMode × probed video color × probed image color × output
|
||||
* format) so the format-gate ordering can't silently regress under a
|
||||
* future cleanup.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { ExtractionResult, VideoColorSpace } from "@hyperframes/engine";
|
||||
import { resolveEffectiveHdrMode } from "./hdrMode.js";
|
||||
|
||||
function makeLog() {
|
||||
return { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() };
|
||||
}
|
||||
|
||||
function extractionWith(colorSpaces: (VideoColorSpace | null)[]): ExtractionResult | undefined {
|
||||
if (colorSpaces.length === 0) return undefined;
|
||||
return {
|
||||
extracted: colorSpaces.map((colorSpace) => ({
|
||||
videoId: "v",
|
||||
outputDir: "/tmp/v",
|
||||
framePaths: new Map<number, string>(),
|
||||
metadata: {
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
durationSeconds: 1,
|
||||
colorSpace,
|
||||
},
|
||||
})),
|
||||
} as unknown as ExtractionResult;
|
||||
}
|
||||
|
||||
const HDR_PQ: VideoColorSpace = {
|
||||
colorTransfer: "smpte2084",
|
||||
colorPrimaries: "bt2020",
|
||||
colorSpace: "bt2020nc",
|
||||
};
|
||||
|
||||
describe("resolveEffectiveHdrMode", () => {
|
||||
it("returns undefined when force-sdr is set, regardless of HDR sources", () => {
|
||||
const log = makeLog();
|
||||
const result = resolveEffectiveHdrMode({
|
||||
hdrMode: "force-sdr",
|
||||
outputFormat: "mp4",
|
||||
extractionResult: extractionWith([HDR_PQ]),
|
||||
imageColorSpaces: [],
|
||||
log,
|
||||
});
|
||||
expect(result).toBeUndefined();
|
||||
expect(log.info).toHaveBeenCalledWith("[Render] SDR forced by --sdr flag");
|
||||
});
|
||||
|
||||
it("auto-detects HDR from video sources when format=mp4", () => {
|
||||
const log = makeLog();
|
||||
const result = resolveEffectiveHdrMode({
|
||||
hdrMode: "auto",
|
||||
outputFormat: "mp4",
|
||||
extractionResult: extractionWith([HDR_PQ]),
|
||||
imageColorSpaces: [],
|
||||
log,
|
||||
});
|
||||
expect(result).toEqual({ transfer: "pq" });
|
||||
expect(log.info).toHaveBeenCalledWith(expect.stringContaining("auto-detected from source(s)"));
|
||||
});
|
||||
|
||||
it("auto-detects SDR with no HDR sources", () => {
|
||||
const log = makeLog();
|
||||
const result = resolveEffectiveHdrMode({
|
||||
hdrMode: "auto",
|
||||
outputFormat: "mp4",
|
||||
extractionResult: extractionWith([]),
|
||||
imageColorSpaces: [],
|
||||
log,
|
||||
});
|
||||
expect(result).toBeUndefined();
|
||||
expect(log.info).toHaveBeenCalledWith("[Render] No HDR sources detected — rendering SDR");
|
||||
});
|
||||
|
||||
it("force-hdr without sources falls back to HLG and warns", () => {
|
||||
const log = makeLog();
|
||||
const result = resolveEffectiveHdrMode({
|
||||
hdrMode: "force-hdr",
|
||||
outputFormat: "mp4",
|
||||
extractionResult: extractionWith([]),
|
||||
imageColorSpaces: [],
|
||||
log,
|
||||
});
|
||||
expect(result).toEqual({ transfer: "hlg" });
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining("HDR forced by --hdr flag, but no HDR sources were detected"),
|
||||
);
|
||||
});
|
||||
|
||||
it("force-hdr uses the dominant probed transfer when sources are HDR", () => {
|
||||
const log = makeLog();
|
||||
const result = resolveEffectiveHdrMode({
|
||||
hdrMode: "force-hdr",
|
||||
outputFormat: "mp4",
|
||||
extractionResult: extractionWith([HDR_PQ]),
|
||||
imageColorSpaces: [],
|
||||
log,
|
||||
});
|
||||
expect(result).toEqual({ transfer: "pq" });
|
||||
expect(log.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("downgrades to SDR with a warning when output format can't carry HDR", () => {
|
||||
for (const fmt of ["webm", "mov", "png-sequence"] as const) {
|
||||
const log = makeLog();
|
||||
const result = resolveEffectiveHdrMode({
|
||||
hdrMode: "auto",
|
||||
outputFormat: fmt,
|
||||
extractionResult: extractionWith([HDR_PQ]),
|
||||
imageColorSpaces: [],
|
||||
log,
|
||||
});
|
||||
expect(result).toBeUndefined();
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining(`format is "${fmt}" — falling back to SDR`),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("force-hdr without sources + non-mp4 format: still downgrades, two warns fire", () => {
|
||||
const log = makeLog();
|
||||
const result = resolveEffectiveHdrMode({
|
||||
hdrMode: "force-hdr",
|
||||
outputFormat: "webm",
|
||||
extractionResult: extractionWith([]),
|
||||
imageColorSpaces: [],
|
||||
log,
|
||||
});
|
||||
// Effective is undefined: format gate wins.
|
||||
expect(result).toBeUndefined();
|
||||
// Both warns fired in order: format-downgrade, then the
|
||||
// forced-without-sources note (preserved verbatim from the in-process
|
||||
// renderer's diagnostic ordering).
|
||||
expect(log.warn).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.stringContaining("HDR was forced without detected HDR sources"),
|
||||
);
|
||||
expect(log.warn).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.stringContaining("HDR forced by --hdr flag, but no HDR sources were detected"),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* HDR / SDR mode resolution at the sequencer boundary.
|
||||
*
|
||||
* Folds three signals — `RenderConfig.hdrMode`, the probed video color
|
||||
* spaces, and the probed image color spaces — into a single
|
||||
* `effectiveHdr` decision, then emits the matching diagnostic log lines.
|
||||
* The format gate (HDR + alpha is unsupported, so non-mp4 output forces
|
||||
* SDR) lives here too so the sequencer doesn't need to know which
|
||||
* formats can carry an HDR signal.
|
||||
*/
|
||||
|
||||
import { analyzeCompositionHdr } from "@hyperframes/engine";
|
||||
import type { ExtractionResult, HdrTransfer, VideoColorSpace } from "@hyperframes/engine";
|
||||
import type { ProducerLogger } from "../../logger.js";
|
||||
import type { RenderConfig } from "../renderOrchestrator.js";
|
||||
|
||||
export function resolveEffectiveHdrMode(input: {
|
||||
hdrMode: RenderConfig["hdrMode"];
|
||||
outputFormat: NonNullable<RenderConfig["format"]>;
|
||||
extractionResult: ExtractionResult | null | undefined;
|
||||
imageColorSpaces: (VideoColorSpace | null)[];
|
||||
log: ProducerLogger;
|
||||
}): { transfer: HdrTransfer } | undefined {
|
||||
const hdrMode = input.hdrMode ?? "auto";
|
||||
const videoColorSpaces = (input.extractionResult?.extracted ?? []).map(
|
||||
(ext) => ext.metadata.colorSpace,
|
||||
);
|
||||
const allColorSpaces = [...videoColorSpaces, ...input.imageColorSpaces];
|
||||
const info = allColorSpaces.length > 0 ? analyzeCompositionHdr(allColorSpaces) : null;
|
||||
|
||||
let effectiveHdr: { transfer: HdrTransfer } | undefined;
|
||||
let forcedHdrWithoutSources = false;
|
||||
|
||||
if (hdrMode === "force-sdr") {
|
||||
effectiveHdr = undefined;
|
||||
} else if (hdrMode === "force-hdr") {
|
||||
if (info?.hasHdr && info.dominantTransfer) {
|
||||
effectiveHdr = { transfer: info.dominantTransfer };
|
||||
} else {
|
||||
effectiveHdr = { transfer: "hlg" };
|
||||
forcedHdrWithoutSources = true;
|
||||
}
|
||||
} else if (info?.hasHdr && info.dominantTransfer) {
|
||||
effectiveHdr = { transfer: info.dominantTransfer };
|
||||
}
|
||||
|
||||
if (effectiveHdr && input.outputFormat !== "mp4") {
|
||||
const hdrSourceReason = forcedHdrWithoutSources
|
||||
? "HDR was forced without detected HDR sources"
|
||||
: "HDR source detected";
|
||||
input.log.warn(
|
||||
`[Render] ${hdrSourceReason}, but format is "${input.outputFormat}" — falling back to SDR. ` +
|
||||
`HDR + alpha is not supported. Use --format mp4 for HDR10 output.`,
|
||||
);
|
||||
effectiveHdr = undefined;
|
||||
}
|
||||
|
||||
if (forcedHdrWithoutSources) {
|
||||
input.log.warn(
|
||||
"[Render] HDR forced by --hdr flag, but no HDR sources were detected — defaulting to HLG. SDR-only compositions may look perceptually wrong on HDR displays.",
|
||||
);
|
||||
}
|
||||
if (effectiveHdr) {
|
||||
let reason: string;
|
||||
if (hdrMode === "force-hdr") {
|
||||
reason = forcedHdrWithoutSources
|
||||
? "forced by --hdr flag (no HDR sources detected — defaulting to HLG)"
|
||||
: "forced by --hdr flag";
|
||||
} else {
|
||||
reason = "auto-detected from source(s)";
|
||||
}
|
||||
input.log.info(
|
||||
`[Render] HDR ${reason} — output: ${effectiveHdr.transfer.toUpperCase()} (BT.2020, 10-bit H.265)`,
|
||||
);
|
||||
} else if (hdrMode === "force-sdr") {
|
||||
input.log.info("[Render] SDR forced by --sdr flag");
|
||||
} else {
|
||||
input.log.info("[Render] No HDR sources detected — rendering SDR");
|
||||
}
|
||||
|
||||
return effectiveHdr;
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
/**
|
||||
* HDR-pipeline perf instrumentation.
|
||||
*
|
||||
* `HdrPerfCollector` accumulates per-phase wall-clock ms for the
|
||||
* layered HDR / shader-transition composite path; `finalizeHdrPerf`
|
||||
* converts the running totals into the `HdrPerfSummary` shape that
|
||||
* lands in `RenderPerfSummary.hdrPerf`.
|
||||
*/
|
||||
|
||||
export type HdrPerfTimingKey =
|
||||
| "frameSeekMs"
|
||||
| "frameInjectMs"
|
||||
| "stackingQueryMs"
|
||||
| "canvasClearMs"
|
||||
| "normalCompositeMs"
|
||||
| "transitionCompositeMs"
|
||||
| "encoderWriteMs"
|
||||
| "hdrVideoReadDecodeMs"
|
||||
| "hdrVideoTransferMs"
|
||||
| "hdrVideoBlitMs"
|
||||
| "hdrImageTransferMs"
|
||||
| "hdrImageBlitMs"
|
||||
| "domLayerSeekMs"
|
||||
| "domLayerInjectMs"
|
||||
| "domMaskApplyMs"
|
||||
| "domScreenshotMs"
|
||||
| "domMaskRemoveMs"
|
||||
| "domPngDecodeMs"
|
||||
| "domBlitMs";
|
||||
|
||||
export interface HdrPerfCollector {
|
||||
frames: number;
|
||||
normalFrames: number;
|
||||
transitionFrames: number;
|
||||
domLayerCaptures: number;
|
||||
hdrVideoLayerBlits: number;
|
||||
hdrImageLayerBlits: number;
|
||||
timings: Record<HdrPerfTimingKey, number>;
|
||||
}
|
||||
|
||||
export interface HdrPerfSummary {
|
||||
frames: number;
|
||||
normalFrames: number;
|
||||
transitionFrames: number;
|
||||
domLayerCaptures: number;
|
||||
hdrVideoLayerBlits: number;
|
||||
hdrImageLayerBlits: number;
|
||||
timings: Record<string, number>;
|
||||
avgMs: Record<string, number>;
|
||||
}
|
||||
|
||||
export function createHdrPerfCollector(): HdrPerfCollector {
|
||||
return {
|
||||
frames: 0,
|
||||
normalFrames: 0,
|
||||
transitionFrames: 0,
|
||||
domLayerCaptures: 0,
|
||||
hdrVideoLayerBlits: 0,
|
||||
hdrImageLayerBlits: 0,
|
||||
timings: {
|
||||
frameSeekMs: 0,
|
||||
frameInjectMs: 0,
|
||||
stackingQueryMs: 0,
|
||||
canvasClearMs: 0,
|
||||
normalCompositeMs: 0,
|
||||
transitionCompositeMs: 0,
|
||||
encoderWriteMs: 0,
|
||||
hdrVideoReadDecodeMs: 0,
|
||||
hdrVideoTransferMs: 0,
|
||||
hdrVideoBlitMs: 0,
|
||||
hdrImageTransferMs: 0,
|
||||
hdrImageBlitMs: 0,
|
||||
domLayerSeekMs: 0,
|
||||
domLayerInjectMs: 0,
|
||||
domMaskApplyMs: 0,
|
||||
domScreenshotMs: 0,
|
||||
domMaskRemoveMs: 0,
|
||||
domPngDecodeMs: 0,
|
||||
domBlitMs: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function addHdrTiming(
|
||||
perf: HdrPerfCollector | undefined,
|
||||
key: HdrPerfTimingKey,
|
||||
startMs: number,
|
||||
) {
|
||||
if (!perf) return;
|
||||
perf.timings[key] += Date.now() - startMs;
|
||||
}
|
||||
|
||||
function averageTiming(totalMs: number, count: number): number {
|
||||
return count > 0 ? Math.round((totalMs / count) * 100) / 100 : 0;
|
||||
}
|
||||
|
||||
export function finalizeHdrPerf(perf: HdrPerfCollector): HdrPerfSummary {
|
||||
const avgMs: Record<string, number> = {};
|
||||
const perFrameKeys: HdrPerfTimingKey[] = [
|
||||
"frameSeekMs",
|
||||
"frameInjectMs",
|
||||
"stackingQueryMs",
|
||||
"canvasClearMs",
|
||||
"encoderWriteMs",
|
||||
];
|
||||
for (const key of perFrameKeys) avgMs[key] = averageTiming(perf.timings[key], perf.frames);
|
||||
avgMs.normalCompositeMs = averageTiming(perf.timings.normalCompositeMs, perf.normalFrames);
|
||||
avgMs.transitionCompositeMs = averageTiming(
|
||||
perf.timings.transitionCompositeMs,
|
||||
perf.transitionFrames,
|
||||
);
|
||||
|
||||
const perDomLayerKeys: HdrPerfTimingKey[] = [
|
||||
"domLayerSeekMs",
|
||||
"domLayerInjectMs",
|
||||
"domMaskApplyMs",
|
||||
"domScreenshotMs",
|
||||
"domMaskRemoveMs",
|
||||
"domPngDecodeMs",
|
||||
"domBlitMs",
|
||||
];
|
||||
for (const key of perDomLayerKeys) {
|
||||
avgMs[key] = averageTiming(perf.timings[key], perf.domLayerCaptures);
|
||||
}
|
||||
|
||||
const perHdrVideoKeys: HdrPerfTimingKey[] = [
|
||||
"hdrVideoReadDecodeMs",
|
||||
"hdrVideoTransferMs",
|
||||
"hdrVideoBlitMs",
|
||||
];
|
||||
for (const key of perHdrVideoKeys) {
|
||||
avgMs[key] = averageTiming(perf.timings[key], perf.hdrVideoLayerBlits);
|
||||
}
|
||||
|
||||
const perHdrImageKeys: HdrPerfTimingKey[] = ["hdrImageTransferMs", "hdrImageBlitMs"];
|
||||
for (const key of perHdrImageKeys) {
|
||||
avgMs[key] = averageTiming(perf.timings[key], perf.hdrImageLayerBlits);
|
||||
}
|
||||
|
||||
return {
|
||||
frames: perf.frames,
|
||||
normalFrames: perf.normalFrames,
|
||||
transitionFrames: perf.transitionFrames,
|
||||
domLayerCaptures: perf.domLayerCaptures,
|
||||
hdrVideoLayerBlits: perf.hdrVideoLayerBlits,
|
||||
hdrImageLayerBlits: perf.hdrImageLayerBlits,
|
||||
timings: { ...perf.timings },
|
||||
avgMs,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* Build the `RenderPerfSummary` that lands on `job.perfSummary` and
|
||||
* the `perf-summary.json` debug artifact.
|
||||
*/
|
||||
|
||||
import { fpsToNumber } from "@hyperframes/core";
|
||||
import type {
|
||||
CaptureAttemptSummary,
|
||||
CaptureCalibrationSample,
|
||||
CaptureCostEstimate,
|
||||
HdrDiagnostics,
|
||||
RenderJob,
|
||||
RenderPerfSummary,
|
||||
} from "../renderOrchestrator.js";
|
||||
import { type HdrPerfCollector, finalizeHdrPerf } from "./hdrPerf.js";
|
||||
|
||||
export function buildRenderPerfSummary(input: {
|
||||
job: RenderJob;
|
||||
workerCount: number;
|
||||
enableChunkedEncode: boolean;
|
||||
chunkedEncodeSize: number;
|
||||
compositionDurationSeconds: number;
|
||||
totalFrames: number;
|
||||
outputWidth: number;
|
||||
outputHeight: number;
|
||||
videoCount: number;
|
||||
audioCount: number;
|
||||
totalElapsedMs: number;
|
||||
perfStages: Record<string, number>;
|
||||
videoExtractBreakdown: RenderPerfSummary["videoExtractBreakdown"];
|
||||
tmpPeakBytes: number;
|
||||
captureCalibration?: {
|
||||
estimate: CaptureCostEstimate;
|
||||
samples: CaptureCalibrationSample[];
|
||||
};
|
||||
captureAttempts: CaptureAttemptSummary[];
|
||||
hdrDiagnostics: HdrDiagnostics;
|
||||
hdrPerf?: HdrPerfCollector;
|
||||
peakRssBytes: number;
|
||||
peakHeapUsedBytes: number;
|
||||
}): RenderPerfSummary {
|
||||
return {
|
||||
renderId: input.job.id,
|
||||
totalElapsedMs: input.totalElapsedMs,
|
||||
// RenderPerfSummary surfaces fps as a decimal because it lands in JSON
|
||||
// payloads (CLI telemetry, regression-harness reports) where a single
|
||||
// number is friendlier than `{num,den}`. Callers needing the rational
|
||||
// back can read `job.config.fps`.
|
||||
fps: fpsToNumber(input.job.config.fps),
|
||||
quality: input.job.config.quality,
|
||||
workers: input.workerCount,
|
||||
chunkedEncode: input.enableChunkedEncode,
|
||||
chunkSizeFrames: input.enableChunkedEncode ? input.chunkedEncodeSize : null,
|
||||
compositionDurationSeconds: input.compositionDurationSeconds,
|
||||
totalFrames: input.totalFrames,
|
||||
resolution: { width: input.outputWidth, height: input.outputHeight },
|
||||
videoCount: input.videoCount,
|
||||
audioCount: input.audioCount,
|
||||
stages: input.perfStages,
|
||||
videoExtractBreakdown: input.videoExtractBreakdown,
|
||||
tmpPeakBytes: input.tmpPeakBytes,
|
||||
captureCalibration: input.captureCalibration
|
||||
? {
|
||||
sampledFrames: input.captureCalibration.samples.map((sample) => sample.frameIndex),
|
||||
p95Ms: input.captureCalibration.estimate.p95Ms,
|
||||
multiplier: input.captureCalibration.estimate.multiplier,
|
||||
reasons: input.captureCalibration.estimate.reasons,
|
||||
}
|
||||
: undefined,
|
||||
captureAttempts: input.captureAttempts.length > 0 ? input.captureAttempts : undefined,
|
||||
hdrDiagnostics:
|
||||
input.hdrDiagnostics.videoExtractionFailures > 0 ||
|
||||
input.hdrDiagnostics.imageDecodeFailures > 0
|
||||
? { ...input.hdrDiagnostics }
|
||||
: undefined,
|
||||
hdrPerf: input.hdrPerf ? finalizeHdrPerf(input.hdrPerf) : undefined,
|
||||
captureAvgMs:
|
||||
input.totalFrames > 0
|
||||
? Math.round((input.perfStages.captureMs ?? 0) / input.totalFrames)
|
||||
: undefined,
|
||||
peakRssMb: Math.round(input.peakRssBytes / (1024 * 1024)),
|
||||
peakHeapUsedMb: Math.round(input.peakHeapUsedBytes / (1024 * 1024)),
|
||||
};
|
||||
}
|
||||
@@ -10,10 +10,15 @@
|
||||
* backwards compatibility with existing test files and external callers.
|
||||
*/
|
||||
|
||||
import { copyFileSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { copyFileSync, cpSync, existsSync, mkdirSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { CANVAS_DIMENSIONS, type CanvasResolution } from "@hyperframes/core";
|
||||
import type { AudioElement, ImageElement, VideoElement } from "@hyperframes/engine";
|
||||
import type {
|
||||
AudioElement,
|
||||
ExtractedFrames,
|
||||
ImageElement,
|
||||
VideoElement,
|
||||
} from "@hyperframes/engine";
|
||||
import type { CompiledComposition } from "../htmlCompiler.js";
|
||||
import { defaultLogger, type ProducerLogger } from "../../logger.js";
|
||||
import { isPathInside } from "../../utils/paths.js";
|
||||
@@ -241,3 +246,177 @@ export function updateJobStatus(
|
||||
if (status === "failed" || status === "complete") job.completedAt = new Date();
|
||||
if (onProgress) onProgress(job, stage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `resolver(framePath)` closure that maps an absolute path to
|
||||
* a frame inside `compiledDir` into a server-relative URL the producer's
|
||||
* file server will serve. Returns `null` for any path that escapes the
|
||||
* compiled directory — the resolver is used by the video frame injector
|
||||
* to rewrite local frame references into HTTP `<video>` srcs.
|
||||
*/
|
||||
export function createCompiledFrameSrcResolver(
|
||||
compiledDir: string,
|
||||
): (framePath: string) => string | null {
|
||||
const compiledRoot = resolve(compiledDir);
|
||||
return (framePath: string): string | null => {
|
||||
const resolvedFramePath = resolve(framePath);
|
||||
if (!isPathInside(resolvedFramePath, compiledRoot)) return null;
|
||||
|
||||
const relativePath = relative(compiledRoot, resolvedFramePath);
|
||||
if (!relativePath || relativePath.startsWith("..") || isAbsolute(relativePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return `/${relativePath
|
||||
.split(/[\\/]+/)
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join("/")}`;
|
||||
};
|
||||
}
|
||||
|
||||
type MaterializedExtractedFrames = Pick<ExtractedFrames, "videoId" | "outputDir" | "framePaths">;
|
||||
|
||||
type MaterializePathModule = {
|
||||
resolve: (...segments: string[]) => string;
|
||||
join: (...segments: string[]) => string;
|
||||
dirname: (path: string) => string;
|
||||
basename: (path: string) => string;
|
||||
relative: (from: string, to: string) => string;
|
||||
isAbsolute: (path: string) => boolean;
|
||||
};
|
||||
|
||||
type MaterializeFileSystem = {
|
||||
existsSync: (path: string) => boolean;
|
||||
mkdirSync: (path: string, options: { recursive: true }) => unknown;
|
||||
symlinkSync: (target: string, path: string) => unknown;
|
||||
cpSync: (src: string, dest: string, options: { recursive: true }) => unknown;
|
||||
};
|
||||
|
||||
type MaterializeExtractedFramesOptions = {
|
||||
pathModule?: MaterializePathModule;
|
||||
fileSystem?: MaterializeFileSystem;
|
||||
/**
|
||||
* When `true`, recursively copy frames into `compiledDir` as real files
|
||||
* instead of creating a single symlink per video. Required for
|
||||
* distributed plan() output where the planDir must be self-contained
|
||||
* across machines (symlinks don't survive S3 / GCS round-trips).
|
||||
* Default `false` preserves the in-process renderer's symlink behavior.
|
||||
*/
|
||||
materializeSymlinks?: boolean;
|
||||
};
|
||||
|
||||
const materializePathModule: MaterializePathModule = {
|
||||
resolve,
|
||||
join,
|
||||
dirname,
|
||||
basename,
|
||||
relative,
|
||||
isAbsolute,
|
||||
};
|
||||
|
||||
const materializeFileSystem: MaterializeFileSystem = {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
symlinkSync,
|
||||
cpSync,
|
||||
};
|
||||
|
||||
/**
|
||||
* Periodic peak-RSS / peak-heapUsed sampler. The benchmark harness reads
|
||||
* the peaks to detect memory regressions (e.g. unbounded image-cache
|
||||
* growth) that wall-clock metrics miss.
|
||||
*
|
||||
* Sampled every 250ms; the interval is `unref`'d so the sampler never
|
||||
* holds the event loop open on its own. Callers MUST invoke `stop()`
|
||||
* in a `finally` block — `stop()` takes one final reading before
|
||||
* clearing the interval so the peak values are accurate up to the
|
||||
* moment the render returns.
|
||||
*/
|
||||
export interface MemorySampler {
|
||||
/** Take an immediate sample then read the peak RSS in bytes. */
|
||||
peakRssBytes: () => number;
|
||||
/** Take an immediate sample then read the peak heap-used in bytes. */
|
||||
peakHeapUsedBytes: () => number;
|
||||
/** Stop the interval after one final sample. Idempotent. */
|
||||
stop: () => void;
|
||||
}
|
||||
|
||||
export function createMemorySampler(intervalMs: number = 250): MemorySampler {
|
||||
let peakRss = 0;
|
||||
let peakHeap = 0;
|
||||
const sample = (): void => {
|
||||
try {
|
||||
const m = process.memoryUsage();
|
||||
if (m.rss > peakRss) peakRss = m.rss;
|
||||
if (m.heapUsed > peakHeap) peakHeap = m.heapUsed;
|
||||
} catch {
|
||||
// Defensive: process.memoryUsage() shouldn't throw, but if it ever
|
||||
// does we don't want to take down the render for a benchmark accessory.
|
||||
}
|
||||
};
|
||||
sample();
|
||||
const interval: NodeJS.Timeout = setInterval(sample, intervalMs);
|
||||
interval.unref();
|
||||
let stopped = false;
|
||||
return {
|
||||
// Resampling at read time means callers see the value at the
|
||||
// moment of inspection, not the last 250ms tick — important for
|
||||
// the success-path perf summary which captures peaks just before
|
||||
// returning.
|
||||
peakRssBytes: () => {
|
||||
sample();
|
||||
return peakRss;
|
||||
},
|
||||
peakHeapUsedBytes: () => {
|
||||
sample();
|
||||
return peakHeap;
|
||||
},
|
||||
stop: () => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
sample();
|
||||
clearInterval(interval);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Symlink (or copy) each extracted-frames directory into a stable path
|
||||
* under `compiledDir/__hyperframes_video_frames/<videoId>/`, and rewrite
|
||||
* the per-frame paths so the file server can serve them.
|
||||
*
|
||||
* Exported for integration tests; not part of the stable public API —
|
||||
* external callers should use `executeRenderJob` instead.
|
||||
*/
|
||||
export function materializeExtractedFramesForCompiledDir(
|
||||
extracted: MaterializedExtractedFrames[],
|
||||
compiledDir: string,
|
||||
options: MaterializeExtractedFramesOptions = {},
|
||||
): void {
|
||||
const pathModule = options.pathModule ?? materializePathModule;
|
||||
const fileSystem = options.fileSystem ?? materializeFileSystem;
|
||||
const resolvedCompiledDir = pathModule.resolve(compiledDir);
|
||||
const compiledFrameRoot = pathModule.join(resolvedCompiledDir, "__hyperframes_video_frames");
|
||||
|
||||
for (const ext of extracted) {
|
||||
const resolvedOut = pathModule.resolve(ext.outputDir);
|
||||
if (isPathInside(resolvedOut, resolvedCompiledDir, { pathModule })) continue;
|
||||
|
||||
const linkPath = pathModule.join(compiledFrameRoot, ext.videoId);
|
||||
if (!fileSystem.existsSync(linkPath)) {
|
||||
fileSystem.mkdirSync(pathModule.dirname(linkPath), { recursive: true });
|
||||
if (options.materializeSymlinks) {
|
||||
fileSystem.cpSync(resolvedOut, linkPath, { recursive: true });
|
||||
} else {
|
||||
fileSystem.symlinkSync(resolvedOut, linkPath);
|
||||
}
|
||||
}
|
||||
|
||||
const remapped = new Map<number, string>();
|
||||
for (const [idx, framePath] of ext.framePaths) {
|
||||
remapped.set(idx, pathModule.join(linkPath, pathModule.basename(framePath)));
|
||||
}
|
||||
ext.framePaths = remapped;
|
||||
ext.outputDir = linkPath;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,24 +9,26 @@ import {
|
||||
buildMissingFrameRetryBatches,
|
||||
collectVideoMetadataHints,
|
||||
collectVideoReadinessSkipIds,
|
||||
createCaptureCalibrationConfig,
|
||||
createCompiledFrameSrcResolver,
|
||||
estimateMeasuredCaptureCostMultiplier,
|
||||
estimateCaptureCostMultiplier,
|
||||
extractStandaloneEntryFromIndex,
|
||||
findMissingFrameRanges,
|
||||
getNextRetryWorkerCount,
|
||||
isRecoverableParallelCaptureError,
|
||||
materializeExtractedFramesForCompiledDir,
|
||||
resolveRenderWorkerCount,
|
||||
resolveCompositeTransfer,
|
||||
selectCaptureCalibrationFrames,
|
||||
shouldFallbackToScreenshotAfterCalibrationError,
|
||||
shouldUseLayeredComposite,
|
||||
shouldUseStreamingEncode,
|
||||
} from "./renderOrchestrator.js";
|
||||
import {
|
||||
createCaptureCalibrationConfig,
|
||||
estimateCaptureCostMultiplier,
|
||||
estimateMeasuredCaptureCostMultiplier,
|
||||
resolveRenderWorkerCount,
|
||||
selectCaptureCalibrationFrames,
|
||||
shouldFallbackToScreenshotAfterCalibrationError,
|
||||
} from "./render/captureCost.js";
|
||||
import {
|
||||
applyRenderModeHints,
|
||||
createCompiledFrameSrcResolver,
|
||||
materializeExtractedFramesForCompiledDir,
|
||||
projectBrowserEndToCompositionTimeline,
|
||||
resolveDeviceScaleFactor,
|
||||
writeCompiledArtifacts,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user