ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-01 19:42:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: speed up video frame injection renders (#596)

Browse Source
This commit is contained in:
Miguel Ángel
2026-05-02 05:11:05 +02:00
committed by GitHub
parent 04bd56a7ae
commit 4750a981dd
5 changed files with 236 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/engine/src/services/frameCapture.ts
+53 -2
View File
@@ -74,6 +74,47 @@ export interface CaptureSession {
// Circular buffer for browser console messages dumped on render failure diagnostics.
// Complex compositions produce 100+ messages; 50 was too small to capture relevant errors.
const BROWSER_CONSOLE_BUFFER_SIZE = 200;
const CAPTURE_SESSION_CLOSE_TIMEOUT_MS = 5_000;
async function waitForCloseWithTimeout(promise: Promise<unknown>): Promise<boolean> {
let timedOut = false;
let timer: ReturnType<typeof setTimeout> | undefined;
await Promise.race([
promise.then(
() => undefined,
() => undefined,
),
new Promise<void>((resolve) => {
timer = setTimeout(() => {
timedOut = true;
resolve();
}, CAPTURE_SESSION_CLOSE_TIMEOUT_MS);
}),
]);
if (timer) clearTimeout(timer);
return !timedOut;
}
function forceKillBrowserProcess(browser: Browser): void {
const browserProcess = (
browser as unknown as {
process?: () => { kill: (signal?: NodeJS.Signals) => boolean; killed?: boolean } | null;
}
).process?.();
if (browserProcess && !browserProcess.killed) {
try {
browserProcess.kill("SIGKILL");
} catch {
// Best-effort cleanup after Puppeteer close has already timed out.
}
}
try {
browser.disconnect();
} catch {
// Best-effort cleanup after Puppeteer close has already timed out.
}
}
export async function createCaptureSession(
serverUrl: string,
@@ -674,11 +715,21 @@ export async function closeCaptureSession(session: CaptureSession): Promise<void
// but browserReleased=false → second call no-ops on page and retries browser.
// This matches the orchestrator's intent for HDR cleanup.
if (!session.pageReleased && session.page) {
await session.page.close().catch(() => {});
const pageClosed = await waitForCloseWithTimeout(session.page.close());
if (!pageClosed) {
console.warn("[FrameCapture] Timed out closing page; forcing browser process shutdown");
forceKillBrowserProcess(session.browser);
}
session.pageReleased = true;
}
if (!session.browserReleased && session.browser) {
await releaseBrowser(session.browser, session.config);
const browserClosed = await waitForCloseWithTimeout(
releaseBrowser(session.browser, session.config),
);
if (!browserClosed) {
console.warn("[FrameCapture] Timed out closing browser; forcing browser process shutdown");
forceKillBrowserProcess(session.browser);
}
session.browserReleased = true;
}
session.isInitialized = false;
packages/engine/src/services/screenshotService.ts
+9 -7
View File
@@ -446,13 +446,15 @@ export async function injectVideoFramesBatch(
}
}
img.decoding = "sync";
img.src = item.dataUri;
pendingDecodes.push(
img
.decode()
.catch(() => undefined)
.then(() => undefined),
);
if (img.getAttribute("src") !== item.dataUri) {
img.src = item.dataUri;
pendingDecodes.push(
img
.decode()
.catch(() => undefined)
.then(() => undefined),
);
}
img.style.opacity = String(computedOpacity);
img.style.visibility = "visible";
// Hide the native <video> with visibility only — never clobber inline
packages/engine/src/services/videoFrameInjector.ts
+15 -3
View File
@@ -14,7 +14,16 @@ import { injectVideoFramesBatch, syncVideoFrameVisibility } from "./screenshotSe
import { type BeforeCaptureHook } from "./frameCapture.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
function createFrameDataUriCache(cacheLimit: number) {
export interface VideoFrameInjectorOptions extends Partial<
Pick<EngineConfig, "frameDataUriCacheLimit">
> {
frameSrcResolver?: (framePath: string) => string | null;
}
function createFrameSourceCache(
cacheLimit: number,
frameSrcResolver?: (framePath: string) => string | null,
) {
const cache = new Map<string, string>();
const inFlight = new Map<string, Promise<string>>();
@@ -33,6 +42,9 @@ function createFrameDataUriCache(cacheLimit: number) {
}
async function get(framePath: string): Promise<string> {
const servedSrc = frameSrcResolver?.(framePath);
if (servedSrc) return servedSrc;
const cached = cache.get(framePath);
if (cached) {
remember(framePath, cached);
@@ -67,7 +79,7 @@ function createFrameDataUriCache(cacheLimit: number) {
*/
export function createVideoFrameInjector(
frameLookup: FrameLookupTable | null,
config?: Partial<Pick<EngineConfig, "frameDataUriCacheLimit">>,
config?: VideoFrameInjectorOptions,
): BeforeCaptureHook | null {
if (!frameLookup) return null;
@@ -75,7 +87,7 @@ export function createVideoFrameInjector(
32,
config?.frameDataUriCacheLimit ?? DEFAULT_CONFIG.frameDataUriCacheLimit,
);
const frameCache = createFrameDataUriCache(cacheLimit);
const frameCache = createFrameSourceCache(cacheLimit, config?.frameSrcResolver);
const lastInjectedFrameByVideo = new Map<string, number>();
return async (page: Page, time: number) => {
packages/producer/src/services/renderOrchestrator.test.ts
+50 -37
View File
@@ -11,6 +11,7 @@ import {
collectVideoMetadataHints,
collectVideoReadinessSkipIds,
createCaptureCalibrationConfig,
createCompiledFrameSrcResolver,
estimateMeasuredCaptureCostMultiplier,
estimateCaptureCostMultiplier,
extractStandaloneEntryFromIndex,
@@ -125,6 +126,22 @@ describe("shouldUseStreamingEncode", () => {
});
});
describe("createCompiledFrameSrcResolver", () => {
it("maps extracted frame paths under compiledDir to encoded server URLs", () => {
const resolver = createCompiledFrameSrcResolver("/tmp/hf job/compiled");
expect(
resolver("/tmp/hf job/compiled/__hyperframes_video_frames/video 1/frame_00001.jpg"),
).toBe("/__hyperframes_video_frames/video%201/frame_00001.jpg");
});
it("returns null for paths outside compiledDir", () => {
const resolver = createCompiledFrameSrcResolver("/tmp/hf-job/compiled");
expect(resolver("/tmp/hf-job/video-frames/frame_00001.jpg")).toBeNull();
});
});
describe("writeCompiledArtifacts — external assets on Windows drive-letter paths (GH #321)", () => {
const tempDirs: string[] = [];
afterEach(() => {
@@ -337,15 +354,6 @@ describe("collectVideoMetadataHints", () => {
describe("resolveRenderWorkerCount", () => {
const cfg = { ...createConfig(), coresPerWorker: 100 };
const audio = {
id: "narration",
src: "narration.wav",
start: 0,
end: 3,
mediaStart: 0,
layer: 9,
type: "audio" as const,
};
it("reduces auto workers for expensive capture workloads", () => {
const log = {
@@ -363,7 +371,6 @@ describe("resolveRenderWorkerCount", () => {
hasShaderTransitions: true,
renderModeHints: { recommendScreenshot: false, reasons: [] },
},
{ videos: [], audios: [audio] },
log,
);
@@ -387,7 +394,6 @@ describe("resolveRenderWorkerCount", () => {
hasShaderTransitions: true,
renderModeHints: { recommendScreenshot: false, reasons: [] },
},
{ videos: [], audios: [audio] },
log,
);
@@ -404,43 +410,50 @@ describe("resolveRenderWorkerCount", () => {
hasShaderTransitions: false,
renderModeHints: { recommendScreenshot: false, reasons: [] },
},
{ videos: [], audios: [] },
undefined,
{ multiplier: 4, reasons: ["calibration-p95=2400ms"] },
);
expect(workers).toBe(1);
});
it("keeps baseline auto workers after screenshot fallback when measured capture is cheap", () => {
const log = {
error: vi.fn(),
warn: vi.fn(),
info: vi.fn(),
debug: vi.fn(),
};
const workers = resolveRenderWorkerCount(
180,
undefined,
{ ...cfg, forceScreenshot: true },
{
hasShaderTransitions: false,
renderModeHints: { recommendScreenshot: false, reasons: [] },
},
log,
{ multiplier: 1, reasons: [], p95Ms: 180 },
);
expect(workers).toBe(6);
expect(log.warn).not.toHaveBeenCalled();
});
});
describe("estimateCaptureCostMultiplier", () => {
it("weights shader transitions, media, and render mode hints", () => {
const cost = estimateCaptureCostMultiplier(
{
hasShaderTransitions: true,
renderModeHints: {
recommendScreenshot: true,
reasons: [{ code: "requestAnimationFrame", message: "raw rAF" }],
},
it("weights shader transitions and render mode hints without charging static media cost", () => {
const cost = estimateCaptureCostMultiplier({
hasShaderTransitions: true,
renderModeHints: {
recommendScreenshot: true,
reasons: [{ code: "requestAnimationFrame", message: "raw rAF" }],
},
{
videos: [],
audios: [
{
id: "narration",
src: "narration.wav",
start: 0,
end: 3,
mediaStart: 0,
layer: 9,
type: "audio" as const,
},
],
},
);
});
expect(cost.multiplier).toBe(4.75);
expect(cost.reasons).toEqual(["shader-transitions", "requestAnimationFrame", "1 audio"]);
expect(cost.multiplier).toBe(4);
expect(cost.reasons).toEqual(["shader-transitions", "requestAnimationFrame"]);
});
});
packages/producer/src/services/renderOrchestrator.ts
+109 -52
View File
@@ -93,7 +93,7 @@ import {
type ElementStackingInfo,
type HfTransitionMeta,
} from "@hyperframes/engine";
import { join, dirname, resolve } from "path";
import { join, dirname, resolve, relative, isAbsolute } from "path";
import { randomUUID } from "crypto";
import { freemem } from "os";
import { fileURLToPath } from "url";
@@ -665,6 +665,26 @@ export function writeCompiledArtifacts(
}
}
export function createCompiledFrameSrcResolver(
compiledDir: string,
): (framePath: string) => string | null {
const compiledRoot = resolve(compiledDir);
return (framePath: string): string | null => {
const resolvedFramePath = resolve(framePath);
if (!isPathInside(resolvedFramePath, compiledRoot)) return null;
const relativePath = relative(compiledRoot, resolvedFramePath);
if (!relativePath || relativePath.startsWith("..") || isAbsolute(relativePath)) {
return null;
}
return `/${relativePath
.split(/[\\/]+/)
.map((segment) => encodeURIComponent(segment))
.join("/")}`;
};
}
export function applyRenderModeHints(
cfg: EngineConfig,
compiled: CompiledComposition,
@@ -728,12 +748,11 @@ export function resolveRenderWorkerCount(
requestedWorkers: number | undefined,
cfg: EngineConfig,
compiled: Pick<CompiledComposition, "hasShaderTransitions" | "renderModeHints">,
composition: Pick<CompositionMetadata, "videos" | "audios">,
log: ProducerLogger = defaultLogger,
measuredCaptureCost?: CaptureCostEstimate,
): number {
const captureCost = combineCaptureCostEstimates(
estimateCaptureCostMultiplier(compiled, composition),
estimateCaptureCostMultiplier(compiled),
measuredCaptureCost,
);
const workerCount = calculateOptimalWorkers(totalFrames, requestedWorkers, {
@@ -763,7 +782,6 @@ export function resolveRenderWorkerCount(
export function estimateCaptureCostMultiplier(
compiled: Pick<CompiledComposition, "hasShaderTransitions" | "renderModeHints">,
composition: Pick<CompositionMetadata, "videos" | "audios">,
): CaptureCostEstimate {
let multiplier = 1;
const reasons: string[] = [];
@@ -783,16 +801,6 @@ export function estimateCaptureCostMultiplier(
reasons.push("iframe");
}
if (composition.videos.length > 0) {
multiplier += Math.min(2, composition.videos.length * 0.75);
reasons.push(`${composition.videos.length} video${composition.videos.length === 1 ? "" : "s"}`);
}
if (composition.audios.length > 0) {
multiplier += Math.min(1, composition.audios.length * 0.75);
reasons.push(`${composition.audios.length} audio${composition.audios.length === 1 ? "" : "s"}`);
}
return {
multiplier: Math.round(multiplier * 100) / 100,
reasons,
@@ -981,6 +989,37 @@ async function measureCaptureCostFromSession(
};
}
function logCaptureCalibrationResult(
calibration: { estimate: CaptureCostEstimate; samples: CaptureCalibrationSample[] },
log: ProducerLogger,
): void {
if (calibration.estimate.multiplier > 1) {
log.warn("[Render] Measured slow frame capture during auto-worker calibration.", {
multiplier: calibration.estimate.multiplier,
p95Ms: calibration.estimate.p95Ms,
sampledFrames: calibration.samples.map((sample) => sample.frameIndex),
});
} else {
log.debug("[Render] Auto-worker calibration kept baseline capture cost.", {
p95Ms: calibration.estimate.p95Ms,
sampledFrames: calibration.samples.map((sample) => sample.frameIndex),
});
}
}
function createFailedCaptureCalibrationEstimate(reason: string): {
estimate: CaptureCostEstimate;
samples: CaptureCalibrationSample[];
} {
return {
estimate: {
multiplier: MAX_MEASURED_CAPTURE_COST_MULTIPLIER,
reasons: [reason],
},
samples: [],
};
}
async function executeDiskCaptureWithAdaptiveRetry(options: {
serverUrl: string;
workDir: string;
@@ -2248,7 +2287,7 @@ export async function executeRenderJob(
extractionResult = await extractAllVideoFrames(
composition.videos,
projectDir,
{ fps: job.config.fps, outputDir: join(workDir, "video-frames") },
{ fps: job.config.fps, outputDir: join(compiledDir, "__hyperframes_video_frames") },
abortSignal,
{ extractCacheDir: cfg.extractCacheDir },
compiledDir,
@@ -2418,6 +2457,12 @@ export async function executeRenderJob(
videoMetadataHints,
skipReadinessVideoIds: videoReadinessSkipIds,
});
const frameSrcResolver = createCompiledFrameSrcResolver(compiledDir);
const createRenderVideoFrameInjector = (): BeforeCaptureHook | null =>
createVideoFrameInjector(frameLookup, {
frameDataUriCacheLimit: cfg.frameDataUriCacheLimit,
frameSrcResolver,
});
let captureCalibration:
| {
@@ -2425,12 +2470,11 @@ export async function executeRenderJob(
samples: CaptureCalibrationSample[];
}
| undefined;
let switchedToScreenshotAfterCalibration = false;
if (job.config.workers === undefined && totalFrames >= 60) {
const calibrationDir = join(workDir, "capture-calibration");
const calibrationCfg = createCaptureCalibrationConfig(cfg);
const videoInjector = createVideoFrameInjector(frameLookup);
const videoInjector = createRenderVideoFrameInjector();
let calibrationSession: CaptureSession | null = null;
try {
calibrationSession = await createCaptureSession(
@@ -2450,48 +2494,66 @@ export async function executeRenderJob(
totalFrames,
job.config.fps,
);
if (captureCalibration.estimate.multiplier > 1) {
log.warn("[Render] Measured slow frame capture during auto-worker calibration.", {
multiplier: captureCalibration.estimate.multiplier,
p95Ms: captureCalibration.estimate.p95Ms,
sampledFrames: captureCalibration.samples.map((sample) => sample.frameIndex),
});
} else {
log.debug("[Render] Auto-worker calibration kept baseline capture cost.", {
p95Ms: captureCalibration.estimate.p95Ms,
sampledFrames: captureCalibration.samples.map((sample) => sample.frameIndex),
});
}
logCaptureCalibrationResult(captureCalibration, log);
} catch (error) {
const shouldFallbackToScreenshot =
!cfg.forceScreenshot && shouldFallbackToScreenshotAfterCalibrationError(error);
if (shouldFallbackToScreenshot) {
cfg.forceScreenshot = true;
switchedToScreenshotAfterCalibration = true;
if (probeSession) {
lastBrowserConsole = probeSession.browserConsoleBuffer;
await closeCaptureSession(probeSession).catch(() => {});
probeSession = null;
}
}
captureCalibration = {
estimate: {
multiplier: MAX_MEASURED_CAPTURE_COST_MULTIPLIER,
reasons: shouldFallbackToScreenshot
? ["calibration-beginframe-timeout", "screenshot-fallback"]
: ["calibration-failed"],
},
samples: [],
};
if (shouldFallbackToScreenshot) {
if (calibrationSession) {
lastBrowserConsole = calibrationSession.browserConsoleBuffer;
await closeCaptureSession(calibrationSession).catch(() => {});
calibrationSession = null;
}
log.warn(
"[Render] BeginFrame auto-worker calibration timed out; falling back to screenshot capture mode.",
"[Render] BeginFrame auto-worker calibration timed out; retrying calibration in screenshot capture mode.",
{
protocolTimeout: calibrationCfg.protocolTimeout,
error: error instanceof Error ? error.message : String(error),
},
);
const screenshotCalibrationCfg = createCaptureCalibrationConfig(cfg);
try {
calibrationSession = await createCaptureSession(
fileServer.url,
join(workDir, "capture-calibration-screenshot"),
buildCaptureOptions(),
createRenderVideoFrameInjector(),
screenshotCalibrationCfg,
);
if (!calibrationSession.isInitialized) {
await initializeSession(calibrationSession);
}
assertNotAborted();
captureCalibration = await measureCaptureCostFromSession(
calibrationSession,
totalFrames,
job.config.fps,
);
logCaptureCalibrationResult(captureCalibration, log);
} catch (fallbackError) {
captureCalibration = createFailedCaptureCalibrationEstimate(
"calibration-screenshot-failed",
);
log.warn(
"[Render] Screenshot auto-worker calibration failed after BeginFrame fallback; using conservative worker budget.",
{
protocolTimeout: screenshotCalibrationCfg.protocolTimeout,
error:
fallbackError instanceof Error ? fallbackError.message : String(fallbackError),
},
);
}
} else {
captureCalibration = createFailedCaptureCalibrationEstimate("calibration-failed");
log.warn("[Render] Auto-worker calibration failed; using conservative worker budget.", {
protocolTimeout: calibrationCfg.protocolTimeout,
error: error instanceof Error ? error.message : String(error),
@@ -2510,15 +2572,10 @@ export async function executeRenderJob(
job.config.workers,
cfg,
compiled,
composition,
log,
captureCalibration?.estimate,
);
if (switchedToScreenshotAfterCalibration && workerCount > 1) {
workerCount = 1;
}
if (workerCount > 1 && probeSession) {
lastBrowserConsole = probeSession.browserConsoleBuffer;
await closeCaptureSession(probeSession);
@@ -2646,7 +2703,7 @@ export async function executeRenderJob(
fileServer.url,
framesDir,
buildCaptureOptions(),
createVideoFrameInjector(frameLookup),
createRenderVideoFrameInjector(),
cfg,
);
// Track lifecycle of resources spawned during HDR rendering so the
@@ -3411,7 +3468,7 @@ export async function executeRenderJob(
workDir,
tasks,
buildCaptureOptions(),
() => createVideoFrameInjector(frameLookup),
createRenderVideoFrameInjector,
abortSignal,
(progress) => {
job.framesRendered = progress.capturedFrames;
@@ -3443,7 +3500,7 @@ export async function executeRenderJob(
} else {
// Sequential capture → streaming encode
const videoInjector = createVideoFrameInjector(frameLookup);
const videoInjector = createRenderVideoFrameInjector();
const session =
probeSession ??
(await createCaptureSession(
@@ -3515,7 +3572,7 @@ export async function executeRenderJob(
allowRetry: job.config.workers === undefined,
frameExt: needsAlpha ? "png" : "jpg",
captureOptions: buildCaptureOptions(),
createBeforeCaptureHook: () => createVideoFrameInjector(frameLookup),
createBeforeCaptureHook: createRenderVideoFrameInjector,
abortSignal,
onProgress: (progress) => {
job.framesRendered = progress.capturedFrames;
@@ -3551,7 +3608,7 @@ export async function executeRenderJob(
} else {
// Sequential capture
const videoInjector = createVideoFrameInjector(frameLookup);
const videoInjector = createRenderVideoFrameInjector();
const session =
probeSession ??
(await createCaptureSession(