ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-11 06:30:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(producer): extract probeStage from executeRenderJob

Browse Source 
Move the browser probe / duration discovery / recompile / media
reconciliation block out of `executeRenderJob` into
`services/render/stages/probeStage.ts`. No behavior change. The sequencer
calls `runProbeStage` at the same code point with identical inputs and
outputs.

The probe stage owns the `FileServerHandle` and the `CaptureSession` it
creates and returns them to the sequencer. The sequencer still tracks
them in its `let fileServer` / `let probeSession` bindings and closes
them in its `finally` block — the resource lifetime is unchanged.

`recompileWithResolutions` lives inside this stage because it depends on
browser-resolved durations even though §2.1 of the distributed plan
lists recompile as a sibling phase.

Preserved invariants:

- `composition` is mutated in place (videos / audios / duration) so
  downstream stages see the reconciled view through the same reference.
- `job.duration` and `job.totalFrames` end up with the same values at
  the same code points. The result type carries `duration: number`
  alongside `totalFrames: number`, and the sequencer re-asserts the
  assignments after the call so TypeScript's control-flow narrowing
  works for the rest of `executeRenderJob`.
- `perfStages.browserProbeMs` and `perfStages.compileMs` are written at
  the same code points with the same values.
- The "Composition duration is 0" diagnostic builds the same hint string
  from the same console-buffer regex and `__timelines` probe.
- The post-probe "failed network requests" warning fires with the same
  regex, the same first-10/first-5 slicing, and the same `console.warn`
  prefix.

Renderer smoke-tested inside `Dockerfile.test` against `font-variant-numeric`,
`many-cuts`, and `variables-prod` — all PSNR / audio correlation baselines
match.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
James
2026-05-11 19:01:08 +00:00
co-authored by Claude Opus 4.7
parent d168397758
commit 90f14703b2
2 changed files with 397 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/producer/src/services/render/stages/probeStage.ts
+369
View File
@@ -0,0 +1,369 @@
/**
* probeStage — browser probe + recompile + media reconciliation.
*
* Runs only when `needsBrowser` is true (root duration unknown OR there are
* unresolved nested compositions). Owns the `FileServerHandle` and the
* `CaptureSession` it creates and returns them so the sequencer can both
* reuse them downstream (the capture stage reuses the probe session) and
* clean them up in its `finally` block.
*
* Hard constraints preserved verbatim from the in-process renderer:
* - `recompileWithResolutions` runs inside this stage because it depends
* on browser-resolved durations, even though §2.1 of the distributed
* plan lists recompile as a sibling phase.
* - `composition` (videos/audios/duration) is mutated in place — callers
* downstream see the reconciled view through the same object reference.
* - `job.duration` and `job.totalFrames` are assigned at the same code
* points.
* - The "Composition duration is 0" diagnostic builds the same hint
* string from the same console-buffer regex and `__timelines` probe.
* - The post-probe "failed network requests" warning fires with the same
* regex, the same first-10/first-5 slicing, and the same `console.warn`
* prefix.
*/
import { join } from "node:path";
import {
type CaptureOptions,
type CaptureSession,
type EngineConfig,
createCaptureSession,
getCompositionDuration,
initializeSession,
} from "@hyperframes/engine";
import { fpsToNumber } from "@hyperframes/core";
import type { CompiledComposition } from "../../htmlCompiler.js";
import {
discoverMediaFromBrowser,
recompileWithResolutions,
resolveCompositionDurations,
} from "../../htmlCompiler.js";
import { createFileServer, type FileServerHandle, VIRTUAL_TIME_SHIM } from "../../fileServer.js";
import type { ProducerLogger } from "../../../logger.js";
import {
projectBrowserEndToCompositionTimeline,
writeCompiledArtifacts,
type CompositionMetadata,
type RenderJob,
} from "../../renderOrchestrator.js";
const BROWSER_MEDIA_EPSILON = 0.0001;
export interface ProbeStageInput {
projectDir: string;
workDir: string;
job: RenderJob;
cfg: EngineConfig;
log: ProducerLogger;
assertNotAborted: () => void;
/** From compileStage. May be replaced via `recompileWithResolutions`. */
compiled: CompiledComposition;
/** From compileStage. Mutated in place (videos/audios pushed, duration set). */
composition: CompositionMetadata;
width: number;
height: number;
needsAlpha: boolean;
deviceScaleFactor: number;
}
export interface ProbeStageResult {
/** May be reassigned from `recompileWithResolutions`. */
compiled: CompiledComposition;
/** Created when `needsBrowser` was true; `null` otherwise. */
fileServer: FileServerHandle | null;
/** Created when `needsBrowser` was true; `null` otherwise. */
probeSession: CaptureSession | null;
/** The probeSession's `browserConsoleBuffer`, or `[]` if no probe ran. */
lastBrowserConsole: string[];
/** Composition duration (post-probe). Guaranteed > 0 — the stage throws on <= 0. */
duration: number;
totalFrames: number;
/** Wall-clock ms for the entire probe phase (0 if `needsBrowser` was false). */
browserProbeMs: number;
}
export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageResult> {
const {
projectDir,
workDir,
job,
cfg,
log,
assertNotAborted,
composition,
width,
height,
needsAlpha,
deviceScaleFactor,
} = input;
let { compiled } = input;
let fileServer: FileServerHandle | null = null;
let probeSession: CaptureSession | null = null;
let lastBrowserConsole: string[] = [];
const probeStart = Date.now();
const needsBrowser = composition.duration <= 0 || compiled.unresolvedCompositions.length > 0;
if (needsBrowser) {
const reasons = [];
if (composition.duration <= 0) reasons.push("root duration unknown");
if (compiled.unresolvedCompositions.length > 0)
reasons.push(`${compiled.unresolvedCompositions.length} unresolved composition(s)`);
fileServer = await createFileServer({
projectDir,
compiledDir: join(workDir, "compiled"),
port: 0,
preHeadScripts: [VIRTUAL_TIME_SHIM],
});
assertNotAborted();
const captureOpts: CaptureOptions = {
width,
height,
fps: job.config.fps,
format: needsAlpha ? "png" : "jpeg",
quality: needsAlpha ? undefined : 80,
deviceScaleFactor,
};
probeSession = await createCaptureSession(
fileServer.url,
join(workDir, "probe"),
captureOpts,
null,
cfg,
);
await initializeSession(probeSession);
assertNotAborted();
lastBrowserConsole = probeSession.browserConsoleBuffer;
// Discover root composition duration
if (composition.duration <= 0) {
const discoveredDuration = await getCompositionDuration(probeSession);
assertNotAborted();
log.info("Probed composition duration from browser", {
discoveredDuration,
staticDuration: compiled.staticDuration,
});
composition.duration = discoveredDuration;
} else {
log.info("Using static duration from data-duration attribute", {
duration: composition.duration,
});
}
// Resolve unresolved composition durations via window.__timelines
if (compiled.unresolvedCompositions.length > 0) {
const resolutions = await resolveCompositionDurations(
probeSession.page,
compiled.unresolvedCompositions,
);
assertNotAborted();
if (resolutions.length > 0) {
compiled = await recompileWithResolutions(
compiled,
resolutions,
projectDir,
join(workDir, "downloads"),
);
assertNotAborted();
// Update composition metadata with re-parsed media
composition.videos = compiled.videos;
composition.audios = compiled.audios;
composition.images = compiled.images;
writeCompiledArtifacts(compiled, workDir, Boolean(job.config.debug));
}
}
// Discover media elements from browser DOM (catches dynamically-set src)
const browserMedia = await discoverMediaFromBrowser(probeSession.page);
assertNotAborted();
if (browserMedia.length > 0) {
const existingVideoIds = new Set(composition.videos.map((v) => v.id));
const existingAudioIds = new Set(composition.audios.map((a) => a.id));
for (const el of browserMedia) {
if (!el.src || el.src === "about:blank") continue;
// Convert absolute localhost URLs back to relative paths
let src = el.src;
if (fileServer && src.startsWith(fileServer.url)) {
src = src.slice(fileServer.url.length).replace(/^\//, "");
}
if (el.tagName === "video") {
if (existingVideoIds.has(el.id)) {
// Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
const existing = composition.videos.find((v) => v.id === el.id);
if (existing) {
if (existing.src !== src) {
existing.src = src;
}
const projectedEnd = projectBrowserEndToCompositionTimeline(
existing.start,
el.start,
el.end,
);
if (
projectedEnd > 0 &&
(existing.end <= 0 || Math.abs(existing.end - projectedEnd) > BROWSER_MEDIA_EPSILON)
) {
existing.end = projectedEnd;
}
if (
el.mediaStart > 0 &&
(existing.mediaStart <= 0 ||
Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
) {
existing.mediaStart = el.mediaStart;
}
if (el.hasAudio && !existing.hasAudio) {
existing.hasAudio = true;
}
if (el.loop && !existing.loop) {
existing.loop = true;
}
}
} else {
// New video discovered from browser
composition.videos.push({
id: el.id,
src,
start: el.start,
end: el.end,
mediaStart: el.mediaStart,
loop: el.loop,
hasAudio: el.hasAudio,
});
existingVideoIds.add(el.id);
}
} else if (el.tagName === "audio") {
if (existingAudioIds.has(el.id)) {
const existing = composition.audios.find((a) => a.id === el.id);
if (existing) {
if (existing.src !== src) {
existing.src = src;
}
const projectedEnd = projectBrowserEndToCompositionTimeline(
existing.start,
el.start,
el.end,
);
if (
projectedEnd > 0 &&
(existing.end <= 0 || Math.abs(existing.end - projectedEnd) > BROWSER_MEDIA_EPSILON)
) {
existing.end = projectedEnd;
}
if (
el.mediaStart > 0 &&
(existing.mediaStart <= 0 ||
Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
) {
existing.mediaStart = el.mediaStart;
}
if (
el.volume > 0 &&
Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON
) {
existing.volume = el.volume;
}
}
} else {
composition.audios.push({
id: el.id,
src,
start: el.start,
end: el.end,
mediaStart: el.mediaStart,
layer: 0,
volume: el.volume,
type: "audio",
});
existingAudioIds.add(el.id);
}
}
}
}
}
const browserProbeMs = Date.now() - probeStart;
job.duration = composition.duration;
job.totalFrames = Math.ceil(composition.duration * fpsToNumber(job.config.fps));
const duration = composition.duration;
const totalFrames = job.totalFrames;
if (duration <= 0) {
// Gather diagnostics to help users understand why the render would produce a black video.
// Wrapped in try/catch because the browser tab may have crashed (which could be
// WHY duration is 0), and we don't want a Puppeteer error to mask the real message.
const diagnostics: string[] = [];
try {
if (probeSession) {
const timelinesInfo = await probeSession.page.evaluate(() => {
const tl = (window as any).__timelines;
const hf = (window as any).__hf;
return {
timelineKeys: tl ? Object.keys(tl) : [],
hfDuration: hf?.duration ?? null,
gsapLoaded: typeof (window as any).gsap !== "undefined",
};
});
if (!timelinesInfo.gsapLoaded) {
diagnostics.push(
"GSAP is not loaded — CDN script may have failed to download. " +
"Bundle GSAP locally in your project instead of using a CDN <script src>.",
);
} else if (timelinesInfo.timelineKeys.length === 0) {
diagnostics.push(
"GSAP is loaded but no timelines were registered on window.__timelines. " +
"Ensure your script creates a timeline and assigns it: " +
'window.__timelines["main"] = gsap.timeline({ paused: true });',
);
}
for (const line of probeSession.browserConsoleBuffer) {
if (/\[Browser:ERROR\]|\[Browser:PAGEERROR\]|404|net::ERR_/i.test(line)) {
diagnostics.push(`Browser: ${line}`);
}
}
}
} catch (err) {
log.warn("Failed to gather browser diagnostics for zero-duration composition", {
error: err instanceof Error ? err.message : String(err),
});
diagnostics.push("(Could not gather browser diagnostics — page may have crashed)");
}
const hint =
diagnostics.length > 0
? "\n\nDiagnostics:\n - " + diagnostics.join("\n - ")
: "\n\nCheck that GSAP timelines are registered on window.__timelines.";
throw new Error("Composition duration is 0 — this would produce a black video." + hint);
}
// Surface browser-side asset failures (404s, script errors) as warnings.
// These don't block the render but indicate missing images, fonts, or
// scripts that may produce unexpected visual artifacts.
if (probeSession) {
const failedRequests = probeSession.browserConsoleBuffer.filter((line) =>
/404|ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|net::ERR_/i.test(line),
);
if (failedRequests.length > 0) {
log.warn("Browser encountered network failures during page load:", {
failures: failedRequests.slice(0, 10),
});
for (const line of failedRequests.slice(0, 5)) {
console.warn(`[Render] Asset load failure: ${line}`);
}
}
}
return {
compiled,
fileServer,
probeSession,
lastBrowserConsole,
duration,
totalFrames,
browserProbeMs,
};
}
packages/producer/src/services/renderOrchestrator.ts
+28 -266
View File
@@ -53,7 +53,6 @@ import {
closeCaptureSession,
captureFrame,
captureFrameToBuffer,
getCompositionDuration,
prepareCaptureSessionForReuse,
type CaptureOptions,
type CaptureVideoMetadataHint,
@@ -108,12 +107,7 @@ import { randomUUID } from "crypto";
import { freemem } from "os";
import { fileURLToPath } from "url";
import { createFileServer, type FileServerHandle, VIRTUAL_TIME_SHIM } from "./fileServer.js";
import {
resolveCompositionDurations,
recompileWithResolutions,
discoverMediaFromBrowser,
type CompiledComposition,
} from "./htmlCompiler.js";
import { type CompiledComposition } from "./htmlCompiler.js";
import { defaultLogger, type ProducerLogger } from "../logger.js";
import { isPathInside } from "../utils/paths.js";
import {
@@ -121,6 +115,7 @@ import {
createHdrImageTransferCache,
} from "./hdrImageTransferCache.js";
import { runCompileStage } from "./render/stages/compileStage.js";
import { runProbeStage } from "./render/stages/probeStage.js";
/**
* Wrap a cleanup operation so it never throws, but logs any failure.
@@ -569,8 +564,6 @@ export interface CompositionMetadata {
height: number;
}
const BROWSER_MEDIA_EPSILON = 0.0001;
/**
* Browser-discovered media inside inlined sub-compositions can still report
* scene-local timing from the merged DOM (e.g. start=0, end=85.52) while the
@@ -2142,263 +2135,32 @@ export async function executeRenderJob(
const { width, height } = composition;
perfStages.compileOnlyMs = compileResult.compileOnlyMs;
const probeStart = Date.now();
const needsBrowser = composition.duration <= 0 || compiled.unresolvedCompositions.length > 0;
if (needsBrowser) {
const reasons = [];
if (composition.duration <= 0) reasons.push("root duration unknown");
if (compiled.unresolvedCompositions.length > 0)
reasons.push(`${compiled.unresolvedCompositions.length} unresolved composition(s)`);
fileServer = await createFileServer({
projectDir,
compiledDir: join(workDir, "compiled"),
port: 0,
preHeadScripts: [VIRTUAL_TIME_SHIM],
});
assertNotAborted();
const captureOpts: CaptureOptions = {
width,
height,
fps: job.config.fps,
format: needsAlpha ? "png" : "jpeg",
quality: needsAlpha ? undefined : 80,
deviceScaleFactor,
};
probeSession = await createCaptureSession(
fileServer.url,
join(workDir, "probe"),
captureOpts,
null,
cfg,
);
await initializeSession(probeSession);
assertNotAborted();
lastBrowserConsole = probeSession.browserConsoleBuffer;
// Discover root composition duration
if (composition.duration <= 0) {
const discoveredDuration = await getCompositionDuration(probeSession);
assertNotAborted();
log.info("Probed composition duration from browser", {
discoveredDuration,
staticDuration: compiled.staticDuration,
});
composition.duration = discoveredDuration;
} else {
log.info("Using static duration from data-duration attribute", {
duration: composition.duration,
});
}
// Resolve unresolved composition durations via window.__timelines
if (compiled.unresolvedCompositions.length > 0) {
const resolutions = await resolveCompositionDurations(
probeSession.page,
compiled.unresolvedCompositions,
);
assertNotAborted();
if (resolutions.length > 0) {
compiled = await recompileWithResolutions(
compiled,
resolutions,
projectDir,
join(workDir, "downloads"),
);
assertNotAborted();
// Update composition metadata with re-parsed media
composition.videos = compiled.videos;
composition.audios = compiled.audios;
composition.images = compiled.images;
writeCompiledArtifacts(compiled, workDir, Boolean(job.config.debug));
}
}
// Discover media elements from browser DOM (catches dynamically-set src)
const browserMedia = await discoverMediaFromBrowser(probeSession.page);
assertNotAborted();
if (browserMedia.length > 0) {
const existingVideoIds = new Set(composition.videos.map((v) => v.id));
const existingAudioIds = new Set(composition.audios.map((a) => a.id));
for (const el of browserMedia) {
if (!el.src || el.src === "about:blank") continue;
// Convert absolute localhost URLs back to relative paths
let src = el.src;
if (fileServer && src.startsWith(fileServer.url)) {
src = src.slice(fileServer.url.length).replace(/^\//, "");
}
if (el.tagName === "video") {
if (existingVideoIds.has(el.id)) {
// Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
const existing = composition.videos.find((v) => v.id === el.id);
if (existing) {
if (existing.src !== src) {
existing.src = src;
}
const projectedEnd = projectBrowserEndToCompositionTimeline(
existing.start,
el.start,
el.end,
);
if (
projectedEnd > 0 &&
(existing.end <= 0 ||
Math.abs(existing.end - projectedEnd) > BROWSER_MEDIA_EPSILON)
) {
existing.end = projectedEnd;
}
if (
el.mediaStart > 0 &&
(existing.mediaStart <= 0 ||
Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
) {
existing.mediaStart = el.mediaStart;
}
if (el.hasAudio && !existing.hasAudio) {
existing.hasAudio = true;
}
if (el.loop && !existing.loop) {
existing.loop = true;
}
}
} else {
// New video discovered from browser
composition.videos.push({
id: el.id,
src,
start: el.start,
end: el.end,
mediaStart: el.mediaStart,
loop: el.loop,
hasAudio: el.hasAudio,
});
existingVideoIds.add(el.id);
}
} else if (el.tagName === "audio") {
if (existingAudioIds.has(el.id)) {
const existing = composition.audios.find((a) => a.id === el.id);
if (existing) {
if (existing.src !== src) {
existing.src = src;
}
const projectedEnd = projectBrowserEndToCompositionTimeline(
existing.start,
el.start,
el.end,
);
if (
projectedEnd > 0 &&
(existing.end <= 0 ||
Math.abs(existing.end - projectedEnd) > BROWSER_MEDIA_EPSILON)
) {
existing.end = projectedEnd;
}
if (
el.mediaStart > 0 &&
(existing.mediaStart <= 0 ||
Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
) {
existing.mediaStart = el.mediaStart;
}
if (
el.volume > 0 &&
Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON
) {
existing.volume = el.volume;
}
}
} else {
composition.audios.push({
id: el.id,
src,
start: el.start,
end: el.end,
mediaStart: el.mediaStart,
layer: 0,
volume: el.volume,
type: "audio",
});
existingAudioIds.add(el.id);
}
}
}
}
}
perfStages.browserProbeMs = Date.now() - probeStart;
job.duration = composition.duration;
job.totalFrames = Math.ceil(composition.duration * fpsToNumber(job.config.fps));
const totalFrames = job.totalFrames;
if (job.duration <= 0) {
// Gather diagnostics to help users understand why the render would produce a black video.
// Wrapped in try/catch because the browser tab may have crashed (which could be
// WHY duration is 0), and we don't want a Puppeteer error to mask the real message.
const diagnostics: string[] = [];
try {
if (probeSession) {
const timelinesInfo = await probeSession.page.evaluate(() => {
const tl = (window as any).__timelines;
const hf = (window as any).__hf;
return {
timelineKeys: tl ? Object.keys(tl) : [],
hfDuration: hf?.duration ?? null,
gsapLoaded: typeof (window as any).gsap !== "undefined",
};
});
if (!timelinesInfo.gsapLoaded) {
diagnostics.push(
"GSAP is not loaded — CDN script may have failed to download. " +
"Bundle GSAP locally in your project instead of using a CDN <script src>.",
);
} else if (timelinesInfo.timelineKeys.length === 0) {
diagnostics.push(
"GSAP is loaded but no timelines were registered on window.__timelines. " +
"Ensure your script creates a timeline and assigns it: " +
'window.__timelines["main"] = gsap.timeline({ paused: true });',
);
}
for (const line of probeSession.browserConsoleBuffer) {
if (/\[Browser:ERROR\]|\[Browser:PAGEERROR\]|404|net::ERR_/i.test(line)) {
diagnostics.push(`Browser: ${line}`);
}
}
}
} catch (err) {
log.warn("Failed to gather browser diagnostics for zero-duration composition", {
error: err instanceof Error ? err.message : String(err),
});
diagnostics.push("(Could not gather browser diagnostics — page may have crashed)");
}
const hint =
diagnostics.length > 0
? "\n\nDiagnostics:\n - " + diagnostics.join("\n - ")
: "\n\nCheck that GSAP timelines are registered on window.__timelines.";
throw new Error("Composition duration is 0 — this would produce a black video." + hint);
}
// Surface browser-side asset failures (404s, script errors) as warnings.
// These don't block the render but indicate missing images, fonts, or
// scripts that may produce unexpected visual artifacts.
if (probeSession) {
const failedRequests = probeSession.browserConsoleBuffer.filter((line) =>
/404|ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|net::ERR_/i.test(line),
);
if (failedRequests.length > 0) {
log.warn("Browser encountered network failures during page load:", {
failures: failedRequests.slice(0, 10),
});
for (const line of failedRequests.slice(0, 5)) {
console.warn(`[Render] Asset load failure: ${line}`);
}
}
}
const probeResult = await runProbeStage({
projectDir,
workDir,
job,
cfg,
log,
assertNotAborted,
compiled,
composition,
width,
height,
needsAlpha,
deviceScaleFactor,
});
compiled = probeResult.compiled;
fileServer = probeResult.fileServer;
probeSession = probeResult.probeSession;
lastBrowserConsole = probeResult.lastBrowserConsole;
// Re-assign through the typed result so the rest of the function sees
// `job.duration` and `job.totalFrames` narrowed to `number` — the
// assignments happened inside `runProbeStage`, but mirroring them here
// restores TypeScript's control-flow narrowing.
job.duration = probeResult.duration;
job.totalFrames = probeResult.totalFrames;
const totalFrames = probeResult.totalFrames;
perfStages.browserProbeMs = probeResult.browserProbeMs;
perfStages.compileMs = Date.now() - stage1Start;
// ── Stage 2: Video frame extraction ─────────────────────────────────