feat(producer): fast-capture render stages + remote bg-image localizer (#1920)

* feat(engine): drawElementImage capture service

* feat(engine): 3D projection + compositor-effect risk gate

* fix(engine): gate filter drop-shadow wherever blur gates (review)

detectCssEffectRisk documented drop-shadow as a ~29dB damage case but only
detected blur( in its three scan paths — a drop-shadow comp stayed on the
fast path despite the gate's own correctness contract. Detect drop-shadow(
in computed styles, stylesheet rules, and GSAP tween vars, pinned by a
focused test that runs the real page-side closure against a DOM shim
(computed / stylesheet / tween coverage + blur regression + effect-free
null).

Addresses miguel-heygen's blocker on #1918.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(engine): frame-capture core — fast-capture routing, worker-encode, dedup extension

# Conflicts:
#	packages/engine/src/services/screenshotService.ts

* fix(engine): document HF_FORCE_DRAWELEMENT as diagnostic-only; make armStaticDedup idempotent (review)

Addresses miguel-heygen's blockers on #1919:

- HF_FORCE_DRAWELEMENT promoted from a stale "SCRATCH/Uncommitted" comment to
  a documented diagnostic flag: it exists for upstream-Chromium repro work
  (gate-vs-API isolation, crbug 521861819 149-vs-151) and R&D on gated effect
  classes; renders under it may be damaged BY DESIGN since it bypasses gates
  whose thresholds encode measured damage. Never production; the safety-net
  blank guard also stands down under it so diagnostic frames arrive unmodified.
- armStaticDedup is now idempotent: the drawElement init path arms dedup
  before canvas injection, then initializeSession called it again — the
  second run overwrote the armed state with skipReason="capture_mode"
  (captureMode is "drawelement" by then), producing contradictory telemetry
  (armed frames + a skip reason), and re-ran the verification seeks on the
  fallback path. It now no-ops once staticFrames or a skip decision exists.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(producer): fast-capture render stages + remote bg-image localizer

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Vance Ingalls
2026-07-06 16:21:39 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent 0e58344dca
commit 1d0dbcd3b2
18 changed files with 858 additions and 50 deletions
@@ -1,6 +1,6 @@
// AUTO-GENERATED by scripts/build-hf-early-stub.ts — do not edit
const HF_EARLY_STUB_IIFE: string =
'"use strict";(()=>{var T=100,_=[],u=[],l=!1,s=!1;function m(n){let i=window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;return typeof i=="function"?i(n):requestAnimationFrame(n)}function y(n){let i=window.__HF_VIRTUAL_TIME__?.originalSetTimeout;if(typeof i=="function"){i(n,0);return}setTimeout(n,0)}function g(n){return n!==null&&typeof n=="object"&&"__hfIsProxy"in n?n.__hfReal:n}function c(n){let i=n.proxy.__hfReal,e=i[n.method];if(typeof e=="function"){let o=n.method==="add"?n.args.map(g):n.args;e.call(i,...o)}}function r(n,i,e){let o={proxy:n,method:i,args:e};return n.__hfQueue.push(o),u.push(o),P(),n}function d(n){let i=n.proxy.__hfQueue.indexOf(n);i>=0&&n.proxy.__hfQueue.splice(i,1)}function t(){for(;u.length>0;){let n=u.shift();n&&(d(n),c(n))}x()}function f(){s=!1,window.__hfTimelinesBuilding=!1;try{window.dispatchEvent(new CustomEvent("hf-timelines-built"))}catch{}}function x(){s||(s=!0,y(()=>{u.length===0?f():s=!1}))}function k(){l=!1;let n=u.splice(0,T);for(let i of n)d(i),c(i);u.length>0?(l=!0,m(k)):f()}function P(){l||(l=!0,window.__hfTimelinesBuilding=!0,m(k))}var O=new Set(["to","from","fromTo","set","add"]);function b(n,i){let e=i;for(;e!==null&&e!==Object.prototype;){for(let o of Object.getOwnPropertyNames(e)){if(o==="constructor"||o==="then"||o in n||O.has(o)||o.charAt(0)==="_")continue;let a=Object.getOwnPropertyDescriptor(e,o);if(!a||typeof a.value!="function")continue;let h=a.value;n[o]=function(...p){t();let w=h.call(i,...p);return w===i?n:w}}e=Object.getPrototypeOf(e)}}function v(n){let i={__hfReal:n,__hfQueue:[],__hfIsProxy:!0,to(...e){return r(i,"to",e)},from(...e){return r(i,"from",e)},fromTo(...e){return r(i,"fromTo",e)},set(...e){return r(i,"set",e)},add(...e){return r(i,"add",e)},pause(...e){return t(),n.pause(...e),i},play(...e){return t(),n.play(...e),i},seek(...e){return t(),n.seek(...e),i},totalTime(...e){return t(),e.length>0?(n.totalTime(...e),i):n.totalTime()},time(...e){return t(),e.length>0?(n.time(...e),i):n.time()},duration(...e){return t(),e.length>0?(n.duration(...e),i):n.duration()},getChildren(...e){t();let o=n.getChildren(...e);return Array.isArray(o)?o:[]},paused(...e){return t(),e.length>0?(n.paused(...e),i):n.paused()},timeScale(...e){return t(),e.length>0?(n.timeScale(...e),i):n.timeScale()},kill(){t(),n.kill()}};return b(i,n),_.push(i),i}if(typeof window<"u"){window.__hf||(window.__hf={}),window.__hfTimelinesBuilding=!1,window.__hfFlushSync=()=>{t(),u.length===0&&window.__hfTimelinesBuilding&&f()};let n=null;try{Object.defineProperty(window,"gsap",{configurable:!0,enumerable:!0,get(){return n},set(i){if(n=i,!i||typeof i.timeline!="function")return;let e=i.timeline.bind(i);i.timeline=o=>v(e(o))}})}catch{}}})();\n';
'"use strict";(()=>{var b=100,O=[],u=[],w=!1,f=!1,c=new Set,p=new Set;function y(n){let e=window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;return typeof e=="function"?e(n):requestAnimationFrame(n)}function x(n){let e=window.__HF_VIRTUAL_TIME__?.originalSetTimeout;if(typeof e=="function"){e(n,0);return}setTimeout(n,0)}function S(n){return n!==null&&typeof n=="object"&&"__hfIsProxy"in n?n.__hfReal:n}function _(n){if(n===null||typeof n!="object"||Array.isArray(n))return n;let e=n;if(!("opacity"in e)||"autoAlpha"in e||"visibility"in e)return n;let o={};for(let t of Object.keys(e))t==="opacity"?o.autoAlpha=e[t]:o[t]=e[t];return o}function T(n,e){if(n!=="add"&&v(e),window.__HF_FAST_CAPTURE_AUTOALPHA__!==!0||n==="add")return e;let o=e.slice(),t=!1;if(o.length>1){let i=_(o[1]);i!==o[1]&&(t=!0),o[1]=i}if(n==="fromTo"&&o.length>2){let i=_(o[2]);i!==o[2]&&(t=!0),o[2]=i}return t&&o[0]!==null&&o[0]!==void 0&&c.add(o[0]),o}function h(n){if(n===null||typeof n!="object"||Array.isArray(n))return!1;let e=n;return"rotationX"in e||"rotationY"in e||"transformPerspective"in e}var m=new Set;function v(n){let e=n[0];if(e==null)return;let o=window;m.has(e)||(m.add(e),o.__hfAllTweenTargets=Array.from(m)),(h(n[1])||h(n[2]))&&(p.add(e),o.__hf3dTweenTargets=Array.from(p))}function R(n){if(typeof n=="string")try{return Array.from(document.querySelectorAll(n))}catch{return[]}if(n instanceof Element)return[n];if(Array.isArray(n)||typeof NodeList<"u"&&n instanceof NodeList){let e=[];for(let o of n)o instanceof Element&&e.push(o);return e}return[]}function F(){if(window.__HF_FAST_CAPTURE_AUTOALPHA__!==!0||c.size===0)return;let n=Array.from(c);c.clear();for(let e of n)for(let o of R(e)){let t=o;if(!t.style||t.style.visibility!=="")continue;let i="";try{i=getComputedStyle(o).opacity}catch{continue}i==="0"&&(t.style.visibility="hidden")}}function g(n){let e=n.proxy.__hfReal,o=e[n.method];if(typeof o=="function"){let t=n.method==="add"?n.args.map(S):n.args;o.call(e,...t)}}function l(n,e,o){let t={proxy:n,method:e,args:T(e,o)};return n.__hfQueue.push(t),u.push(t),I(),n}function A(n){let e=n.proxy.__hfQueue.indexOf(n);e>=0&&n.proxy.__hfQueue.splice(e,1)}function r(){for(;u.length>0;){let n=u.shift();n&&(A(n),g(n))}E()}function k(){f=!1,F(),window.__hfTimelinesBuilding=!1;try{window.dispatchEvent(new CustomEvent("hf-timelines-built"))}catch{}}function E(){f||(f=!0,x(()=>{u.length===0?k():f=!1}))}function P(){w=!1;let n=u.splice(0,b);for(let e of n)A(e),g(e);u.length>0?(w=!0,y(P)):k()}function I(){w||(w=!0,window.__hfTimelinesBuilding=!0,y(P))}var C=new Set(["to","from","fromTo","set","add"]);function G(n,e){let o=e;for(;o!==null&&o!==Object.prototype;){for(let t of Object.getOwnPropertyNames(o)){if(t==="constructor"||t==="then"||t in n||C.has(t)||t.charAt(0)==="_")continue;let i=Object.getOwnPropertyDescriptor(o,t);if(!i||typeof i.value!="function")continue;let s=i.value;n[t]=function(...d){r();let a=s.call(e,...d);return a===e?n:a}}o=Object.getPrototypeOf(o)}}function H(n){let e={__hfReal:n,__hfQueue:[],__hfIsProxy:!0,to(...o){return l(e,"to",o)},from(...o){return l(e,"from",o)},fromTo(...o){return l(e,"fromTo",o)},set(...o){return l(e,"set",o)},add(...o){return l(e,"add",o)},pause(...o){return r(),n.pause(...o),e},play(...o){return r(),n.play(...o),e},seek(...o){return r(),n.seek(...o),e},totalTime(...o){return r(),o.length>0?(n.totalTime(...o),e):n.totalTime()},time(...o){return r(),o.length>0?(n.time(...o),e):n.time()},duration(...o){return r(),o.length>0?(n.duration(...o),e):n.duration()},getChildren(...o){r();let t=n.getChildren(...o);return Array.isArray(t)?t:[]},paused(...o){return r(),o.length>0?(n.paused(...o),e):n.paused()},timeScale(...o){return r(),o.length>0?(n.timeScale(...o),e):n.timeScale()},kill(){r(),n.kill()}};return G(e,n),O.push(e),e}if(typeof window<"u"){window.__hf||(window.__hf={}),window.__hfTimelinesBuilding=!1,window.__hfFlushSync=()=>{r(),u.length===0&&window.__hfTimelinesBuilding&&k()};let n=null;try{Object.defineProperty(window,"gsap",{configurable:!0,enumerable:!0,get(){return n},set(e){if(n=e,!e||typeof e.timeline!="function")return;let o=e.timeline.bind(e);e.timeline=i=>H(o(i));for(let i of["to","from","set"]){let s=e[i];if(typeof s!="function")continue;let d=s.bind(e);e[i]=(...a)=>d(...T(i,a))}let t=e.fromTo;if(typeof t=="function"){let i=t.bind(e);e.fromTo=(...s)=>i(...T("fromTo",s))}}})}catch{}}})();\n';
/**
* Returns the pre-built HyperFrames early stub IIFE as a string constant.
+35 -11
View File
@@ -113,6 +113,13 @@ type TestMetadata = {
workers?: number; // Optional: auto-calculates if omitted
/** Force HDR in the harness; omitted/false preserves historical SDR-only test behavior. */
hdr?: boolean;
/**
* Render this suite with the experimental fast-capture path
* (drawElementImage, `--experimental-fast-capture`). The golden must be
* regenerated with the flag on. Used by the `fast-capture` regression
* guard; omit for the default screenshot/BeginFrame capture.
*/
experimentalFastCapture?: boolean;
/**
* Render-time variable overrides, equivalent to `hyperframes render
* --variables '<json>'`. Injected as `window.__hfVariables` before any
@@ -374,6 +381,11 @@ function validateMetadata(meta: unknown): TestMetadata {
if (rc.hdr !== undefined && typeof rc.hdr !== "boolean") {
throw new Error("meta.json: 'renderConfig.hdr' must be a boolean (or omit for false)");
}
if (rc.experimentalFastCapture !== undefined && typeof rc.experimentalFastCapture !== "boolean") {
throw new Error(
"meta.json: 'renderConfig.experimentalFastCapture' must be a boolean (or omit for false)",
);
}
if (
rc.variables !== undefined &&
(rc.variables === null || typeof rc.variables !== "object" || Array.isArray(rc.variables))
@@ -1017,18 +1029,30 @@ async function runTestSuite(
await runDistributedSimulatedRender(distributedInput);
}
} else {
const job = createRenderJob({
fps: suite.meta.renderConfig.fps,
quality: "high", // Always use max quality for tests
format: outputFormat,
workers: suite.meta.renderConfig.workers,
useGpu: false,
debug: false,
hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr",
variables: suite.meta.renderConfig.variables,
});
// Opt-in fast capture (drawElementImage): drives resolveConfig via the env
// var, scoped to this suite's render so it never leaks to other suites.
const useFast = suite.meta.renderConfig.experimentalFastCapture === true;
const prevFast = process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
if (useFast) process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = "true";
try {
const job = createRenderJob({
fps: suite.meta.renderConfig.fps,
quality: "high", // Always use max quality for tests
format: outputFormat,
workers: suite.meta.renderConfig.workers,
useGpu: false,
debug: false,
hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr",
variables: suite.meta.renderConfig.variables,
});
await executeRenderJob(job, tempSrcDir, renderedOutputPath);
await executeRenderJob(job, tempSrcDir, renderedOutputPath);
} finally {
if (useFast) {
if (prevFast === undefined) delete process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
else process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = prevFast;
}
}
}
console.log(JSON.stringify({ event: "rendering_complete", suite: suite.id }));
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication
import { describe, expect, it, mock, beforeAll } from "bun:test";
import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -8,6 +9,7 @@ import {
compileForRender,
detectRenderModeHints,
detectShaderTransitionUsage,
detectThreeDTransformUsage,
discoverAudioVolumeAutomationFromTimeline,
inlineExternalScripts,
localizeRemoteMediaSources,
@@ -649,6 +651,50 @@ describe("detectRenderModeHints", () => {
});
});
describe("detectThreeDTransformUsage", () => {
it("detects CSS perspective property", () => {
expect(detectThreeDTransformUsage("<style>.s { perspective: 1000px; }</style>")).toBe(true);
});
it("detects transform-style preserve-3d", () => {
expect(detectThreeDTransformUsage("<style>.c { transform-style: preserve-3d; }</style>")).toBe(
true,
);
});
it("detects backface-visibility", () => {
expect(detectThreeDTransformUsage("<style>.f { backface-visibility: hidden; }</style>")).toBe(
true,
);
});
it("detects perspective() transform function", () => {
expect(detectThreeDTransformUsage('<div style="transform: perspective(500px)"></div>')).toBe(
true,
);
});
it("detects GSAP transformPerspective", () => {
expect(
detectThreeDTransformUsage("<script>gsap.to(el, { transformPerspective: 800 })</script>"),
).toBe(true);
});
it("does not match flat GSAP rotationX without a perspective context", () => {
expect(detectThreeDTransformUsage("<script>gsap.to(el, { rotationX: 180 })</script>")).toBe(
false,
);
});
it("does not match translateZ(0) promotion hack", () => {
expect(detectThreeDTransformUsage('<div style="transform: translateZ(0)"></div>')).toBe(false);
});
it("does not match perspective: none", () => {
expect(detectThreeDTransformUsage("<style>.s { perspective: none; }</style>")).toBe(false);
});
});
describe("detectShaderTransitionUsage", () => {
it("detects authored HyperShader initialization", () => {
const html = `<!doctype html>
+92 -1
View File
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication complexity
/**
* HTML Compiler for Producer
*
@@ -67,6 +68,10 @@ export interface CompiledComposition {
staticDuration: number;
renderModeHints: RenderModeHints;
hasShaderTransitions: boolean;
/** Author HTML/CSS/scripts use a CSS 3D rendering context (pre-CDN-inline scan). */
usesThreeDTransforms: boolean;
/** Author HTML/CSS use mix-blend-mode (pre-CDN-inline scan). */
usesMixBlendMode: boolean;
}
/** Adapts linkedom's `parseHTML` to the `checkSubCompositionUsability` contract. */
@@ -273,6 +278,35 @@ export function detectRenderModeHints(html: string): RenderModeHints {
};
}
/**
* 3D rendering-context signals. drawElementImage paints elements inside a
* CSS 3D rendering context incorrectly: backface-visibility:hidden is
* ignored (mid-flip elements show their mirrored backface), sibling content
* of the 3D context can drop out of the capture, and the context's
* background is lost. Observed on real-world gen_os comps (flip-card and
* rotationX scene-entrance patterns) on macOS hardware GPU — this is a
* drawElementImage limitation, not a SwiftShader artifact.
*
* Only genuine 3D-context signals are matched: `perspective` (property or
* transform function), `transform-style: preserve-3d`, `backface-visibility`,
* `matrix3d(` / `rotate3d(`, and GSAP's `transformPerspective`. Flat
* rotationX/Y tweens without a perspective context render as 2D and are
* deliberately NOT matched, nor is the ubiquitous `translateZ(0)` promotion
* hack.
*/
const THREE_D_CONTEXT_PATTERN =
/transform-style\s*:\s*preserve-3d|backface-visibility\s*:|perspective\s*:\s*[0-9]|perspective\s*\(|matrix3d\s*\(|rotate3d\s*\(|\btransformPerspective\b/i;
export function detectThreeDTransformUsage(html: string): boolean {
return THREE_D_CONTEXT_PATTERN.test(html);
}
const MIX_BLEND_MODE_PATTERN = /mix-blend-mode\s*:/i;
function detectMixBlendModeUsage(html: string): boolean {
return MIX_BLEND_MODE_PATTERN.test(html);
}
const SHADER_TRANSITION_USAGE_PATTERN =
/\b(?:(?:window|globalThis)\s*\.\s*)?HyperShader\s*\.\s*init\s*\(|\b__hf\s*\.\s*transitions\s*=/;
@@ -1176,6 +1210,47 @@ export async function localizeRemoteImageSources(
);
}
// Match a remote url() inside a `background` / `background-image` CSS declaration
// (style blocks or inline style attrs). `[^;}"']*?` lets position/color tokens
// precede the url() in the shorthand while stopping at the declaration boundary.
const REMOTE_BG_URL_RE =
/background(?:-image)?\s*:\s*[^;}"']*?url\(\s*["']?(https?:\/\/[^"')]+)["']?\s*\)/gi;
/**
* Download remote CSS `background-image: url(https://...)` references and rewrite
* them to local same-origin paths.
*
* Why: `drawElementImage` (fast capture) OMITS cross-origin content, so a remote
* background image renders BLACK on the drawElement path while the screenshot
* baseline captures it (origin-agnostic) — a whole-region mismatch (e.g. 10f79c0b
* picsum.photos backgrounds, 9.3 dB). `<img>`/`<video>`/`@font-face` are localized
* by their own passes; this closes the background-image gap so the fast path sees
* the same pixels as the baseline.
*
* @internal exported for unit testing only
*/
export async function localizeRemoteBackgroundImages(
html: string,
downloadDir: string,
): Promise<{ html: string; remoteMediaAssets: Map<string, string> }> {
const urlSet = new Set<string>();
const re = new RegExp(REMOTE_BG_URL_RE.source, REMOTE_BG_URL_RE.flags);
let m: RegExpExecArray | null;
while ((m = re.exec(html)) !== null) {
if (m[1] && !isGoogleFontsUrl(m[1])) urlSet.add(m[1]);
}
return downloadAndRewriteUrls(
urlSet,
html,
join(downloadDir, REMOTE_MEDIA_SUBDIR),
"Remote background-image download failed for",
"Localized remote background-image(s)",
// Quoted url('..')/url("..") are rewritten by downloadAndRewriteUrls' default
// replaceAll; this handles the unquoted url(https://..) form.
(h, url, rel) => h.replaceAll(`url(${url})`, `url(${rel})`),
);
}
// Match url("https://...") or url('https://...') inside @font-face blocks.
// We scan the full HTML (which includes <style> blocks) — matching against
// @font-face context precisely would require a CSS parser; instead we match
@@ -1548,6 +1623,11 @@ export async function compileForRender(
);
const renderModeHints = detectRenderModeHints(sanitizedHtml);
const hasShaderTransitions = detectShaderTransitionUsage(sanitizedHtml);
// Detected BEFORE inlineExternalScripts: GSAP's own source contains
// `transformPerspective`, so scanning post-inline HTML would flag every
// composition that loads GSAP from a CDN.
const usesThreeDTransforms = detectThreeDTransformUsage(sanitizedHtml);
const usesMixBlendMode = detectMixBlendModeUsage(sanitizedHtml);
const normalizedFontHtml = normalizeSystemFontPrimaryFamilies(
injectTextRenderingRule(
@@ -1602,12 +1682,18 @@ export async function compileForRender(
const { html: htmlWithLocalImages, remoteMediaAssets: remoteImageAssets } =
await localizeRemoteImageSources(htmlWithLocalMedia, downloadDir);
// Download remote CSS background-image url() references. drawElementImage omits
// cross-origin content, so remote backgrounds render black on the fast path;
// localising them to same-origin closes that gap.
const { html: htmlWithLocalBg, remoteMediaAssets: remoteBgAssets } =
await localizeRemoteBackgroundImages(htmlWithLocalImages, downloadDir);
// Download remote @font-face src URLs and rewrite to local paths.
// Remote font URLs fail with a CORS rejection at render time (S3 does not
// allow http://localhost:PORT as origin), causing Chrome to silently fall
// back to the next font in the stack.
const { html: htmlWithLocalizedFonts, remoteMediaAssets: remoteFontAssets } =
await localizeRemoteFontFaces(htmlWithLocalImages, downloadDir);
await localizeRemoteFontFaces(htmlWithLocalBg, downloadDir);
const gifSourceAssets = new Map<string, string>(remoteImageAssets);
const {
@@ -1638,6 +1724,9 @@ export async function compileForRender(
for (const [relPath, absPath] of remoteImageAssets) {
externalAssets.set(relPath, absPath);
}
for (const [relPath, absPath] of remoteBgAssets) {
externalAssets.set(relPath, absPath);
}
for (const [relPath, absPath] of remoteFontAssets) {
externalAssets.set(relPath, absPath);
}
@@ -1711,6 +1800,8 @@ export async function compileForRender(
staticDuration,
renderModeHints,
hasShaderTransitions,
usesThreeDTransforms,
usesMixBlendMode,
};
}
@@ -43,6 +43,8 @@ import {
type CaptureSession,
type EngineConfig,
captureFrame,
captureFrameToBufferPipelined,
writeCapturedFrame,
closeCaptureSession,
createCaptureSession,
getCapturePerfSummary,
@@ -271,16 +273,8 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
const rangeEnd = frameRange?.endFrame ?? totalFrames;
const rangeFrames = rangeEnd - rangeStart;
for (let i = 0; i < rangeFrames; i++) {
assertNotAborted();
const absoluteIdx = rangeStart + i;
const time = (absoluteIdx * job.config.fps.den) / job.config.fps.num;
await captureFrame(session, i, time);
job.framesRendered = i + 1;
const frameProgress = (i + 1) / rangeFrames;
const progress = 25 + frameProgress * 45;
const reportFrame = (fileIndex: number): void => {
job.framesRendered = fileIndex + 1;
// Keep status cadence identical to the streaming sequential path; the
// capture error wrapper below must remain separate from finally so it
// can throw with the browser console before cleanup overwrites flow.
@@ -288,10 +282,43 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
updateJobStatus(
job,
"rendering",
`Capturing frame ${i + 1}/${rangeFrames}`,
Math.round(progress),
`Capturing frame ${fileIndex + 1}/${rangeFrames}`,
Math.round(25 + ((fileIndex + 1) / rangeFrames) * 45),
onProgress,
);
};
if (session.workerEncodeEnabled) {
// Worker-encode depth-2 pipeline on the DISK path (mirrors the streaming
// path): frame N's in-page Worker encodes while frame N+1's main thread
// does seek+paint+drawElement. Long comps (>streaming cap) land here, so
// without this they'd fall back to synchronous toDataURL and lose the
// ~1.5-2x worker-encode speedup entirely.
let prev: { fileIndex: number; encodeResult: Promise<Buffer> } | null = null;
const drainPrev = async (): Promise<void> => {
if (!prev) return;
assertNotAborted();
const buf = await prev.encodeResult;
writeCapturedFrame(session, prev.fileIndex, buf);
reportFrame(prev.fileIndex);
};
for (let i = 0; i < rangeFrames; i++) {
assertNotAborted();
const absoluteIdx = rangeStart + i;
const time = (absoluteIdx * job.config.fps.den) / job.config.fps.num;
const { encodeResult } = await captureFrameToBufferPipelined(session, i, time);
await drainPrev();
prev = { fileIndex: i, encodeResult };
}
await drainPrev();
} else {
for (let i = 0; i < rangeFrames; i++) {
assertNotAborted();
const absoluteIdx = rangeStart + i;
const time = (absoluteIdx * job.config.fps.den) / job.config.fps.num;
await captureFrame(session, i, time);
reportFrame(i);
}
}
// Capture the sequential session's static-dedup perf before close (the
// counters are valid only while the session is live).
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication complexity
/**
* captureStreamingStage single-machine fused capture + encode path.
*
@@ -48,6 +49,7 @@ import {
type EngineConfig,
type StreamingEncoder,
captureFrameToBuffer,
captureFrameToBufferPipelined,
closeCaptureSession,
createCaptureSession,
createFrameReorderBuffer,
@@ -134,6 +136,52 @@ export type CaptureStreamingStageResult =
success: false;
};
async function runWorkerEncodePipelineLoop(
session: CaptureSession,
totalFrames: number,
job: CaptureStreamingStageInput["job"],
currentEncoder: StreamingEncoder,
reorderBuffer: ReturnType<typeof createFrameReorderBuffer>,
assertNotAborted: () => void,
onProgress: CaptureStreamingStageInput["onProgress"],
): Promise<void> {
let prev: { idx: number; encodeResult: Promise<Buffer> } | null = null;
const drainPrev = async (): Promise<void> => {
if (!prev) return;
// Observe aborts while parked here (the encode wait + ffmpeg write are the
// longest stretch of the loop); without this an abort isn't seen until the
// next produce iteration.
assertNotAborted();
const buf = await prev.encodeResult;
await reorderBuffer.waitForFrame(prev.idx);
ensureFrameWritten(await currentEncoder.writeFrame(buf), prev.idx, currentEncoder);
reorderBuffer.advanceTo(prev.idx + 1);
job.framesRendered = prev.idx + 1;
updateJobStatus(
job,
"rendering",
`Streaming frame ${prev.idx + 1}/${totalFrames}`,
Math.round(25 + ((prev.idx + 1) / totalFrames) * 55),
onProgress,
);
};
// On abort/throw the just-produced frame's encode is still in flight and never
// awaited (it isn't `prev` yet); cleanupDrawElementWorkerEncode rejects it on
// close. produceDrawElementFrame attaches a no-op catch to every encodeResult
// at creation so that orphaned rejection is never an unhandled rejection — so
// the loop needs no special guard here.
for (let i = 0; i < totalFrames; i++) {
assertNotAborted();
const time = (i * job.config.fps.den) / job.config.fps.num;
const { encodeResult } = await captureFrameToBufferPipelined(session, i, time);
await drainPrev();
prev = { idx: i, encodeResult };
}
await drainPrev();
}
export async function runCaptureStreamingStage(
input: CaptureStreamingStageInput,
): Promise<CaptureStreamingStageResult> {
@@ -275,29 +323,43 @@ export async function runCaptureStreamingStage(
assertNotAborted();
lastBrowserConsole = session.browserConsoleBuffer;
for (let i = 0; i < totalFrames; i++) {
assertNotAborted();
const time = (i * job.config.fps.den) / job.config.fps.num;
const { buffer } = await captureFrameToBuffer(session, i, time);
await reorderBuffer.waitForFrame(i);
ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
reorderBuffer.advanceTo(i + 1);
job.framesRendered = i + 1;
const frameProgress = (i + 1) / totalFrames;
const progress = 25 + frameProgress * 55;
// Keep status cadence identical to disk sequential capture; the
// capture error wrapper below must remain separate from finally so it
// can throw with the browser console before encoder cleanup runs.
// fallow-ignore-next-line code-duplication
updateJobStatus(
if (session.workerEncodeEnabled) {
// Worker-encode pipeline: depth-2. Frame N's in-page Worker encodes
// while frame N+1's main thread does seek+paint+drawElement+kick.
await runWorkerEncodePipelineLoop(
session,
totalFrames,
job,
"rendering",
`Streaming frame ${i + 1}/${totalFrames}`,
Math.round(progress),
currentEncoder,
reorderBuffer,
assertNotAborted,
onProgress,
);
} else {
for (let i = 0; i < totalFrames; i++) {
assertNotAborted();
const time = (i * job.config.fps.den) / job.config.fps.num;
const { buffer } = await captureFrameToBuffer(session, i, time);
await reorderBuffer.waitForFrame(i);
ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
reorderBuffer.advanceTo(i + 1);
job.framesRendered = i + 1;
const frameProgress = (i + 1) / totalFrames;
const progress = 25 + frameProgress * 55;
// Keep status cadence identical to disk sequential capture; the
// capture error wrapper below must remain separate from finally so it
// can throw with the browser console before encoder cleanup runs.
// fallow-ignore-next-line code-duplication
updateJobStatus(
job,
"rendering",
`Streaming frame ${i + 1}/${totalFrames}`,
Math.round(progress),
onProgress,
);
}
}
// Capture the session's static-dedup perf before close (counters valid
// only while the session is live).
@@ -137,8 +137,63 @@ export async function runCompileStage(input: CompileStageInput): Promise<Compile
// composition's `renderModeHints.recommendScreenshot`. The single
// write to `cfg.forceScreenshot` happens at the end of this block so
// the contract is enforceable by inspection.
const callerForced = cfg.forceScreenshot || needsAlpha;
const { forceScreenshot } = applyRenderModeHints(callerForced, compiled, log);
// Alpha output forces screenshot because BeginFrame doesn't preserve alpha —
// but drawElement fast capture self-manages alpha (screenshot-launched
// browser + png drawElementImage, pixel-perfect; see createCaptureSession).
// Folding needsAlpha here would make the engine's forceScreenshot guard
// disable fast capture for every transparent render, so skip the fold when
// fast capture is on. Render-mode hints (e.g. raw requestAnimationFrame)
// still force screenshot below — those are correctness routings that
// drawElement must honor.
// The fast-capture video gate was REMOVED here once Chrome 151 fixed crbug
// 521861819. It keyed on compiled.videos.length > 0 as a proxy for the
// word-by-word caption opacity pattern (drawElementImage dropped the promoted
// opacity layers mid-fade, ~12 dB). On the 151 pinned floor, video + nested-fade
// comps render correctly on the drawElement path (verified PSNR=inf vs baseline);
// see docs/fast-capture-limitations.md Lim 2.
// Fast-capture 3D-transform gate, same shape as the video gate above.
// drawElementImage paints CSS 3D rendering contexts incorrectly:
// backface-visibility:hidden is ignored (mid-flip elements capture their
// mirrored backface), siblings of the 3D context can drop out, and the
// context's background is lost. Reproduced on macOS hardware GPU with
// real-world flip-card / rotationX-entrance comps (full-stream PSNR
// 2746 dB avg, 17 dB min vs baseline). Routes to the platform's baseline
// capture; HF_FAST_CAPTURE_3D=true bypasses for R&D.
// Detection runs inside the compiler on PRE-CDN-inline HTML — GSAP's own
// source contains `transformPerspective`, so scanning compiled.html here
// would flag every composition that loads GSAP.
if (
cfg.useDrawElement &&
process.env.HF_FAST_CAPTURE_3D !== "true" &&
compiled.usesThreeDTransforms
) {
cfg.useDrawElement = false;
log.info(
"[Render] Fast capture: composition uses a CSS 3D rendering context " +
"(perspective / preserve-3d / backface-visibility) — disabling drawElementImage " +
"for this render. Capture uses the platform's baseline route.",
);
}
// Fast-capture mix-blend-mode gate, same shape as the 3D gate above.
// drawElementImage captures each element's paint records before the
// compositor resolves blend equations — blended layers render as if
// mix-blend-mode were absent, producing saturated/damaged composites
// (measured: 42 dB min vs 53 dB floor on real blend+filter comps, macOS GPU).
// HF_FAST_CAPTURE_BLEND=true bypasses for R&D.
if (
cfg.useDrawElement &&
process.env.HF_FAST_CAPTURE_BLEND !== "true" &&
compiled.usesMixBlendMode
) {
cfg.useDrawElement = false;
log.info(
"[Render] Fast capture: composition uses mix-blend-mode — disabling drawElementImage " +
"for this render. Capture uses the platform's baseline route.",
);
}
const callerForced = cfg.forceScreenshot || (needsAlpha && !cfg.useDrawElement);
const { forceScreenshot: hintForced } = applyRenderModeHints(callerForced, compiled, log);
let forceScreenshot = hintForced;
cfg.forceScreenshot = forceScreenshot;
writeCompiledArtifacts(compiled, workDir, Boolean(job.config.debug));
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication complexity
/**
* probeStage browser probe + recompile + media reconciliation.
*
@@ -38,6 +39,7 @@ import {
getCompositionDuration,
initializeSession,
isTransientBrowserError,
probeBeginFrameLiveness,
} from "@hyperframes/engine";
import { fpsToNumber } from "@hyperframes/core";
import type { CompiledComposition } from "../../htmlCompiler.js";
@@ -95,6 +97,14 @@ export interface ProbeStageResult {
totalFrames: number;
/** Wall-clock ms for the entire probe phase (near-zero when `needsBrowser` was false). */
browserProbeMs: number;
/**
* True when the BeginFrame liveness probe timed out on this host (SwiftShader
* stalls the first BeginFrame indefinitely for heavy-layer compositions
* style-N caption comps). The probe session has already been relaunched in
* screenshot mode; the sequencer must flip its `captureForceScreenshot`
* local so downstream capture stages follow.
*/
beginFrameStalled: boolean;
}
export function hasScriptedAudioVolumeAutomation(html: string, audioCount: number): boolean {
@@ -147,6 +157,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
let fileServer: FileServerHandle | null = null;
let probeSession: CaptureSession | null = null;
let beginFrameStalled = false;
let lastBrowserConsole: string[] = [];
const probeStart = Date.now();
@@ -258,6 +269,65 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
probeSession = session;
lastBrowserConsole = session.browserConsoleBuffer;
// BeginFrame liveness probe. On SwiftShader, heavy-layer compositions
// (multi-group nested opacity caption animations — style-N prod comps)
// stall the FIRST BeginFrame indefinitely (tested to 30 min). The
// auto-worker calibration catches this via its capped protocol timeout,
// but renders with explicit `--workers N` skip calibration and would
// hang for the full protocol timeout. One bounded BeginFrame here gives
// ground truth for every render that probes a browser: on stall,
// relaunch the probe session in screenshot mode and tell the sequencer
// (via `beginFrameStalled`) to route the whole render through
// screenshot capture — the path the baseline already uses for these
// comps. Healthy comps pay one extra composited frame (<1s on GPU, a
// few seconds on SwiftShader).
if (probeSession.launchCaptureMode === "beginframe") {
const probeTimeoutMs =
Number(process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS) > 0
? Number(process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS)
: 30_000;
const livenessStart = Date.now();
// Tick inside the post-warmup cushion: warmup < probe < first capture
// keeps the session's BeginFrame frameTimeTicks monotonic.
const probeTick = Math.max(
0,
probeSession.beginFrameTimeTicks - 5 * probeSession.beginFrameIntervalMs,
);
const alive = await probeBeginFrameLiveness(
probeSession.page,
probeTimeoutMs,
probeTick,
probeSession.beginFrameIntervalMs,
);
assertNotAborted();
if (alive) {
log.info("BeginFrame liveness probe passed", {
probeMs: Date.now() - livenessStart,
});
} else {
beginFrameStalled = true;
log.warn(
"[Render] BeginFrame liveness probe timed out — this composition stalls " +
"BeginFrame on this host (SwiftShader heavy-layer pattern). Relaunching " +
"the probe browser in screenshot capture mode; the render will use " +
"screenshot capture throughout.",
{ probeTimeoutMs },
);
lastBrowserConsole = probeSession.browserConsoleBuffer;
await closeCaptureSession(probeSession).catch(() => {});
probeSession = await createCaptureSession(
fileServer.url,
join(workDir, "probe-screenshot"),
captureOpts,
null,
{ ...probeCfg, forceScreenshot: true },
);
await initializeSession(probeSession);
assertNotAborted();
lastBrowserConsole = probeSession.browserConsoleBuffer;
}
}
// Discover root composition duration
if (composition.duration <= 0) {
log.info("Discovering composition duration...");
@@ -315,6 +385,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
}
if (el.tagName === "video") {
// fallow-ignore-next-line code-duplication
if (existingVideoIds.has(el.id)) {
// Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
const existing = composition.videos.find((v) => v.id === el.id);
@@ -361,6 +432,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
existingVideoIds.add(el.id);
}
} else if (el.tagName === "audio") {
// fallow-ignore-next-line code-duplication
if (existingAudioIds.has(el.id)) {
const existing = composition.audios.find((a) => a.id === el.id);
if (existing) {
@@ -535,5 +607,6 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
duration,
totalFrames,
browserProbeMs,
beginFrameStalled,
};
}
@@ -1266,10 +1266,19 @@ export async function executeRenderJob(
perfStages.browserProbeMs = probeResult.browserProbeMs;
perfStages.compileMs = Date.now() - stage1Start;
// BeginFrame liveness: the probe stage already relaunched its session in
// screenshot mode when the first BeginFrame stalled (SwiftShader
// heavy-layer comps) — flip the sequencer's capture routing to match so
// calibration and capture stages never issue another BeginFrame.
if (probeResult.beginFrameStalled && !captureForceScreenshot) {
captureForceScreenshot = true;
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
}
observability.checkpoint("browser_probe", "duration resolved", {
durationSeconds: probeResult.duration,
totalFrames,
compositionHash,
beginFrameStalled: probeResult.beginFrameStalled,
});
// ── Stage 2: Video frame extraction ─────────────────────────────────