mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
feat(producer): fast-capture render stages + remote bg-image localizer (#1920)
* feat(engine): drawElementImage capture service * feat(engine): 3D projection + compositor-effect risk gate * fix(engine): gate filter drop-shadow wherever blur gates (review) detectCssEffectRisk documented drop-shadow as a ~29dB damage case but only detected blur( in its three scan paths — a drop-shadow comp stayed on the fast path despite the gate's own correctness contract. Detect drop-shadow( in computed styles, stylesheet rules, and GSAP tween vars, pinned by a focused test that runs the real page-side closure against a DOM shim (computed / stylesheet / tween coverage + blur regression + effect-free null). Addresses miguel-heygen's blocker on #1918. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(engine): frame-capture core — fast-capture routing, worker-encode, dedup extension # Conflicts: # packages/engine/src/services/screenshotService.ts * fix(engine): document HF_FORCE_DRAWELEMENT as diagnostic-only; make armStaticDedup idempotent (review) Addresses miguel-heygen's blockers on #1919: - HF_FORCE_DRAWELEMENT promoted from a stale "SCRATCH/Uncommitted" comment to a documented diagnostic flag: it exists for upstream-Chromium repro work (gate-vs-API isolation, crbug 521861819 149-vs-151) and R&D on gated effect classes; renders under it may be damaged BY DESIGN since it bypasses gates whose thresholds encode measured damage. Never production; the safety-net blank guard also stands down under it so diagnostic frames arrive unmodified. - armStaticDedup is now idempotent: the drawElement init path arms dedup before canvas injection, then initializeSession called it again — the second run overwrote the armed state with skipReason="capture_mode" (captureMode is "drawelement" by then), producing contradictory telemetry (armed frames + a skip reason), and re-ran the verification seeks on the fallback path. It now no-ops once staticFrames or a skip decision exists. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(producer): fast-capture render stages + remote bg-image localizer --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
0e58344dca
commit
1d0dbcd3b2
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Validate the fast-capture (drawElementImage) VIDEO path on real Linux.
|
||||
*
|
||||
* drawElementImage draws a snapshot taken at the paint event; capturing video
|
||||
* needs a fresh per-frame paint. On Linux headless-shell that paint comes from
|
||||
* the per-frame HeadlessExperimental.beginFrame — so video should capture
|
||||
* correctly there (see docs/fast-capture-limitations.md, Limitation 2). This
|
||||
* could not be validated under Docker-on-rosetta (renders hung); this script is
|
||||
* meant to run on a native amd64 Linux runner inside Dockerfile.test.
|
||||
*
|
||||
* Renders a video composition twice — baseline (screenshot) and fast
|
||||
* (drawElement) — and asserts the fast output matches the baseline (PSNR above
|
||||
* threshold), proving the video was captured and not dropped to black.
|
||||
*
|
||||
* PRODUCER_VALIDATE_COMP=sub-composition-video \
|
||||
* bunx tsx scripts/validate-fast-video.ts
|
||||
*
|
||||
* Exit 0 = fast video matches baseline; exit 1 = regression (black/stale video).
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { createRenderJob, executeRenderJob } from "../src/index.js";
|
||||
|
||||
// `||` not `??` — the workflow passes empty strings on a push trigger (inputs
|
||||
// are only populated for workflow_dispatch), and "" must fall through to the default.
|
||||
const COMP = process.env.PRODUCER_VALIDATE_COMP || "sub-composition-video";
|
||||
const MIN_PSNR = Number.parseFloat(process.env.PRODUCER_VALIDATE_MIN_PSNR || "25");
|
||||
const work = mkdtempSync(join(tmpdir(), "fastvideo-"));
|
||||
|
||||
process.env.PRODUCER_ENABLE_BROWSER_POOL = "false";
|
||||
|
||||
async function render(mode: "baseline" | "fast", out: string): Promise<void> {
|
||||
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = mode === "fast" ? "true" : "false";
|
||||
const job = createRenderJob({
|
||||
fps: 30,
|
||||
quality: "high",
|
||||
format: "mp4",
|
||||
workers: 1,
|
||||
useGpu: false,
|
||||
hdrMode: "force-sdr",
|
||||
});
|
||||
await executeRenderJob(job, resolve("tests", COMP, "src"), out);
|
||||
}
|
||||
|
||||
function psnr(a: string, b: string): number {
|
||||
const out = execFileSync(
|
||||
"bash",
|
||||
["-c", `ffmpeg -y -i "${a}" -i "${b}" -lavfi psnr -f null - 2>&1`],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
const m = out.match(/average:(\S+)/);
|
||||
if (!m) throw new Error(`ffmpeg psnr produced no average:\n${out}`);
|
||||
return m[1] === "inf" ? Number.POSITIVE_INFINITY : Number.parseFloat(m[1]);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const baseline = join(work, "baseline.mp4");
|
||||
const fast = join(work, "fast.mp4");
|
||||
console.log(`[validate-fast-video] comp=${COMP} minPsnr=${MIN_PSNR}`);
|
||||
await render("baseline", baseline);
|
||||
await render("fast", fast);
|
||||
const db = psnr(baseline, fast);
|
||||
console.log(`[validate-fast-video] fast-vs-baseline PSNR = ${db} dB`);
|
||||
if (db < MIN_PSNR) {
|
||||
console.error(
|
||||
`[validate-fast-video] FAIL — ${db} dB < ${MIN_PSNR} dB. Fast capture dropped video ` +
|
||||
`(stale/black snapshot). The Linux BeginFrame paint path is not capturing video.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("[validate-fast-video] PASS — fast video matches baseline.");
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
// AUTO-GENERATED by scripts/build-hf-early-stub.ts — do not edit
|
||||
const HF_EARLY_STUB_IIFE: string =
|
||||
'"use strict";(()=>{var T=100,_=[],u=[],l=!1,s=!1;function m(n){let i=window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;return typeof i=="function"?i(n):requestAnimationFrame(n)}function y(n){let i=window.__HF_VIRTUAL_TIME__?.originalSetTimeout;if(typeof i=="function"){i(n,0);return}setTimeout(n,0)}function g(n){return n!==null&&typeof n=="object"&&"__hfIsProxy"in n?n.__hfReal:n}function c(n){let i=n.proxy.__hfReal,e=i[n.method];if(typeof e=="function"){let o=n.method==="add"?n.args.map(g):n.args;e.call(i,...o)}}function r(n,i,e){let o={proxy:n,method:i,args:e};return n.__hfQueue.push(o),u.push(o),P(),n}function d(n){let i=n.proxy.__hfQueue.indexOf(n);i>=0&&n.proxy.__hfQueue.splice(i,1)}function t(){for(;u.length>0;){let n=u.shift();n&&(d(n),c(n))}x()}function f(){s=!1,window.__hfTimelinesBuilding=!1;try{window.dispatchEvent(new CustomEvent("hf-timelines-built"))}catch{}}function x(){s||(s=!0,y(()=>{u.length===0?f():s=!1}))}function k(){l=!1;let n=u.splice(0,T);for(let i of n)d(i),c(i);u.length>0?(l=!0,m(k)):f()}function P(){l||(l=!0,window.__hfTimelinesBuilding=!0,m(k))}var O=new Set(["to","from","fromTo","set","add"]);function b(n,i){let e=i;for(;e!==null&&e!==Object.prototype;){for(let o of Object.getOwnPropertyNames(e)){if(o==="constructor"||o==="then"||o in n||O.has(o)||o.charAt(0)==="_")continue;let a=Object.getOwnPropertyDescriptor(e,o);if(!a||typeof a.value!="function")continue;let h=a.value;n[o]=function(...p){t();let w=h.call(i,...p);return w===i?n:w}}e=Object.getPrototypeOf(e)}}function v(n){let i={__hfReal:n,__hfQueue:[],__hfIsProxy:!0,to(...e){return r(i,"to",e)},from(...e){return r(i,"from",e)},fromTo(...e){return r(i,"fromTo",e)},set(...e){return r(i,"set",e)},add(...e){return r(i,"add",e)},pause(...e){return t(),n.pause(...e),i},play(...e){return t(),n.play(...e),i},seek(...e){return t(),n.seek(...e),i},totalTime(...e){return t(),e.length>0?(n.totalTime(...e),i):n.totalTime()},time(...e){return t(),e.length>0?(n.time(...e),i):n.time()},duration(...e){return t(),e.length>0?(n.duration(...e),i):n.duration()},getChildren(...e){t();let o=n.getChildren(...e);return Array.isArray(o)?o:[]},paused(...e){return t(),e.length>0?(n.paused(...e),i):n.paused()},timeScale(...e){return t(),e.length>0?(n.timeScale(...e),i):n.timeScale()},kill(){t(),n.kill()}};return b(i,n),_.push(i),i}if(typeof window<"u"){window.__hf||(window.__hf={}),window.__hfTimelinesBuilding=!1,window.__hfFlushSync=()=>{t(),u.length===0&&window.__hfTimelinesBuilding&&f()};let n=null;try{Object.defineProperty(window,"gsap",{configurable:!0,enumerable:!0,get(){return n},set(i){if(n=i,!i||typeof i.timeline!="function")return;let e=i.timeline.bind(i);i.timeline=o=>v(e(o))}})}catch{}}})();\n';
|
||||
'"use strict";(()=>{var b=100,O=[],u=[],w=!1,f=!1,c=new Set,p=new Set;function y(n){let e=window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;return typeof e=="function"?e(n):requestAnimationFrame(n)}function x(n){let e=window.__HF_VIRTUAL_TIME__?.originalSetTimeout;if(typeof e=="function"){e(n,0);return}setTimeout(n,0)}function S(n){return n!==null&&typeof n=="object"&&"__hfIsProxy"in n?n.__hfReal:n}function _(n){if(n===null||typeof n!="object"||Array.isArray(n))return n;let e=n;if(!("opacity"in e)||"autoAlpha"in e||"visibility"in e)return n;let o={};for(let t of Object.keys(e))t==="opacity"?o.autoAlpha=e[t]:o[t]=e[t];return o}function T(n,e){if(n!=="add"&&v(e),window.__HF_FAST_CAPTURE_AUTOALPHA__!==!0||n==="add")return e;let o=e.slice(),t=!1;if(o.length>1){let i=_(o[1]);i!==o[1]&&(t=!0),o[1]=i}if(n==="fromTo"&&o.length>2){let i=_(o[2]);i!==o[2]&&(t=!0),o[2]=i}return t&&o[0]!==null&&o[0]!==void 0&&c.add(o[0]),o}function h(n){if(n===null||typeof n!="object"||Array.isArray(n))return!1;let e=n;return"rotationX"in e||"rotationY"in e||"transformPerspective"in e}var m=new Set;function v(n){let e=n[0];if(e==null)return;let o=window;m.has(e)||(m.add(e),o.__hfAllTweenTargets=Array.from(m)),(h(n[1])||h(n[2]))&&(p.add(e),o.__hf3dTweenTargets=Array.from(p))}function R(n){if(typeof n=="string")try{return Array.from(document.querySelectorAll(n))}catch{return[]}if(n instanceof Element)return[n];if(Array.isArray(n)||typeof NodeList<"u"&&n instanceof NodeList){let e=[];for(let o of n)o instanceof Element&&e.push(o);return e}return[]}function F(){if(window.__HF_FAST_CAPTURE_AUTOALPHA__!==!0||c.size===0)return;let n=Array.from(c);c.clear();for(let e of n)for(let o of R(e)){let t=o;if(!t.style||t.style.visibility!=="")continue;let i="";try{i=getComputedStyle(o).opacity}catch{continue}i==="0"&&(t.style.visibility="hidden")}}function g(n){let e=n.proxy.__hfReal,o=e[n.method];if(typeof o=="function"){let t=n.method==="add"?n.args.map(S):n.args;o.call(e,...t)}}function l(n,e,o){let t={proxy:n,method:e,args:T(e,o)};return n.__hfQueue.push(t),u.push(t),I(),n}function A(n){let e=n.proxy.__hfQueue.indexOf(n);e>=0&&n.proxy.__hfQueue.splice(e,1)}function r(){for(;u.length>0;){let n=u.shift();n&&(A(n),g(n))}E()}function k(){f=!1,F(),window.__hfTimelinesBuilding=!1;try{window.dispatchEvent(new CustomEvent("hf-timelines-built"))}catch{}}function E(){f||(f=!0,x(()=>{u.length===0?k():f=!1}))}function P(){w=!1;let n=u.splice(0,b);for(let e of n)A(e),g(e);u.length>0?(w=!0,y(P)):k()}function I(){w||(w=!0,window.__hfTimelinesBuilding=!0,y(P))}var C=new Set(["to","from","fromTo","set","add"]);function G(n,e){let o=e;for(;o!==null&&o!==Object.prototype;){for(let t of Object.getOwnPropertyNames(o)){if(t==="constructor"||t==="then"||t in n||C.has(t)||t.charAt(0)==="_")continue;let i=Object.getOwnPropertyDescriptor(o,t);if(!i||typeof i.value!="function")continue;let s=i.value;n[t]=function(...d){r();let a=s.call(e,...d);return a===e?n:a}}o=Object.getPrototypeOf(o)}}function H(n){let e={__hfReal:n,__hfQueue:[],__hfIsProxy:!0,to(...o){return l(e,"to",o)},from(...o){return l(e,"from",o)},fromTo(...o){return l(e,"fromTo",o)},set(...o){return l(e,"set",o)},add(...o){return l(e,"add",o)},pause(...o){return r(),n.pause(...o),e},play(...o){return r(),n.play(...o),e},seek(...o){return r(),n.seek(...o),e},totalTime(...o){return r(),o.length>0?(n.totalTime(...o),e):n.totalTime()},time(...o){return r(),o.length>0?(n.time(...o),e):n.time()},duration(...o){return r(),o.length>0?(n.duration(...o),e):n.duration()},getChildren(...o){r();let t=n.getChildren(...o);return Array.isArray(t)?t:[]},paused(...o){return r(),o.length>0?(n.paused(...o),e):n.paused()},timeScale(...o){return r(),o.length>0?(n.timeScale(...o),e):n.timeScale()},kill(){r(),n.kill()}};return G(e,n),O.push(e),e}if(typeof window<"u"){window.__hf||(window.__hf={}),window.__hfTimelinesBuilding=!1,window.__hfFlushSync=()=>{r(),u.length===0&&window.__hfTimelinesBuilding&&k()};let n=null;try{Object.defineProperty(window,"gsap",{configurable:!0,enumerable:!0,get(){return n},set(e){if(n=e,!e||typeof e.timeline!="function")return;let o=e.timeline.bind(e);e.timeline=i=>H(o(i));for(let i of["to","from","set"]){let s=e[i];if(typeof s!="function")continue;let d=s.bind(e);e[i]=(...a)=>d(...T(i,a))}let t=e.fromTo;if(typeof t=="function"){let i=t.bind(e);e.fromTo=(...s)=>i(...T("fromTo",s))}}})}catch{}}})();\n';
|
||||
|
||||
/**
|
||||
* Returns the pre-built HyperFrames early stub IIFE as a string constant.
|
||||
|
||||
@@ -113,6 +113,13 @@ type TestMetadata = {
|
||||
workers?: number; // Optional: auto-calculates if omitted
|
||||
/** Force HDR in the harness; omitted/false preserves historical SDR-only test behavior. */
|
||||
hdr?: boolean;
|
||||
/**
|
||||
* Render this suite with the experimental fast-capture path
|
||||
* (drawElementImage, `--experimental-fast-capture`). The golden must be
|
||||
* regenerated with the flag on. Used by the `fast-capture` regression
|
||||
* guard; omit for the default screenshot/BeginFrame capture.
|
||||
*/
|
||||
experimentalFastCapture?: boolean;
|
||||
/**
|
||||
* Render-time variable overrides, equivalent to `hyperframes render
|
||||
* --variables '<json>'`. Injected as `window.__hfVariables` before any
|
||||
@@ -374,6 +381,11 @@ function validateMetadata(meta: unknown): TestMetadata {
|
||||
if (rc.hdr !== undefined && typeof rc.hdr !== "boolean") {
|
||||
throw new Error("meta.json: 'renderConfig.hdr' must be a boolean (or omit for false)");
|
||||
}
|
||||
if (rc.experimentalFastCapture !== undefined && typeof rc.experimentalFastCapture !== "boolean") {
|
||||
throw new Error(
|
||||
"meta.json: 'renderConfig.experimentalFastCapture' must be a boolean (or omit for false)",
|
||||
);
|
||||
}
|
||||
if (
|
||||
rc.variables !== undefined &&
|
||||
(rc.variables === null || typeof rc.variables !== "object" || Array.isArray(rc.variables))
|
||||
@@ -1017,18 +1029,30 @@ async function runTestSuite(
|
||||
await runDistributedSimulatedRender(distributedInput);
|
||||
}
|
||||
} else {
|
||||
const job = createRenderJob({
|
||||
fps: suite.meta.renderConfig.fps,
|
||||
quality: "high", // Always use max quality for tests
|
||||
format: outputFormat,
|
||||
workers: suite.meta.renderConfig.workers,
|
||||
useGpu: false,
|
||||
debug: false,
|
||||
hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr",
|
||||
variables: suite.meta.renderConfig.variables,
|
||||
});
|
||||
// Opt-in fast capture (drawElementImage): drives resolveConfig via the env
|
||||
// var, scoped to this suite's render so it never leaks to other suites.
|
||||
const useFast = suite.meta.renderConfig.experimentalFastCapture === true;
|
||||
const prevFast = process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
|
||||
if (useFast) process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = "true";
|
||||
try {
|
||||
const job = createRenderJob({
|
||||
fps: suite.meta.renderConfig.fps,
|
||||
quality: "high", // Always use max quality for tests
|
||||
format: outputFormat,
|
||||
workers: suite.meta.renderConfig.workers,
|
||||
useGpu: false,
|
||||
debug: false,
|
||||
hdrMode: suite.meta.renderConfig.hdr ? "force-hdr" : "force-sdr",
|
||||
variables: suite.meta.renderConfig.variables,
|
||||
});
|
||||
|
||||
await executeRenderJob(job, tempSrcDir, renderedOutputPath);
|
||||
await executeRenderJob(job, tempSrcDir, renderedOutputPath);
|
||||
} finally {
|
||||
if (useFast) {
|
||||
if (prevFast === undefined) delete process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
|
||||
else process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = prevFast;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ event: "rendering_complete", suite: suite.id }));
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { describe, expect, it, mock, beforeAll } from "bun:test";
|
||||
import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -8,6 +9,7 @@ import {
|
||||
compileForRender,
|
||||
detectRenderModeHints,
|
||||
detectShaderTransitionUsage,
|
||||
detectThreeDTransformUsage,
|
||||
discoverAudioVolumeAutomationFromTimeline,
|
||||
inlineExternalScripts,
|
||||
localizeRemoteMediaSources,
|
||||
@@ -649,6 +651,50 @@ describe("detectRenderModeHints", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectThreeDTransformUsage", () => {
|
||||
it("detects CSS perspective property", () => {
|
||||
expect(detectThreeDTransformUsage("<style>.s { perspective: 1000px; }</style>")).toBe(true);
|
||||
});
|
||||
|
||||
it("detects transform-style preserve-3d", () => {
|
||||
expect(detectThreeDTransformUsage("<style>.c { transform-style: preserve-3d; }</style>")).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("detects backface-visibility", () => {
|
||||
expect(detectThreeDTransformUsage("<style>.f { backface-visibility: hidden; }</style>")).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("detects perspective() transform function", () => {
|
||||
expect(detectThreeDTransformUsage('<div style="transform: perspective(500px)"></div>')).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("detects GSAP transformPerspective", () => {
|
||||
expect(
|
||||
detectThreeDTransformUsage("<script>gsap.to(el, { transformPerspective: 800 })</script>"),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not match flat GSAP rotationX without a perspective context", () => {
|
||||
expect(detectThreeDTransformUsage("<script>gsap.to(el, { rotationX: 180 })</script>")).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not match translateZ(0) promotion hack", () => {
|
||||
expect(detectThreeDTransformUsage('<div style="transform: translateZ(0)"></div>')).toBe(false);
|
||||
});
|
||||
|
||||
it("does not match perspective: none", () => {
|
||||
expect(detectThreeDTransformUsage("<style>.s { perspective: none; }</style>")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectShaderTransitionUsage", () => {
|
||||
it("detects authored HyperShader initialization", () => {
|
||||
const html = `<!doctype html>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// fallow-ignore-file code-duplication complexity
|
||||
/**
|
||||
* HTML Compiler for Producer
|
||||
*
|
||||
@@ -67,6 +68,10 @@ export interface CompiledComposition {
|
||||
staticDuration: number;
|
||||
renderModeHints: RenderModeHints;
|
||||
hasShaderTransitions: boolean;
|
||||
/** Author HTML/CSS/scripts use a CSS 3D rendering context (pre-CDN-inline scan). */
|
||||
usesThreeDTransforms: boolean;
|
||||
/** Author HTML/CSS use mix-blend-mode (pre-CDN-inline scan). */
|
||||
usesMixBlendMode: boolean;
|
||||
}
|
||||
|
||||
/** Adapts linkedom's `parseHTML` to the `checkSubCompositionUsability` contract. */
|
||||
@@ -273,6 +278,35 @@ export function detectRenderModeHints(html: string): RenderModeHints {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 3D rendering-context signals. drawElementImage paints elements inside a
|
||||
* CSS 3D rendering context incorrectly: backface-visibility:hidden is
|
||||
* ignored (mid-flip elements show their mirrored backface), sibling content
|
||||
* of the 3D context can drop out of the capture, and the context's
|
||||
* background is lost. Observed on real-world gen_os comps (flip-card and
|
||||
* rotationX scene-entrance patterns) on macOS hardware GPU — this is a
|
||||
* drawElementImage limitation, not a SwiftShader artifact.
|
||||
*
|
||||
* Only genuine 3D-context signals are matched: `perspective` (property or
|
||||
* transform function), `transform-style: preserve-3d`, `backface-visibility`,
|
||||
* `matrix3d(` / `rotate3d(`, and GSAP's `transformPerspective`. Flat
|
||||
* rotationX/Y tweens without a perspective context render as 2D and are
|
||||
* deliberately NOT matched, nor is the ubiquitous `translateZ(0)` promotion
|
||||
* hack.
|
||||
*/
|
||||
const THREE_D_CONTEXT_PATTERN =
|
||||
/transform-style\s*:\s*preserve-3d|backface-visibility\s*:|perspective\s*:\s*[0-9]|perspective\s*\(|matrix3d\s*\(|rotate3d\s*\(|\btransformPerspective\b/i;
|
||||
|
||||
export function detectThreeDTransformUsage(html: string): boolean {
|
||||
return THREE_D_CONTEXT_PATTERN.test(html);
|
||||
}
|
||||
|
||||
const MIX_BLEND_MODE_PATTERN = /mix-blend-mode\s*:/i;
|
||||
|
||||
function detectMixBlendModeUsage(html: string): boolean {
|
||||
return MIX_BLEND_MODE_PATTERN.test(html);
|
||||
}
|
||||
|
||||
const SHADER_TRANSITION_USAGE_PATTERN =
|
||||
/\b(?:(?:window|globalThis)\s*\.\s*)?HyperShader\s*\.\s*init\s*\(|\b__hf\s*\.\s*transitions\s*=/;
|
||||
|
||||
@@ -1176,6 +1210,47 @@ export async function localizeRemoteImageSources(
|
||||
);
|
||||
}
|
||||
|
||||
// Match a remote url() inside a `background` / `background-image` CSS declaration
|
||||
// (style blocks or inline style attrs). `[^;}"']*?` lets position/color tokens
|
||||
// precede the url() in the shorthand while stopping at the declaration boundary.
|
||||
const REMOTE_BG_URL_RE =
|
||||
/background(?:-image)?\s*:\s*[^;}"']*?url\(\s*["']?(https?:\/\/[^"')]+)["']?\s*\)/gi;
|
||||
|
||||
/**
|
||||
* Download remote CSS `background-image: url(https://...)` references and rewrite
|
||||
* them to local same-origin paths.
|
||||
*
|
||||
* Why: `drawElementImage` (fast capture) OMITS cross-origin content, so a remote
|
||||
* background image renders BLACK on the drawElement path while the screenshot
|
||||
* baseline captures it (origin-agnostic) — a whole-region mismatch (e.g. 10f79c0b
|
||||
* picsum.photos backgrounds, 9.3 dB). `<img>`/`<video>`/`@font-face` are localized
|
||||
* by their own passes; this closes the background-image gap so the fast path sees
|
||||
* the same pixels as the baseline.
|
||||
*
|
||||
* @internal exported for unit testing only
|
||||
*/
|
||||
export async function localizeRemoteBackgroundImages(
|
||||
html: string,
|
||||
downloadDir: string,
|
||||
): Promise<{ html: string; remoteMediaAssets: Map<string, string> }> {
|
||||
const urlSet = new Set<string>();
|
||||
const re = new RegExp(REMOTE_BG_URL_RE.source, REMOTE_BG_URL_RE.flags);
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(html)) !== null) {
|
||||
if (m[1] && !isGoogleFontsUrl(m[1])) urlSet.add(m[1]);
|
||||
}
|
||||
return downloadAndRewriteUrls(
|
||||
urlSet,
|
||||
html,
|
||||
join(downloadDir, REMOTE_MEDIA_SUBDIR),
|
||||
"Remote background-image download failed for",
|
||||
"Localized remote background-image(s)",
|
||||
// Quoted url('..')/url("..") are rewritten by downloadAndRewriteUrls' default
|
||||
// replaceAll; this handles the unquoted url(https://..) form.
|
||||
(h, url, rel) => h.replaceAll(`url(${url})`, `url(${rel})`),
|
||||
);
|
||||
}
|
||||
|
||||
// Match url("https://...") or url('https://...') inside @font-face blocks.
|
||||
// We scan the full HTML (which includes <style> blocks) — matching against
|
||||
// @font-face context precisely would require a CSS parser; instead we match
|
||||
@@ -1548,6 +1623,11 @@ export async function compileForRender(
|
||||
);
|
||||
const renderModeHints = detectRenderModeHints(sanitizedHtml);
|
||||
const hasShaderTransitions = detectShaderTransitionUsage(sanitizedHtml);
|
||||
// Detected BEFORE inlineExternalScripts: GSAP's own source contains
|
||||
// `transformPerspective`, so scanning post-inline HTML would flag every
|
||||
// composition that loads GSAP from a CDN.
|
||||
const usesThreeDTransforms = detectThreeDTransformUsage(sanitizedHtml);
|
||||
const usesMixBlendMode = detectMixBlendModeUsage(sanitizedHtml);
|
||||
|
||||
const normalizedFontHtml = normalizeSystemFontPrimaryFamilies(
|
||||
injectTextRenderingRule(
|
||||
@@ -1602,12 +1682,18 @@ export async function compileForRender(
|
||||
const { html: htmlWithLocalImages, remoteMediaAssets: remoteImageAssets } =
|
||||
await localizeRemoteImageSources(htmlWithLocalMedia, downloadDir);
|
||||
|
||||
// Download remote CSS background-image url() references. drawElementImage omits
|
||||
// cross-origin content, so remote backgrounds render black on the fast path;
|
||||
// localising them to same-origin closes that gap.
|
||||
const { html: htmlWithLocalBg, remoteMediaAssets: remoteBgAssets } =
|
||||
await localizeRemoteBackgroundImages(htmlWithLocalImages, downloadDir);
|
||||
|
||||
// Download remote @font-face src URLs and rewrite to local paths.
|
||||
// Remote font URLs fail with a CORS rejection at render time (S3 does not
|
||||
// allow http://localhost:PORT as origin), causing Chrome to silently fall
|
||||
// back to the next font in the stack.
|
||||
const { html: htmlWithLocalizedFonts, remoteMediaAssets: remoteFontAssets } =
|
||||
await localizeRemoteFontFaces(htmlWithLocalImages, downloadDir);
|
||||
await localizeRemoteFontFaces(htmlWithLocalBg, downloadDir);
|
||||
|
||||
const gifSourceAssets = new Map<string, string>(remoteImageAssets);
|
||||
const {
|
||||
@@ -1638,6 +1724,9 @@ export async function compileForRender(
|
||||
for (const [relPath, absPath] of remoteImageAssets) {
|
||||
externalAssets.set(relPath, absPath);
|
||||
}
|
||||
for (const [relPath, absPath] of remoteBgAssets) {
|
||||
externalAssets.set(relPath, absPath);
|
||||
}
|
||||
for (const [relPath, absPath] of remoteFontAssets) {
|
||||
externalAssets.set(relPath, absPath);
|
||||
}
|
||||
@@ -1711,6 +1800,8 @@ export async function compileForRender(
|
||||
staticDuration,
|
||||
renderModeHints,
|
||||
hasShaderTransitions,
|
||||
usesThreeDTransforms,
|
||||
usesMixBlendMode,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,8 @@ import {
|
||||
type CaptureSession,
|
||||
type EngineConfig,
|
||||
captureFrame,
|
||||
captureFrameToBufferPipelined,
|
||||
writeCapturedFrame,
|
||||
closeCaptureSession,
|
||||
createCaptureSession,
|
||||
getCapturePerfSummary,
|
||||
@@ -271,16 +273,8 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
|
||||
const rangeEnd = frameRange?.endFrame ?? totalFrames;
|
||||
const rangeFrames = rangeEnd - rangeStart;
|
||||
|
||||
for (let i = 0; i < rangeFrames; i++) {
|
||||
assertNotAborted();
|
||||
const absoluteIdx = rangeStart + i;
|
||||
const time = (absoluteIdx * job.config.fps.den) / job.config.fps.num;
|
||||
await captureFrame(session, i, time);
|
||||
job.framesRendered = i + 1;
|
||||
|
||||
const frameProgress = (i + 1) / rangeFrames;
|
||||
const progress = 25 + frameProgress * 45;
|
||||
|
||||
const reportFrame = (fileIndex: number): void => {
|
||||
job.framesRendered = fileIndex + 1;
|
||||
// Keep status cadence identical to the streaming sequential path; the
|
||||
// capture error wrapper below must remain separate from finally so it
|
||||
// can throw with the browser console before cleanup overwrites flow.
|
||||
@@ -288,10 +282,43 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
|
||||
updateJobStatus(
|
||||
job,
|
||||
"rendering",
|
||||
`Capturing frame ${i + 1}/${rangeFrames}`,
|
||||
Math.round(progress),
|
||||
`Capturing frame ${fileIndex + 1}/${rangeFrames}`,
|
||||
Math.round(25 + ((fileIndex + 1) / rangeFrames) * 45),
|
||||
onProgress,
|
||||
);
|
||||
};
|
||||
|
||||
if (session.workerEncodeEnabled) {
|
||||
// Worker-encode depth-2 pipeline on the DISK path (mirrors the streaming
|
||||
// path): frame N's in-page Worker encodes while frame N+1's main thread
|
||||
// does seek+paint+drawElement. Long comps (>streaming cap) land here, so
|
||||
// without this they'd fall back to synchronous toDataURL and lose the
|
||||
// ~1.5-2x worker-encode speedup entirely.
|
||||
let prev: { fileIndex: number; encodeResult: Promise<Buffer> } | null = null;
|
||||
const drainPrev = async (): Promise<void> => {
|
||||
if (!prev) return;
|
||||
assertNotAborted();
|
||||
const buf = await prev.encodeResult;
|
||||
writeCapturedFrame(session, prev.fileIndex, buf);
|
||||
reportFrame(prev.fileIndex);
|
||||
};
|
||||
for (let i = 0; i < rangeFrames; i++) {
|
||||
assertNotAborted();
|
||||
const absoluteIdx = rangeStart + i;
|
||||
const time = (absoluteIdx * job.config.fps.den) / job.config.fps.num;
|
||||
const { encodeResult } = await captureFrameToBufferPipelined(session, i, time);
|
||||
await drainPrev();
|
||||
prev = { fileIndex: i, encodeResult };
|
||||
}
|
||||
await drainPrev();
|
||||
} else {
|
||||
for (let i = 0; i < rangeFrames; i++) {
|
||||
assertNotAborted();
|
||||
const absoluteIdx = rangeStart + i;
|
||||
const time = (absoluteIdx * job.config.fps.den) / job.config.fps.num;
|
||||
await captureFrame(session, i, time);
|
||||
reportFrame(i);
|
||||
}
|
||||
}
|
||||
// Capture the sequential session's static-dedup perf before close (the
|
||||
// counters are valid only while the session is live).
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// fallow-ignore-file code-duplication complexity
|
||||
/**
|
||||
* captureStreamingStage — single-machine fused capture + encode path.
|
||||
*
|
||||
@@ -48,6 +49,7 @@ import {
|
||||
type EngineConfig,
|
||||
type StreamingEncoder,
|
||||
captureFrameToBuffer,
|
||||
captureFrameToBufferPipelined,
|
||||
closeCaptureSession,
|
||||
createCaptureSession,
|
||||
createFrameReorderBuffer,
|
||||
@@ -134,6 +136,52 @@ export type CaptureStreamingStageResult =
|
||||
success: false;
|
||||
};
|
||||
|
||||
async function runWorkerEncodePipelineLoop(
|
||||
session: CaptureSession,
|
||||
totalFrames: number,
|
||||
job: CaptureStreamingStageInput["job"],
|
||||
currentEncoder: StreamingEncoder,
|
||||
reorderBuffer: ReturnType<typeof createFrameReorderBuffer>,
|
||||
assertNotAborted: () => void,
|
||||
onProgress: CaptureStreamingStageInput["onProgress"],
|
||||
): Promise<void> {
|
||||
let prev: { idx: number; encodeResult: Promise<Buffer> } | null = null;
|
||||
|
||||
const drainPrev = async (): Promise<void> => {
|
||||
if (!prev) return;
|
||||
// Observe aborts while parked here (the encode wait + ffmpeg write are the
|
||||
// longest stretch of the loop); without this an abort isn't seen until the
|
||||
// next produce iteration.
|
||||
assertNotAborted();
|
||||
const buf = await prev.encodeResult;
|
||||
await reorderBuffer.waitForFrame(prev.idx);
|
||||
ensureFrameWritten(await currentEncoder.writeFrame(buf), prev.idx, currentEncoder);
|
||||
reorderBuffer.advanceTo(prev.idx + 1);
|
||||
job.framesRendered = prev.idx + 1;
|
||||
updateJobStatus(
|
||||
job,
|
||||
"rendering",
|
||||
`Streaming frame ${prev.idx + 1}/${totalFrames}`,
|
||||
Math.round(25 + ((prev.idx + 1) / totalFrames) * 55),
|
||||
onProgress,
|
||||
);
|
||||
};
|
||||
|
||||
// On abort/throw the just-produced frame's encode is still in flight and never
|
||||
// awaited (it isn't `prev` yet); cleanupDrawElementWorkerEncode rejects it on
|
||||
// close. produceDrawElementFrame attaches a no-op catch to every encodeResult
|
||||
// at creation so that orphaned rejection is never an unhandled rejection — so
|
||||
// the loop needs no special guard here.
|
||||
for (let i = 0; i < totalFrames; i++) {
|
||||
assertNotAborted();
|
||||
const time = (i * job.config.fps.den) / job.config.fps.num;
|
||||
const { encodeResult } = await captureFrameToBufferPipelined(session, i, time);
|
||||
await drainPrev();
|
||||
prev = { idx: i, encodeResult };
|
||||
}
|
||||
await drainPrev();
|
||||
}
|
||||
|
||||
export async function runCaptureStreamingStage(
|
||||
input: CaptureStreamingStageInput,
|
||||
): Promise<CaptureStreamingStageResult> {
|
||||
@@ -275,29 +323,43 @@ export async function runCaptureStreamingStage(
|
||||
assertNotAborted();
|
||||
lastBrowserConsole = session.browserConsoleBuffer;
|
||||
|
||||
for (let i = 0; i < totalFrames; i++) {
|
||||
assertNotAborted();
|
||||
const time = (i * job.config.fps.den) / job.config.fps.num;
|
||||
const { buffer } = await captureFrameToBuffer(session, i, time);
|
||||
await reorderBuffer.waitForFrame(i);
|
||||
ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
|
||||
reorderBuffer.advanceTo(i + 1);
|
||||
job.framesRendered = i + 1;
|
||||
|
||||
const frameProgress = (i + 1) / totalFrames;
|
||||
const progress = 25 + frameProgress * 55;
|
||||
|
||||
// Keep status cadence identical to disk sequential capture; the
|
||||
// capture error wrapper below must remain separate from finally so it
|
||||
// can throw with the browser console before encoder cleanup runs.
|
||||
// fallow-ignore-next-line code-duplication
|
||||
updateJobStatus(
|
||||
if (session.workerEncodeEnabled) {
|
||||
// Worker-encode pipeline: depth-2. Frame N's in-page Worker encodes
|
||||
// while frame N+1's main thread does seek+paint+drawElement+kick.
|
||||
await runWorkerEncodePipelineLoop(
|
||||
session,
|
||||
totalFrames,
|
||||
job,
|
||||
"rendering",
|
||||
`Streaming frame ${i + 1}/${totalFrames}`,
|
||||
Math.round(progress),
|
||||
currentEncoder,
|
||||
reorderBuffer,
|
||||
assertNotAborted,
|
||||
onProgress,
|
||||
);
|
||||
} else {
|
||||
for (let i = 0; i < totalFrames; i++) {
|
||||
assertNotAborted();
|
||||
const time = (i * job.config.fps.den) / job.config.fps.num;
|
||||
const { buffer } = await captureFrameToBuffer(session, i, time);
|
||||
await reorderBuffer.waitForFrame(i);
|
||||
ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
|
||||
reorderBuffer.advanceTo(i + 1);
|
||||
job.framesRendered = i + 1;
|
||||
|
||||
const frameProgress = (i + 1) / totalFrames;
|
||||
const progress = 25 + frameProgress * 55;
|
||||
|
||||
// Keep status cadence identical to disk sequential capture; the
|
||||
// capture error wrapper below must remain separate from finally so it
|
||||
// can throw with the browser console before encoder cleanup runs.
|
||||
// fallow-ignore-next-line code-duplication
|
||||
updateJobStatus(
|
||||
job,
|
||||
"rendering",
|
||||
`Streaming frame ${i + 1}/${totalFrames}`,
|
||||
Math.round(progress),
|
||||
onProgress,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Capture the session's static-dedup perf before close (counters valid
|
||||
// only while the session is live).
|
||||
|
||||
@@ -137,8 +137,63 @@ export async function runCompileStage(input: CompileStageInput): Promise<Compile
|
||||
// composition's `renderModeHints.recommendScreenshot`. The single
|
||||
// write to `cfg.forceScreenshot` happens at the end of this block so
|
||||
// the contract is enforceable by inspection.
|
||||
const callerForced = cfg.forceScreenshot || needsAlpha;
|
||||
const { forceScreenshot } = applyRenderModeHints(callerForced, compiled, log);
|
||||
// Alpha output forces screenshot because BeginFrame doesn't preserve alpha —
|
||||
// but drawElement fast capture self-manages alpha (screenshot-launched
|
||||
// browser + png drawElementImage, pixel-perfect; see createCaptureSession).
|
||||
// Folding needsAlpha here would make the engine's forceScreenshot guard
|
||||
// disable fast capture for every transparent render, so skip the fold when
|
||||
// fast capture is on. Render-mode hints (e.g. raw requestAnimationFrame)
|
||||
// still force screenshot below — those are correctness routings that
|
||||
// drawElement must honor.
|
||||
// The fast-capture video gate was REMOVED here once Chrome 151 fixed crbug
|
||||
// 521861819. It keyed on compiled.videos.length > 0 as a proxy for the
|
||||
// word-by-word caption opacity pattern (drawElementImage dropped the promoted
|
||||
// opacity layers mid-fade, ~12 dB). On the 151 pinned floor, video + nested-fade
|
||||
// comps render correctly on the drawElement path (verified PSNR=inf vs baseline);
|
||||
// see docs/fast-capture-limitations.md Lim 2.
|
||||
// Fast-capture 3D-transform gate, same shape as the video gate above.
|
||||
// drawElementImage paints CSS 3D rendering contexts incorrectly:
|
||||
// backface-visibility:hidden is ignored (mid-flip elements capture their
|
||||
// mirrored backface), siblings of the 3D context can drop out, and the
|
||||
// context's background is lost. Reproduced on macOS hardware GPU with
|
||||
// real-world flip-card / rotationX-entrance comps (full-stream PSNR
|
||||
// 27–46 dB avg, 17 dB min vs baseline). Routes to the platform's baseline
|
||||
// capture; HF_FAST_CAPTURE_3D=true bypasses for R&D.
|
||||
// Detection runs inside the compiler on PRE-CDN-inline HTML — GSAP's own
|
||||
// source contains `transformPerspective`, so scanning compiled.html here
|
||||
// would flag every composition that loads GSAP.
|
||||
if (
|
||||
cfg.useDrawElement &&
|
||||
process.env.HF_FAST_CAPTURE_3D !== "true" &&
|
||||
compiled.usesThreeDTransforms
|
||||
) {
|
||||
cfg.useDrawElement = false;
|
||||
log.info(
|
||||
"[Render] Fast capture: composition uses a CSS 3D rendering context " +
|
||||
"(perspective / preserve-3d / backface-visibility) — disabling drawElementImage " +
|
||||
"for this render. Capture uses the platform's baseline route.",
|
||||
);
|
||||
}
|
||||
// Fast-capture mix-blend-mode gate, same shape as the 3D gate above.
|
||||
// drawElementImage captures each element's paint records before the
|
||||
// compositor resolves blend equations — blended layers render as if
|
||||
// mix-blend-mode were absent, producing saturated/damaged composites
|
||||
// (measured: 42 dB min vs 53 dB floor on real blend+filter comps, macOS GPU).
|
||||
// HF_FAST_CAPTURE_BLEND=true bypasses for R&D.
|
||||
if (
|
||||
cfg.useDrawElement &&
|
||||
process.env.HF_FAST_CAPTURE_BLEND !== "true" &&
|
||||
compiled.usesMixBlendMode
|
||||
) {
|
||||
cfg.useDrawElement = false;
|
||||
log.info(
|
||||
"[Render] Fast capture: composition uses mix-blend-mode — disabling drawElementImage " +
|
||||
"for this render. Capture uses the platform's baseline route.",
|
||||
);
|
||||
}
|
||||
const callerForced = cfg.forceScreenshot || (needsAlpha && !cfg.useDrawElement);
|
||||
const { forceScreenshot: hintForced } = applyRenderModeHints(callerForced, compiled, log);
|
||||
let forceScreenshot = hintForced;
|
||||
cfg.forceScreenshot = forceScreenshot;
|
||||
writeCompiledArtifacts(compiled, workDir, Boolean(job.config.debug));
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// fallow-ignore-file code-duplication complexity
|
||||
/**
|
||||
* probeStage — browser probe + recompile + media reconciliation.
|
||||
*
|
||||
@@ -38,6 +39,7 @@ import {
|
||||
getCompositionDuration,
|
||||
initializeSession,
|
||||
isTransientBrowserError,
|
||||
probeBeginFrameLiveness,
|
||||
} from "@hyperframes/engine";
|
||||
import { fpsToNumber } from "@hyperframes/core";
|
||||
import type { CompiledComposition } from "../../htmlCompiler.js";
|
||||
@@ -95,6 +97,14 @@ export interface ProbeStageResult {
|
||||
totalFrames: number;
|
||||
/** Wall-clock ms for the entire probe phase (near-zero when `needsBrowser` was false). */
|
||||
browserProbeMs: number;
|
||||
/**
|
||||
* True when the BeginFrame liveness probe timed out on this host (SwiftShader
|
||||
* stalls the first BeginFrame indefinitely for heavy-layer compositions —
|
||||
* style-N caption comps). The probe session has already been relaunched in
|
||||
* screenshot mode; the sequencer must flip its `captureForceScreenshot`
|
||||
* local so downstream capture stages follow.
|
||||
*/
|
||||
beginFrameStalled: boolean;
|
||||
}
|
||||
|
||||
export function hasScriptedAudioVolumeAutomation(html: string, audioCount: number): boolean {
|
||||
@@ -147,6 +157,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
|
||||
let fileServer: FileServerHandle | null = null;
|
||||
let probeSession: CaptureSession | null = null;
|
||||
let beginFrameStalled = false;
|
||||
let lastBrowserConsole: string[] = [];
|
||||
|
||||
const probeStart = Date.now();
|
||||
@@ -258,6 +269,65 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
probeSession = session;
|
||||
lastBrowserConsole = session.browserConsoleBuffer;
|
||||
|
||||
// BeginFrame liveness probe. On SwiftShader, heavy-layer compositions
|
||||
// (multi-group nested opacity caption animations — style-N prod comps)
|
||||
// stall the FIRST BeginFrame indefinitely (tested to 30 min). The
|
||||
// auto-worker calibration catches this via its capped protocol timeout,
|
||||
// but renders with explicit `--workers N` skip calibration and would
|
||||
// hang for the full protocol timeout. One bounded BeginFrame here gives
|
||||
// ground truth for every render that probes a browser: on stall,
|
||||
// relaunch the probe session in screenshot mode and tell the sequencer
|
||||
// (via `beginFrameStalled`) to route the whole render through
|
||||
// screenshot capture — the path the baseline already uses for these
|
||||
// comps. Healthy comps pay one extra composited frame (<1s on GPU, a
|
||||
// few seconds on SwiftShader).
|
||||
if (probeSession.launchCaptureMode === "beginframe") {
|
||||
const probeTimeoutMs =
|
||||
Number(process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS) > 0
|
||||
? Number(process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS)
|
||||
: 30_000;
|
||||
const livenessStart = Date.now();
|
||||
// Tick inside the post-warmup cushion: warmup < probe < first capture
|
||||
// keeps the session's BeginFrame frameTimeTicks monotonic.
|
||||
const probeTick = Math.max(
|
||||
0,
|
||||
probeSession.beginFrameTimeTicks - 5 * probeSession.beginFrameIntervalMs,
|
||||
);
|
||||
const alive = await probeBeginFrameLiveness(
|
||||
probeSession.page,
|
||||
probeTimeoutMs,
|
||||
probeTick,
|
||||
probeSession.beginFrameIntervalMs,
|
||||
);
|
||||
assertNotAborted();
|
||||
if (alive) {
|
||||
log.info("BeginFrame liveness probe passed", {
|
||||
probeMs: Date.now() - livenessStart,
|
||||
});
|
||||
} else {
|
||||
beginFrameStalled = true;
|
||||
log.warn(
|
||||
"[Render] BeginFrame liveness probe timed out — this composition stalls " +
|
||||
"BeginFrame on this host (SwiftShader heavy-layer pattern). Relaunching " +
|
||||
"the probe browser in screenshot capture mode; the render will use " +
|
||||
"screenshot capture throughout.",
|
||||
{ probeTimeoutMs },
|
||||
);
|
||||
lastBrowserConsole = probeSession.browserConsoleBuffer;
|
||||
await closeCaptureSession(probeSession).catch(() => {});
|
||||
probeSession = await createCaptureSession(
|
||||
fileServer.url,
|
||||
join(workDir, "probe-screenshot"),
|
||||
captureOpts,
|
||||
null,
|
||||
{ ...probeCfg, forceScreenshot: true },
|
||||
);
|
||||
await initializeSession(probeSession);
|
||||
assertNotAborted();
|
||||
lastBrowserConsole = probeSession.browserConsoleBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Discover root composition duration
|
||||
if (composition.duration <= 0) {
|
||||
log.info("Discovering composition duration...");
|
||||
@@ -315,6 +385,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
}
|
||||
|
||||
if (el.tagName === "video") {
|
||||
// fallow-ignore-next-line code-duplication
|
||||
if (existingVideoIds.has(el.id)) {
|
||||
// Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
|
||||
const existing = composition.videos.find((v) => v.id === el.id);
|
||||
@@ -361,6 +432,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
existingVideoIds.add(el.id);
|
||||
}
|
||||
} else if (el.tagName === "audio") {
|
||||
// fallow-ignore-next-line code-duplication
|
||||
if (existingAudioIds.has(el.id)) {
|
||||
const existing = composition.audios.find((a) => a.id === el.id);
|
||||
if (existing) {
|
||||
@@ -535,5 +607,6 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
duration,
|
||||
totalFrames,
|
||||
browserProbeMs,
|
||||
beginFrameStalled,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1266,10 +1266,19 @@ export async function executeRenderJob(
|
||||
|
||||
perfStages.browserProbeMs = probeResult.browserProbeMs;
|
||||
perfStages.compileMs = Date.now() - stage1Start;
|
||||
// BeginFrame liveness: the probe stage already relaunched its session in
|
||||
// screenshot mode when the first BeginFrame stalled (SwiftShader
|
||||
// heavy-layer comps) — flip the sequencer's capture routing to match so
|
||||
// calibration and capture stages never issue another BeginFrame.
|
||||
if (probeResult.beginFrameStalled && !captureForceScreenshot) {
|
||||
captureForceScreenshot = true;
|
||||
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
|
||||
}
|
||||
observability.checkpoint("browser_probe", "duration resolved", {
|
||||
durationSeconds: probeResult.duration,
|
||||
totalFrames,
|
||||
compositionHash,
|
||||
beginFrameStalled: probeResult.beginFrameStalled,
|
||||
});
|
||||
|
||||
// ── Stage 2: Video frame extraction ─────────────────────────────────
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// fallow-ignore-file unused-file
|
||||
// fallow-ignore-file unused-file complexity
|
||||
/**
|
||||
* HyperFrames early stub — injected at the very start of `<head>` before any
|
||||
* other scripts run. Compiled to an IIFE by scripts/build-hf-early-stub.ts.
|
||||
@@ -62,6 +62,12 @@ declare global {
|
||||
originalRequestAnimationFrame?: typeof window.requestAnimationFrame;
|
||||
originalSetTimeout?: typeof window.setTimeout;
|
||||
};
|
||||
/**
|
||||
* Set by the engine (evaluateOnNewDocument, before this stub) when fast
|
||||
* capture (drawElementImage) is active. Opt out per render with
|
||||
* HF_FAST_CAPTURE_AUTOALPHA=false.
|
||||
*/
|
||||
__HF_FAST_CAPTURE_AUTOALPHA__?: boolean;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,6 +133,23 @@ const activeProxies: TimelineProxy[] = [];
|
||||
const pendingOperations: TimelineOperation[] = [];
|
||||
let batchScheduled = false;
|
||||
let publishCheckScheduled = false;
|
||||
/**
|
||||
* Tween targets whose vars got the opacity → autoAlpha rewrite. Resolved and
|
||||
* visibility-hidden at flush completion when still at computed opacity 0 —
|
||||
* see hideTransparentAutoAlphaTargets.
|
||||
*/
|
||||
const autoAlphaRewrittenTargets = new Set<unknown>();
|
||||
|
||||
/**
|
||||
* Tween targets animated with 3D transform vars (rotationX / rotationY /
|
||||
* transformPerspective). drawElementImage cannot paint 3D transforms — the
|
||||
* engine's threeDProjection module re-projects these elements via WebGL and
|
||||
* reads this list at init to find targets whose transform is still flat at
|
||||
* t=0 (to()-style tweens never show up in a computed-style scan). Exposed as
|
||||
* window.__hf3dTweenTargets. rotationZ/rotation stay 2D and are not
|
||||
* recorded; a bare `z` without perspective has no visual effect.
|
||||
*/
|
||||
const threeDTweenTargets = new Set<unknown>();
|
||||
|
||||
function requestBatchFrame(callback: FrameRequestCallback): number {
|
||||
const originalRequestAnimationFrame = window.__HF_VIRTUAL_TIME__?.originalRequestAnimationFrame;
|
||||
@@ -158,6 +181,155 @@ function unwrapTimelineArg(arg: unknown): unknown {
|
||||
return arg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fast-capture autoAlpha rewrite.
|
||||
*
|
||||
* Elements at `opacity: 0` still paint as transparent promoted compositor
|
||||
* layers. Stacked opacity-0 containers (the word-by-word caption pattern)
|
||||
* break drawElementImage capture (blackout / misplaced paint records) and
|
||||
* stall SwiftShader's first BeginFrame. GSAP's `autoAlpha` is the same fade
|
||||
* but additionally sets `visibility: hidden` at 0, removing the element from
|
||||
* the paint tree — measured on the chat CI comp: 29.4 → 49.6 dB
|
||||
* fast-vs-baseline with zero blackout frames.
|
||||
*
|
||||
* When the engine signals fast capture (`__HF_FAST_CAPTURE_AUTOALPHA__`),
|
||||
* rewrite `opacity` → `autoAlpha` in tween vars. Skipped when the author
|
||||
* already manages `autoAlpha` or `visibility` themselves. Baseline renders
|
||||
* never see this — the flag is only set on fast-capture sessions.
|
||||
*/
|
||||
function convertVarsOpacityToAutoAlpha(vars: unknown): unknown {
|
||||
if (vars === null || typeof vars !== "object" || Array.isArray(vars)) return vars;
|
||||
const record = vars as Record<string, unknown>;
|
||||
if (!("opacity" in record) || "autoAlpha" in record || "visibility" in record) return vars;
|
||||
const converted: Record<string, unknown> = {};
|
||||
for (const key of Object.keys(record)) {
|
||||
if (key === "opacity") converted.autoAlpha = record[key];
|
||||
else converted[key] = record[key];
|
||||
}
|
||||
return converted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the autoAlpha rewrite to the vars argument(s) of a tween call.
|
||||
* `to`/`from`/`set` carry vars at index 1; `fromTo` at indexes 1 and 2.
|
||||
*/
|
||||
function convertTweenArgs(method: TimelineOperationMethod, args: unknown[]): unknown[] {
|
||||
// Always record tween targets (3D + all-targets) regardless of the autoAlpha
|
||||
// rewrite flag, so HF_FAST_CAPTURE_AUTOALPHA=false doesn't blind the 3D
|
||||
// projection's quad-animation check.
|
||||
if (method !== "add") recordThreeDTweenTarget(args);
|
||||
if (window.__HF_FAST_CAPTURE_AUTOALPHA__ !== true) return args;
|
||||
if (method === "add") return args;
|
||||
const out = args.slice();
|
||||
let rewritten = false;
|
||||
if (out.length > 1) {
|
||||
const converted = convertVarsOpacityToAutoAlpha(out[1]);
|
||||
if (converted !== out[1]) rewritten = true;
|
||||
out[1] = converted;
|
||||
}
|
||||
if (method === "fromTo" && out.length > 2) {
|
||||
const converted = convertVarsOpacityToAutoAlpha(out[2]);
|
||||
if (converted !== out[2]) rewritten = true;
|
||||
out[2] = converted;
|
||||
}
|
||||
// convertVarsOpacityToAutoAlpha returns the input object untouched when no
|
||||
// rewrite applies, so identity inequality means this target's opacity is
|
||||
// GSAP-controlled via autoAlpha and is safe to visibility-hide at flush.
|
||||
if (rewritten && out[0] !== null && out[0] !== undefined) {
|
||||
autoAlphaRewrittenTargets.add(out[0]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function varsHasThreeD(vars: unknown): boolean {
|
||||
if (vars === null || typeof vars !== "object" || Array.isArray(vars)) return false;
|
||||
const record = vars as Record<string, unknown>;
|
||||
return "rotationX" in record || "rotationY" in record || "transformPerspective" in record;
|
||||
}
|
||||
|
||||
/** Every tween target, regardless of vars — the 3D projection's quad
|
||||
* textures are rasterized once at init, so any GSAP-animated element inside
|
||||
* a quad's subtree makes that quad unprojectable (the engine falls back). */
|
||||
const allTweenTargets = new Set<unknown>();
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
function recordThreeDTweenTarget(args: unknown[]): void {
|
||||
const target = args[0];
|
||||
if (target === null || target === undefined) return;
|
||||
const w = window as Window & {
|
||||
__hf3dTweenTargets?: unknown[];
|
||||
__hfAllTweenTargets?: unknown[];
|
||||
};
|
||||
if (!allTweenTargets.has(target)) {
|
||||
allTweenTargets.add(target);
|
||||
w.__hfAllTweenTargets = Array.from(allTweenTargets);
|
||||
}
|
||||
if (varsHasThreeD(args[1]) || varsHasThreeD(args[2])) {
|
||||
threeDTweenTargets.add(target);
|
||||
w.__hf3dTweenTargets = Array.from(threeDTweenTargets);
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve a GSAP tween target (selector / Element / NodeList / array) to elements. */
|
||||
function resolveTweenTargets(target: unknown): Element[] {
|
||||
if (typeof target === "string") {
|
||||
try {
|
||||
return Array.from(document.querySelectorAll(target));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
if (target instanceof Element) return [target];
|
||||
if (Array.isArray(target) || (typeof NodeList !== "undefined" && target instanceof NodeList)) {
|
||||
const out: Element[] = [];
|
||||
for (const item of target as Iterable<unknown>) {
|
||||
if (item instanceof Element) out.push(item);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush-time transparent-layer hide.
|
||||
*
|
||||
* The autoAlpha rewrite only fixes opacity GSAP touches. Elements created
|
||||
* with inline `opacity: 0` (the gen_os caption-pill pattern:
|
||||
* `pill.style.cssText = "... opacity:0"` then `tl.set(pill, {opacity: 1})`
|
||||
* at each caption start) sit in the paint tree as transparent promoted
|
||||
* layers from frame 0 until their first autoAlpha event — long enough to
|
||||
* trigger the drawElementImage caption blackout mid-video.
|
||||
*
|
||||
* At flush completion (timeline built, still paused, nothing applied yet)
|
||||
* every recorded rewrite target that is STILL at computed opacity 0 gets
|
||||
* `visibility: hidden`. Safe by construction: only elements with a queued
|
||||
* autoAlpha tween are touched, and that tween restores visibility
|
||||
* (autoAlpha > 0 sets `visibility: inherit`) on its first applied frame.
|
||||
* Elements faded in by CSS animations, WAAPI, or raw style writes are never
|
||||
* recorded here and are left alone. Authors who set inline `visibility`
|
||||
* themselves are also skipped, mirroring convertVarsOpacityToAutoAlpha.
|
||||
*/
|
||||
function hideTransparentAutoAlphaTargets(): void {
|
||||
if (window.__HF_FAST_CAPTURE_AUTOALPHA__ !== true) return;
|
||||
if (autoAlphaRewrittenTargets.size === 0) return;
|
||||
const targets = Array.from(autoAlphaRewrittenTargets);
|
||||
autoAlphaRewrittenTargets.clear();
|
||||
for (const target of targets) {
|
||||
for (const el of resolveTweenTargets(target)) {
|
||||
const styled = el as Element & { style?: CSSStyleDeclaration };
|
||||
if (!styled.style) continue;
|
||||
if (styled.style.visibility !== "") continue;
|
||||
let computedOpacity = "";
|
||||
try {
|
||||
computedOpacity = getComputedStyle(el).opacity;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (computedOpacity === "0") styled.style.visibility = "hidden";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function applyTimelineOperation(entry: TimelineOperation): void {
|
||||
const real = entry.proxy.__hfReal;
|
||||
const fn = real[entry.method];
|
||||
@@ -172,7 +344,7 @@ function enqueueTimelineOperation(
|
||||
method: TimelineOperationMethod,
|
||||
args: unknown[],
|
||||
): TimelineProxy {
|
||||
const entry = { proxy, method, args };
|
||||
const entry = { proxy, method, args: convertTweenArgs(method, args) };
|
||||
proxy.__hfQueue.push(entry);
|
||||
pendingOperations.push(entry);
|
||||
scheduleBatch();
|
||||
@@ -196,6 +368,7 @@ function flushPendingOperations(): void {
|
||||
|
||||
function publishTimelinesBuilt(): void {
|
||||
publishCheckScheduled = false;
|
||||
hideTransparentAutoAlphaTargets();
|
||||
window.__hfTimelinesBuilding = false;
|
||||
try {
|
||||
window.dispatchEvent(new CustomEvent("hf-timelines-built"));
|
||||
@@ -419,6 +592,20 @@ if (typeof window !== "undefined") {
|
||||
if (!g || typeof g.timeline !== "function") return;
|
||||
const origTimeline = g.timeline.bind(g) as (params?: unknown) => GsapTimeline;
|
||||
g.timeline = (params?: unknown): GsapTimeline => wrapTimeline(origTimeline(params));
|
||||
// Fast-capture autoAlpha rewrite for top-level gsap.to/from/set/fromTo
|
||||
// calls (compositions often use `gsap.set(el, { opacity: 0 })` for
|
||||
// initial state — the timeline proxy never sees those).
|
||||
for (const method of ["to", "from", "set"] as const) {
|
||||
const orig = g[method];
|
||||
if (typeof orig !== "function") continue;
|
||||
const bound = (orig as (...a: unknown[]) => unknown).bind(g);
|
||||
g[method] = (...args: unknown[]): unknown => bound(...convertTweenArgs(method, args));
|
||||
}
|
||||
const origFromTo = g.fromTo;
|
||||
if (typeof origFromTo === "function") {
|
||||
const bound = (origFromTo as (...a: unknown[]) => unknown).bind(g);
|
||||
g.fromTo = (...args: unknown[]): unknown => bound(...convertTweenArgs("fromTo", args));
|
||||
}
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root" data-composition-id="css-spinner" data-width="1920" data-height="1080" data-start="0" data-duration="5">
|
||||
<div id="root" data-composition-id="css-spinner" data-width="1920" data-height="1080" data-start="0" data-duration="5" data-no-timeline>
|
||||
<div class="stage clip" data-start="0" data-duration="5">
|
||||
<div class="spinner"></div>
|
||||
<div class="label">Loading</div>
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>fast-capture-3d</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
html, body { background: #ece8dd; }
|
||||
#stage { position: relative; width: 960px; height: 540px; background: #ece8dd; overflow: hidden; }
|
||||
.headline {
|
||||
position: absolute; top: 40px; left: 0; width: 100%; text-align: center;
|
||||
font: 700 44px Georgia, serif; color: #1a2640;
|
||||
}
|
||||
/* classic flip card: perspective container + preserve-3d + backface culling */
|
||||
.flip-wrap {
|
||||
position: absolute; top: 140px; left: 330px; width: 300px; height: 90px;
|
||||
perspective: 1000px;
|
||||
}
|
||||
.flip-card { position: relative; width: 100%; height: 100%; transform-style: preserve-3d; }
|
||||
.face {
|
||||
position: absolute; inset: 0; backface-visibility: hidden;
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
font: 700 34px Georgia, serif; border-radius: 8px;
|
||||
}
|
||||
.front { background: #1a2640; color: #fff; }
|
||||
.back { background: #c8a870; color: #1a2640; font-style: italic; transform: rotateX(-180deg); }
|
||||
/* standalone 3D entrance: GSAP rotationX on a sized div, static content */
|
||||
.badge {
|
||||
position: absolute; top: 300px; left: 380px; width: 200px; height: 64px;
|
||||
background: #406080; color: #fff; font: 700 26px Georgia, serif;
|
||||
text-align: center; line-height: 64px; border-radius: 8px;
|
||||
}
|
||||
.footer {
|
||||
position: absolute; top: 430px; left: 0; width: 100%; text-align: center;
|
||||
font: 700 28px Georgia, serif; color: #406080;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="stage" data-composition-id="fast-capture-3d" data-width="960" data-height="540" data-duration="4">
|
||||
<div class="headline">3D PROJECTION TEST</div>
|
||||
<div class="flip-wrap"><div class="flip-card" id="card">
|
||||
<div class="face front">FRONT</div>
|
||||
<div class="face back">BACK</div>
|
||||
</div></div>
|
||||
<div class="badge" id="badge">ENTRANCE</div>
|
||||
<div class="footer">STATIC FOOTER</div>
|
||||
</div>
|
||||
<script src="https://cdn.jsdelivr.net/npm/gsap@3.14.2/dist/gsap.min.js"></script>
|
||||
<script>
|
||||
const tl = gsap.timeline({ paused: true });
|
||||
// flip card: two half-turns with rests between
|
||||
tl.to("#card", { rotationX: 180, duration: 1.2, ease: "power2.inOut" }, 0.4);
|
||||
tl.to("#card", { rotationX: 360, duration: 1.2, ease: "power2.inOut" }, 2.4);
|
||||
// standalone 3D entrance (perspective-free rotationX — drawElementImage
|
||||
// silently drops the rotation without projection)
|
||||
tl.fromTo("#badge",
|
||||
{ rotationX: -90, opacity: 0, transformPerspective: 800 },
|
||||
{ rotationX: 0, opacity: 1, duration: 1.0, ease: "back.out(1.7)" }, 0.2);
|
||||
window.__timelines = window.__timelines || {};
|
||||
window.__timelines["fast-capture-3d"] = tl;
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "fast-capture-gsap",
|
||||
"description": "Regression guard for the experimental fast-capture (drawElementImage) path. Renders an opaque GSAP transform animation with --experimental-fast-capture; the golden is drawElement output, so this catches regressions in the canvas-injection / drawElement capture path on the Linux/Docker CI platform.",
|
||||
"tags": ["regression", "fast-capture"],
|
||||
"minPsnr": 30,
|
||||
"maxFrameFailures": 0,
|
||||
"minAudioCorrelation": 0,
|
||||
"maxAudioLagWindows": 1,
|
||||
"renderConfig": {
|
||||
"fps": 30,
|
||||
"experimentalFastCapture": true
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8974304164d6698be0fe168be5f485e3076cf32981565f343996b749a7352c7c
|
||||
size 120088
|
||||
@@ -0,0 +1,32 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=1080, height=1080">
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/gsap/3.12.2/gsap.min.js"></script>
|
||||
<style>
|
||||
body { margin: 0; background: #0d1117; width: 1080px; height: 1080px; overflow: hidden; }
|
||||
#box {
|
||||
position: absolute; left: 440px; top: 440px; width: 200px; height: 200px;
|
||||
background: #58a6ff; border-radius: 28px;
|
||||
}
|
||||
#label {
|
||||
position: absolute; top: 90px; width: 1080px; text-align: center;
|
||||
color: #e6edf3; font: bold 56px sans-serif;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root" data-composition-id="root" data-width="1080" data-height="1080" data-duration="2"
|
||||
style="position:absolute;width:1080px;height:1080px;">
|
||||
<div id="label">FAST CAPTURE</div>
|
||||
<div id="box"></div>
|
||||
</div>
|
||||
<script>
|
||||
const tl = gsap.timeline({ paused: true });
|
||||
tl.to("#box", { rotation: 360, x: 220, y: -120, duration: 2, ease: "none" }, 0);
|
||||
tl.to("#box", { backgroundColor: "#3fb950", duration: 2, ease: "none" }, 0);
|
||||
window.__timelines["root"] = tl;
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -6,6 +6,7 @@
|
||||
<body style="margin: 0">
|
||||
<div
|
||||
data-composition-id="render-symlinked-assets"
|
||||
data-no-timeline
|
||||
data-width="320"
|
||||
data-height="180"
|
||||
data-duration="5"
|
||||
|
||||
Reference in New Issue
Block a user