From 0e58344dca3052ca1e83337d22c54b977852f5b3 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Mon, 6 Jul 2026 16:21:12 -0700 Subject: [PATCH] =?UTF-8?q?feat(engine):=20frame-capture=20core=20?= =?UTF-8?q?=E2=80=94=20fast-capture=20routing,=20worker-encode,=20dedup=20?= =?UTF-8?q?extension=20(#1919)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(engine): drawElementImage capture service * chore(ci): ignore drawElementService exports pending upstack consumers Fallow's per-PR audit diffs against the merge base, so the bottom of the fast-capture stack (#1917) sees drawElementService's exports as unused — their consumers (frameCapture) land in #1919, two PRs upstack. ignoreExports entry documents this and can be dropped once #1919 merges. Co-Authored-By: Claude Fable 5 * feat(engine): 3D projection + compositor-effect risk gate * fix(engine): gate filter drop-shadow wherever blur gates (review) detectCssEffectRisk documented drop-shadow as a ~29dB damage case but only detected blur( in its three scan paths — a drop-shadow comp stayed on the fast path despite the gate's own correctness contract. Detect drop-shadow( in computed styles, stylesheet rules, and GSAP tween vars, pinned by a focused test that runs the real page-side closure against a DOM shim (computed / stylesheet / tween coverage + blur regression + effect-free null). Addresses miguel-heygen's blocker on #1918. Co-Authored-By: Claude Fable 5 * feat(engine): frame-capture core — fast-capture routing, worker-encode, dedup extension # Conflicts: # packages/engine/src/services/screenshotService.ts * fix(engine): document HF_FORCE_DRAWELEMENT as diagnostic-only; make armStaticDedup idempotent (review) Addresses miguel-heygen's blockers on #1919: - HF_FORCE_DRAWELEMENT promoted from a stale "SCRATCH/Uncommitted" comment to a documented diagnostic flag: it exists for upstream-Chromium repro work (gate-vs-API isolation, crbug 521861819 149-vs-151) and R&D on gated effect classes; renders under it may be damaged BY DESIGN since it bypasses gates whose thresholds encode measured damage. Never production; the safety-net blank guard also stands down under it so diagnostic frames arrive unmodified. - armStaticDedup is now idempotent: the drawElement init path arms dedup before canvas injection, then initializeSession called it again — the second run overwrote the armed state with skipReason="capture_mode" (captureMode is "drawelement" by then), producing contradictory telemetry (armed frames + a skip reason), and re-ran the verification seeks on the fallback path. It now no-ops once staticFrames or a skip decision exists. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- packages/engine/src/index.ts | 3 + .../engine/src/services/browserManager.ts | 3 +- packages/engine/src/services/frameCapture.ts | 731 +++++++++++++++++- .../engine/src/services/screenshotService.ts | 51 ++ 4 files changed, 754 insertions(+), 34 deletions(-) diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 6aedbb772..42192bdf5 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -82,6 +82,8 @@ export { closeCaptureSession, captureFrame, captureFrameToBuffer, + captureFrameToBufferPipelined, + writeCapturedFrame, discardWarmupCapture, getCompositionDuration, getCapturePerfSummary, @@ -101,6 +103,7 @@ export { injectVideoFramesBatch, syncVideoFrameVisibility, cdpSessionCache, + probeBeginFrameLiveness, initTransparentBackground, captureAlphaPng, applyDomLayerMask, diff --git a/packages/engine/src/services/browserManager.ts b/packages/engine/src/services/browserManager.ts index 4df2e3734..942f16834 100644 --- a/packages/engine/src/services/browserManager.ts +++ b/packages/engine/src/services/browserManager.ts @@ -98,7 +98,8 @@ async function probeHardwareWebGlInfo( // "beginframe" = atomic compositor control via HeadlessExperimental.beginFrame (Linux only) // "screenshot" = renderSeek + Page.captureScreenshot (all platforms) -export type CaptureMode = "beginframe" | "screenshot"; +// "drawelement" = BeginFrame compositor advance + canvas.drawElementImage capture +export type CaptureMode = "beginframe" | "screenshot" | "drawelement"; export interface AcquiredBrowser { browser: Browser; diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index ea1c4e46a..99643a6db 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -1,4 +1,4 @@ -// fallow-ignore-file complexity +// fallow-ignore-file complexity code-duplication /** * Frame Capture Service * @@ -30,6 +30,17 @@ import { initTransparentBackground, shouldDefaultCaptureBeyondViewport, } from "./screenshotService.js"; +import { + detectSwiftShader, + injectDrawElementCanvas, + captureDrawElementFrame, + resolveDrawElementCaptureMode, + instrumentAcceleratedCanvases, + initDrawElementWorkerEncode, + cleanupDrawElementWorkerEncode, + produceDrawElementFrame, +} from "./drawElementService.js"; +import { initThreeDProjection, detectCssEffectRisk } from "./threeDProjection.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import type { CaptureOptions, @@ -94,6 +105,13 @@ export interface CaptureSession { totalMs: number; }; captureMode: CaptureMode; + /** + * Browser LAUNCH mode, immutable after createCaptureSession. `captureMode` + * is reassigned by initializeSession (e.g. to "drawelement"), so callers + * that need to know whether this browser actually drives BeginFrame (the + * SwiftShader liveness probe) read this field instead. + */ + launchCaptureMode: CaptureMode; // BeginFrame state beginFrameTimeTicks: number; beginFrameIntervalMs: number; @@ -101,6 +119,32 @@ export interface CaptureSession { beginFrameNoDamageCount: number; /** Optional producer config — when set, overrides module-level env var constants. */ config?: Partial; + /** True if running on SwiftShader (detected at init). Undefined before init. */ + isSwiftShader?: boolean; + /** drawElementImage canvas was injected and is ready for capture. */ + drawElementReady?: boolean; + /** + * Worker-encode pipeline is active for this session. Set by + * `initDrawElementOrTransparentBackground` when `enableDrawElementWorkerEncode` + * is true and capture mode resolved to "drawelement". + */ + workerEncodeEnabled?: boolean; + /** + * Frame indices that must be captured via screenshot rather than drawElement. + * Populated at init by the clip-cut boundary predictor (Lim 6): frames where + * the outgoing clip is dropped a frame before the incoming clip's paint record + * is ready → black frame. Controlled by `HF_FAST_CAPTURE_BOUNDARY_SS=false`. + * Empty/undefined when the predictor produces no frames. + */ + clipBoundaryFrames?: Set; + /** Rolling drawElement frame byte-sizes (last ~60), for silent-blank-drop detection: + * drawElement intermittently returns an anomalously small (blank) frame with no + * throw; a frame far below the running median is re-captured via screenshot. */ + deFrameSizes?: number[]; + /** Last non-deduped encode result, reused for a static frame on the drawElement + * worker-encode path (mirrors `lastFrameBuffer` on the screenshot path). Only set + * when static-frame dedup is armed on the drawElement path. */ + lastEncodeResult?: Promise; } // Circular buffer for browser console messages dumped on render failure diagnostics. @@ -342,6 +386,234 @@ async function waitForCloseWithTimeout(promise: Promise): Promise 1): + * `drawElementImage` reads the canvas at CSS pixels and has no equivalent of + * `Page.captureScreenshot`'s clip+scale, so it would silently capture at 1x and + * drop the requested supersample. Such renders fall through to the screenshot + * path (preMode already forces "screenshot" for DPR > 1). + */ +async function initDrawElementOrTransparentBackground( + session: CaptureSession, + page: Page, + logInitPhase: (phase: string) => void, +): Promise { + const supersampling = (session.options.deviceScaleFactor ?? 1) > 1; + // forceScreenshot is an explicit routing decision made upstream (render-mode + // compat hints like raw requestAnimationFrame, alpha formats, low-memory) — + // drawElement must not override it. Concretely: an rAF-compat comp on + // SwiftShader gets a screenshot-launched (free-running) browser, where + // drawElement runs in paint-event-sync mode; SwiftShader never refreshes a + // 2d canvas bitmap inside a cached paint record there, so every canvas + // captures frozen-blank (raf-ball rendered fully black). On a GPU the same + // path happens to work, but the hint asked for screenshot — honor it. + const forceScreenshot = session.config?.forceScreenshot ?? false; + // DIAGNOSTIC ONLY — HF_FORCE_DRAWELEMENT=1 forces the drawElement path, + // bypassing every compile/init gate AND the compatibility hints (it overrides + // forceScreenshot). Exists for upstream-Chromium repro work (isolating gate + // behavior from drawElementImage behavior, e.g. the crbug 521861819 149-vs-151 + // comparison) and for R&D on gated effect classes. Renders under this flag may + // be DAMAGED by design — the gates it skips exist because measured damage + // (blur/backdrop ~18-49dB, 3D backface, SwiftShader sub-layer drops) is real. + // Never set it in production; it is intentionally not documented in user-facing + // help, and the safety-net blank guard also stands down under it so diagnostic + // frames arrive unmodified. + const forceDE = process.env.HF_FORCE_DRAWELEMENT === "1"; + const useDrawElement = + ((session.config?.useDrawElement ?? false) || forceDE) && + !supersampling && + (!forceScreenshot || forceDE); + if ((session.config?.useDrawElement ?? false) && supersampling) { + console.log( + "[engine] --experimental-fast-capture disabled for this render: drawElementImage " + + "ignores deviceScaleFactor, so supersampled (DPR > 1) output uses screenshot capture.", + ); + } + if ((session.config?.useDrawElement ?? false) && !supersampling && forceScreenshot) { + console.log( + "[engine] fast capture: falling back to screenshot — render-mode compatibility " + + "hint forced screenshot capture (e.g. raw requestAnimationFrame composition).", + ); + } + // Retract the per-page autoAlpha rewrite flag when a runtime gate routes the + // session to screenshot mode. evaluateOnNewDocument already fired; a follow-up + // evaluate overrides it in the live page context so hideTransparentAutoAlpha- + // Targets does not hide elements on the fallback screenshot render + // (up to 21 dB damage if not retracted, A/B proven 2026-06-12). + async function retractAutoAlphaFlag(): Promise { + await page.evaluate(() => { + ( + window as Window & { __HF_FAST_CAPTURE_AUTOALPHA__?: boolean } + ).__HF_FAST_CAPTURE_AUTOALPHA__ = false; + }); + } + if (useDrawElement) { + session.isSwiftShader = await detectSwiftShader(page); + const transparent = session.options.format === "png"; + async function routeToFallback(): Promise { + session.captureMode = session.launchCaptureMode; + if (transparent) { + await initTransparentBackground(session.page); + } + await retractAutoAlphaFlag(); + // Static-frame dedup is capture-mode-independent (the serial path reuses + // lastFrameBuffer regardless of how the frame was captured) and lossless + // (anchor-verified). A comp only reaches THIS fallback with useDrawElement=true + // AND forceScreenshot=false — i.e. it is deterministic: raw-rAF / iframe / + // htmlInCanvas comps are forced to screenshot upstream (forceScreenshot=true) and + // never enter this block, so they never arm dedup. The comps that DO fall back here + // (blur / backdrop / 3D / at-risk) carry only a compositor + // EFFECT drawElement can't paint, not nondeterminism, so their predicted-static set + // is sound. Verification seeks via Page.captureScreenshot, which hangs on a + // BeginFrame-launched browser — gate on the launch mode (macOS fast-capture launches + // screenshot-mode; Linux/Docker launches beginframe and is skipped). + if (session.launchCaptureMode === "screenshot") { + await armStaticDedup(session, page, logInitPhase); + } + } + // SwiftShader gate: drawElement's only advantage is skipping the GPU→CPU + // screenshot-readback IPC. On a software rasterizer (Docker/CI, no GPU) both + // paths block on identical software raster, so drawElement is parity-or-slower + // — route to the platform baseline. + // + // Two gates were REMOVED here once Chrome 151 fixed crbug 521861819 + // (drawElementImage dropped compositor-promoted opacity layers mid-fade): + // - the