ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-03 04:38:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(engine): preserve video frame replacement geometry (#838)

Browse Source 
* fix(engine): preserve video frame replacement geometry

* test(producer): cover video overlay stretch regression

* fix(engine): always pass clip to Page.captureScreenshot

Without an explicit clip, Chrome can resolve replaced-element sizing
differently at dpr=1 when full-bleed absolute videos interact with
overlay layers — producing anisotropic frame stretching on some
compositor paths. Always passing clip with scale=dpr (including 1)
ensures geometry is locked to the measured viewport dimensions.

Credit: brian-t-allen (#837)

* test(producer): regenerate style-9-prod baseline for always-clip capture path

The always-clip change in screenshotService.ts routes Chrome through a
different compositor capture path at dpr=1, producing different video
frame compression artifacts. Regenerated inside Dockerfile.test to match
CI environment.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Miguel Ángel
2026-05-15 18:58:50 +02:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 952d2658b0
commit 8e0cfc33a7
10 changed files with 1451 additions and 730 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/engine/src/services/screenshotService.test.ts
+107 -6
View File
@@ -1,7 +1,12 @@
// @vitest-environment node
import { describe, it, expect, vi } from "vitest";
import { parseHTML } from "linkedom";
import { type Page } from "puppeteer-core";
import { pageScreenshotCapture, cdpSessionCache } from "./screenshotService.js";
import {
pageScreenshotCapture,
cdpSessionCache,
injectVideoFramesBatch,
} from "./screenshotService.js";
// Stub a Page + CDPSession just enough that pageScreenshotCapture can call
// `client.send("Page.captureScreenshot", ...)` and we can inspect the args.
@@ -20,7 +25,7 @@ describe("pageScreenshotCapture supersample plumbing", () => {
const ONE_PIXEL_PNG_B64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII=";
it("omits `clip` when deviceScaleFactor is undefined (default 1)", async () => {
it("passes `clip` with scale 1 when deviceScaleFactor is undefined (default 1)", async () => {
const send = vi.fn().mockResolvedValue({ data: ONE_PIXEL_PNG_B64 });
const page = makeFakePageWithCdp(send);
@@ -34,11 +39,13 @@ describe("pageScreenshotCapture supersample plumbing", () => {
expect(send).toHaveBeenCalledWith(
"Page.captureScreenshot",
expect.not.objectContaining({ clip: expect.anything() }),
expect.objectContaining({
clip: { x: 0, y: 0, width: 1920, height: 1080, scale: 1 },
}),
);
});
it("omits `clip` when deviceScaleFactor is exactly 1", async () => {
it("passes `clip` with scale 1 when deviceScaleFactor is exactly 1", async () => {
const send = vi.fn().mockResolvedValue({ data: ONE_PIXEL_PNG_B64 });
const page = makeFakePageWithCdp(send);
@@ -50,8 +57,8 @@ describe("pageScreenshotCapture supersample plumbing", () => {
deviceScaleFactor: 1,
});
const params = send.mock.calls[0]?.[1] as { clip?: unknown };
expect(params.clip).toBeUndefined();
const params = send.mock.calls[0]?.[1] as { clip?: { scale: number } };
expect(params.clip).toEqual({ x: 0, y: 0, width: 1920, height: 1080, scale: 1 });
});
it("passes `clip` with `scale = dpr` when deviceScaleFactor > 1 (the supersample contract)", async () => {
@@ -90,3 +97,97 @@ describe("pageScreenshotCapture supersample plumbing", () => {
expect(params.clip?.scale).toBe(3);
});
});
describe("injectVideoFramesBatch replacement layout", () => {
it("does not copy opposing inset constraints onto the injected frame image", async () => {
const { window, document } = parseHTML(
'<html><body><div id="root"><video id="clip" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover"></video></div></body></html>',
);
Object.defineProperty(window.HTMLImageElement.prototype, "decode", {
configurable: true,
value: () => Promise.resolve(),
});
const video = document.getElementById("clip") as HTMLVideoElement;
Object.defineProperties(video, {
offsetLeft: { configurable: true, get: () => 0 },
offsetTop: { configurable: true, get: () => 0 },
offsetWidth: { configurable: true, get: () => 1920 },
offsetHeight: { configurable: true, get: () => 1080 },
});
video.getBoundingClientRect = () =>
({
x: 0,
y: 0,
left: 0,
top: 0,
right: 1920,
bottom: 1080,
width: 1920,
height: 1080,
toJSON: () => ({}),
}) as DOMRect;
const computedStyle = document.createElement("div").style;
computedStyle.position = "absolute";
computedStyle.width = "1920px";
computedStyle.height = "1080px";
computedStyle.top = "0px";
computedStyle.left = "0px";
computedStyle.right = "0px";
computedStyle.bottom = "0px";
computedStyle.inset = "0px";
computedStyle.objectFit = "cover";
computedStyle.objectPosition = "center center";
computedStyle.zIndex = "3";
computedStyle.opacity = "1";
Object.defineProperty(window, "getComputedStyle", {
configurable: true,
value: () => computedStyle,
});
const globals = globalThis as unknown as {
window?: typeof window;
document?: Document;
};
const previousWindow = globals.window;
const previousDocument = globals.document;
globals.window = window;
globals.document = document;
try {
const page = {
evaluate: async (
fn: (
updates: Array<{ videoId: string; dataUri: string }>,
visualProperties: string[],
) => Promise<void>,
updates: Array<{ videoId: string; dataUri: string }>,
visualProperties: string[],
) => fn(updates, visualProperties),
} as unknown as Page;
await injectVideoFramesBatch(page, [
{
videoId: "clip",
dataUri:
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII=",
},
]);
} finally {
globals.window = previousWindow;
globals.document = previousDocument;
}
const img = video.nextElementSibling as HTMLImageElement | null;
expect(img).not.toBeNull();
expect(img?.style.position).toBe("absolute");
expect(img?.style.left).toBe("0px");
expect(img?.style.top).toBe("0px");
expect(img?.style.width).toBe("1920px");
expect(img?.style.height).toBe("1080px");
expect(img?.style.right).toBe("auto");
expect(img?.style.bottom).toBe("auto");
expect(img?.style.inset).toBe("auto");
});
});
packages/engine/src/services/screenshotService.ts
+36 -32
View File
@@ -130,19 +130,14 @@ export async function pageScreenshotCapture(page: Page, options: CaptureOptions)
const client = await getCdpSession(page);
const isPng = options.format === "png";
const dpr = options.deviceScaleFactor ?? 1;
// When supersampling, pass an explicit clip with `scale` so Chrome emits a
// screenshot at device-pixel dimensions (`width × height × dpr`). Without
// this, `Page.captureScreenshot` returns at CSS dimensions regardless of
// the viewport's deviceScaleFactor.
const clip =
dpr > 1 ? { x: 0, y: 0, width: options.width, height: options.height, scale: dpr } : undefined;
const clip = { x: 0, y: 0, width: options.width, height: options.height, scale: dpr };
const result = await client.send("Page.captureScreenshot", {
format: isPng ? "png" : "jpeg",
quality: isPng ? undefined : (options.quality ?? 80),
fromSurface: true,
captureBeyondViewport: false,
optimizeForSpeed: !isPng,
...(clip ? { clip } : {}),
clip,
});
return Buffer.from(result.data, "base64");
}
@@ -382,6 +377,15 @@ export async function injectVideoFramesBatch(
await page.evaluate(
async (items: Array<{ videoId: string; dataUri: string }>, visualProperties: string[]) => {
const pendingDecodes: Array<Promise<void>> = [];
const replacementLayoutProperties = new Set([
"width",
"height",
"top",
"left",
"right",
"bottom",
"inset",
]);
for (const item of items) {
const video = document.getElementById(item.videoId) as HTMLVideoElement | null;
if (!video) continue;
@@ -395,7 +399,6 @@ export async function injectVideoFramesBatch(
// and accurately reflects the user's intent on every frame.
const opacityParsed = parseFloat(computedStyle.opacity);
const computedOpacity = Number.isNaN(opacityParsed) ? 1 : opacityParsed;
const sourceIsStatic = !computedStyle.position || computedStyle.position === "static";
if (isNewImage) {
img = document.createElement("img");
@@ -406,10 +409,35 @@ export async function injectVideoFramesBatch(
}
if (!img) continue;
for (const property of visualProperties) {
// Opacity is handled explicitly via `computedOpacity` below — copying
// via the generic loop would race against the opacity:0 hide applied
// to the <video> at the end of this function. GSAP may animate
// opacity either on a wrapper (the <img> inherits via the stacking
// context) or directly on the <video> (we must copy it to the <img>
// since they are siblings). Reading computedStyle.opacity before
// hiding the <video> handles both cases correctly.
if (property === "opacity") continue;
// Layout is set from the video's used box below. Copying authored
// opposing constraints such as `inset: 0` / `right: 0` onto the
// replacement <img> can overconstrain replaced-image sizing and make
// some Chrome capture paths resample the frame anisotropically.
if (replacementLayoutProperties.has(property)) {
continue;
}
const value = computedStyle.getPropertyValue(property);
if (value) {
img.style.setProperty(property, value);
}
}
// Always use absolute positioning so the <img> overlays the <video>
// instead of flowing below it. With position:relative, both elements
// stack vertically — the <img> lands below the video and gets clipped
// by any overflow:hidden ancestor (e.g., border-radius wrappers).
//
// Apply this after visual style copying so the measured used box is
// the final authority for replacement frame geometry.
{
const videoRect = video.getBoundingClientRect();
const offsetLeft = Number.isFinite(video.offsetLeft) ? video.offsetLeft : 0;
@@ -429,30 +457,6 @@ export async function injectVideoFramesBatch(
img.style.objectPosition = computedStyle.objectPosition;
img.style.zIndex = computedStyle.zIndex;
for (const property of visualProperties) {
// Opacity is handled explicitly via `computedOpacity` below — copying
// via the generic loop would race against the opacity:0 hide applied
// to the <video> at the end of this function. GSAP may animate
// opacity either on a wrapper (the <img> inherits via the stacking
// context) or directly on the <video> (we must copy it to the <img>
// since they are siblings). Reading computedStyle.opacity before
// hiding the <video> handles both cases correctly.
if (property === "opacity") continue;
if (
sourceIsStatic &&
(property === "top" ||
property === "left" ||
property === "right" ||
property === "bottom" ||
property === "inset")
) {
continue;
}
const value = computedStyle.getPropertyValue(property);
if (value) {
img.style.setProperty(property, value);
}
}
img.decoding = "sync";
if (img.getAttribute("src") !== item.dataUri) {
img.src = item.dataUri;
packages/producer/tests/render-video-overlay-stretch/meta.json
+13
View File
@@ -0,0 +1,13 @@
{
"name": "render-video-overlay-stretch",
"description": "Regression fixture for #837. Renders two full-bleed MP4 clips with position:absolute/inset:0 and an image overlay across the cut. The injected video-frame <img> must preserve the source video's measured 16:9 box instead of inheriting opposing inset constraints that can stretch the frame during overlay transitions.",
"tags": ["regression", "video", "overlay"],
"minPsnr": 30,
"maxFrameFailures": 0,
"minAudioCorrelation": 0,
"maxAudioLagWindows": 1,
"renderConfig": {
"fps": 30,
"workers": 1
}
}
packages/producer/tests/render-video-overlay-stretch/output/compiled.html
+78
View File
@@ -0,0 +1,78 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=480, height=270">
<title>Video Overlay Stretch Regression</title>
<style>
* {
box-sizing: border-box;
}
html,
body {
margin: 0;
padding: 0;
width: 480px;
height: 270px;
overflow: hidden;
background: #000;
}
#root {
position: relative;
width: 480px;
height: 270px;
overflow: hidden;
background: #000;
}
.scene-video {
position: absolute;
inset: 0;
width: 100%;
height: 100%;
object-fit: cover;
object-position: center center;
}
.transition-overlay {
position: absolute;
inset: 0;
width: 100%;
height: 100%;
object-fit: cover;
opacity: 0.82;
}
.cut-label {
position: absolute;
right: 12px;
bottom: 12px;
z-index: 20;
padding: 5px 8px;
border: 2px solid rgba(255, 255, 255, 0.88);
color: #fff;
background: rgba(0, 0, 0, 0.68);
font: 700 13px Arial, Helvetica, sans-serif;
}
</style>
</head>
<body>
<div id="root" data-composition-id="video-overlay-stretch" data-start="0" data-duration="4" data-width="480" data-height="270">
<video id="clip-a" class="clip scene-video" src="clip.mp4" data-start="0" data-duration="2" data-track-index="0" muted playsinline data-end="2" data-has-audio="false"></video>
<video id="clip-b" class="clip scene-video" src="clip.mp4" data-start="2" data-duration="2" data-track-index="1" muted playsinline data-end="4" data-has-audio="false"></video>
<img id="transition" class="clip transition-overlay" src="transition-overlay.png" data-start="1.6" data-duration="0.8" data-track-index="10" alt="">
<div class="clip cut-label" data-start="0" data-duration="2" data-track-index="20">
CLIP A
</div>
<div class="clip cut-label" data-start="2" data-duration="2" data-track-index="21">
CLIP B
</div>
</div>
<script>window.__timelines = window.__timelines || {};</script></body>
</html>
packages/producer/tests/render-video-overlay-stretch/output/output.mp4
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:95f96e2676be7808074ca22dc06f68bb845c83ff3f2036afd75730c4c487cc3a
size 20066
packages/producer/tests/render-video-overlay-stretch/src/clip.mp4
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:da59ab7e77075ae083b9c957bc6d7896a0e0d62eeef4f50be63e278a1feb7303
size 5358
packages/producer/tests/render-video-overlay-stretch/src/index.html
+113
View File
@@ -0,0 +1,113 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=480, height=270" />
<title>Video Overlay Stretch Regression</title>
<style>
* {
box-sizing: border-box;
}
html,
body {
margin: 0;
padding: 0;
width: 480px;
height: 270px;
overflow: hidden;
background: #000;
}
#root {
position: relative;
width: 480px;
height: 270px;
overflow: hidden;
background: #000;
}
.scene-video {
position: absolute;
inset: 0;
width: 100%;
height: 100%;
object-fit: cover;
object-position: center center;
}
.transition-overlay {
position: absolute;
inset: 0;
width: 100%;
height: 100%;
object-fit: cover;
opacity: 0.82;
}
.cut-label {
position: absolute;
right: 12px;
bottom: 12px;
z-index: 20;
padding: 5px 8px;
border: 2px solid rgba(255, 255, 255, 0.88);
color: #fff;
background: rgba(0, 0, 0, 0.68);
font: 700 13px Arial, Helvetica, sans-serif;
}
</style>
</head>
<body>
<div
id="root"
data-composition-id="video-overlay-stretch"
data-start="0"
data-duration="4"
data-width="480"
data-height="270"
>
<video
id="clip-a"
class="clip scene-video"
src="clip.mp4"
data-start="0"
data-duration="2"
data-track-index="0"
muted
playsinline
></video>
<video
id="clip-b"
class="clip scene-video"
src="clip.mp4"
data-start="2"
data-duration="2"
data-track-index="1"
muted
playsinline
></video>
<img
id="transition"
class="clip transition-overlay"
src="transition-overlay.png"
data-start="1.6"
data-duration="0.8"
data-track-index="10"
alt=""
/>
<div class="clip cut-label" data-start="0" data-duration="2" data-track-index="20">
CLIP A
</div>
<div class="clip cut-label" data-start="2" data-duration="2" data-track-index="21">
CLIP B
</div>
</div>
<script>
window.__timelines = window.__timelines || {};
</script>
</body>
</html>
packages/producer/tests/render-video-overlay-stretch/src/transition-overlay.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:79aa1c819cfedf44f09b80e6bb1f161eea3d8090dfc0eb3929a421c6b9e46664
size 1207
packages/producer/tests/style-9-prod/output/compiled.html
+1093 -690
View File
File diff suppressed because one or more lines are too long
packages/producer/tests/style-9-prod/output/output.mp4
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9e1103ae6d6f666f28c9dc8b814f08e625eb9e9da7bd9b321aa206cdf1463913
size 7984836
oid sha256:a256872dd11c18a5cbb776d105af04f13432264c5c0689fa79c6ca697c86cf6b
size 13478856