fix(core): keep the colour grade on a graded video's first rendered frame (#5575)

* fix(core): keep the colour grade on a graded video's first rendered frame

A graded clip rendered ungraded for one frame wherever a capture session
first drew it: the clip's own first frame, a parallel worker's first frame
inside it, or a capture segment's first frame inside it. The grade dropped
the clip while its injected frame was still decoding, so the redraw after
decode had nothing to draw.

Part of #5548

* chore: pin the producer test-classification hash and drop an added comment

packages/core/src/runtime/colorGrading.test.ts

@@ -1041,6 +1041,29 @@ describe("createColorGradingRuntime", () => {
     expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();
   });
 
+  it("grades a clip's first producer frame when the DOM observer runs before it decodes", () => {
+    const video = makeDrawableVideo();
+    video.style.visibility = "hidden";
+    document.body.appendChild(video);
+    runtime = createColorGradingRuntime();
+
+    let decoded = false;
+    const frame = document.createElement("img");
+    frame.id = "__render_frame_hero-video__";
+    frame.className = "__render_frame__";
+    frame.style.visibility = "visible";
+    Object.defineProperty(frame, "complete", { get: () => decoded, configurable: true });
+    Object.defineProperty(frame, "naturalWidth", { get: () => (decoded ? 640 : 0) });
+    Object.defineProperty(frame, "naturalHeight", { get: () => (decoded ? 360 : 0) });
+    video.after(frame);
+    runtime.refresh();
+    decoded = true;
+
+    expect(runtime.redraw()).toBe(1);
+    expect(texImage2DCalls.at(-1)).toContain(frame);
+    expect(frame.nextSibling).toBe(document.querySelector("[data-hf-color-grading-canvas]"));
+  });
+
   it("does not recreate an inactive clip from its hidden producer frame", () => {
     const { video, canvas } = startRuntimeWithVideo();
     const frame = document.createElement("img");

packages/core/src/runtime/colorGrading.ts

@@ -2494,7 +2494,7 @@ function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null
 
 function hasInjectedRenderFrame(element: ColorGradingMediaElement): boolean {
   if (!isVideoElement(element)) return false;
-  const frame = findRenderFrameImage(element);
+  const frame = findInjectedRenderFrame(element);
   if (!frame) return false;
   const style = window.getComputedStyle(frame);
   return style.display !== "none" && style.visibility !== "hidden";

packages/producer/scripts/test-classification.mjs

@@ -33,6 +33,7 @@ const INTEGRATION_TEST_FILES = new Set([
   "src/services/render/hlsRender.integration.test.ts",
   "src/services/render/motionBlurVideo.integration.test.ts",
   "src/services/render/renderCancel.integration.test.ts",
+  "src/services/render/segmentedColorGrading.integration.test.ts",
   "src/services/render/stages/compileStage.mediaType.test.ts",
   "src/services/render/stages/gifEncodeArgs.test.ts",
   "src/services/videoFrameBorderClip.test.ts",

packages/producer/src/services/render/segmentedColorGrading.integration.test.ts

@@ -0,0 +1,195 @@
+// A graded <video>, rendered for real (#5548): the first frame a capture session draws inside the
+// clip (a segment's or a parallel worker's), and the clip's own first frame, keep the grade.
+// Needs Chrome + ffmpeg: integration lane.
+
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { getFfmpegBinary, resolveConfig } from "@hyperframes/engine";
+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";
+
+const FFMPEG = getFfmpegBinary();
+const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"], { encoding: "utf-8" }).status === 0;
+
+const WIDTH = 160;
+const HEIGHT = 90;
+const FPS = 30;
+const FRAME_COUNT = 60;
+const CLIP_FIRST = 15;
+const CLIP_LAST = 44;
+// The smallest segment size. With two workers, both the second segment and the second worker's
+// frame range start here, mid-clip.
+const SECOND_SESSION_FIRST = 30;
+const CLIP_GREY = 128;
+
+const COMPOSITION = `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <style>
+    * { margin: 0; padding: 0; box-sizing: border-box; }
+    body { overflow: hidden; background: #000; }
+    #clip { position: absolute; left: 0; top: 0; width: ${WIDTH}px; height: ${HEIGHT}px; }
+  </style>
+</head>
+<body>
+  <div
+    id="root"
+    data-composition-id="segmented-color-grading"
+    data-start="0"
+    data-duration="${FRAME_COUNT / FPS}"
+    data-width="${WIDTH}"
+    data-height="${HEIGHT}"
+    data-fps="${FPS}"
+  >
+    <video id="clip" class="clip" src="assets/clip.mp4" data-start="${CLIP_FIRST / FPS}"
+      data-duration="${(CLIP_LAST + 1 - CLIP_FIRST) / FPS}" data-track-index="0" muted playsinline
+      data-color-grading='{"adjust":{"exposure":-1}}'></video>
+  </div>
+  <script>window.__timelines = window.__timelines || {};</script>
+</body>
+</html>
+`;
+
+let root: string;
+
+/** Mean RGB level of every frame of an encoded video, 0-255. */
+function frameLevels(videoPath: string): number[] {
+  const raw = spawnSync(
+    FFMPEG,
+    ["-v", "error", "-i", videoPath, "-f", "rawvideo", "-pix_fmt", "gray", "-"],
+    {
+      maxBuffer: 64 * 1024 * 1024,
+    },
+  );
+  if (raw.status !== 0) throw new Error(`decode failed: ${raw.stderr}`);
+  const frameBytes = WIDTH * HEIGHT;
+  const levels: number[] = [];
+  for (let offset = 0; offset + frameBytes <= raw.stdout.length; offset += frameBytes) {
+    let sum = 0;
+    for (let i = offset; i < offset + frameBytes; i++) sum += raw.stdout[i]!;
+    levels.push(sum / frameBytes);
+  }
+  return levels;
+}
+
+interface Render {
+  levels: number[];
+  phases: string[];
+  workerCount: number;
+}
+
+async function render(name: string, segmented: boolean, workers: number): Promise<Render> {
+  const phases: string[] = [];
+  let workerCount = 0;
+  const job = createRenderJob({
+    fps: FPS,
+    quality: "draft",
+    format: "mp4",
+    hdrMode: "force-sdr",
+    workers,
+    logger: {
+      info(message: string, meta?: Record<string, unknown>) {
+        if (message !== "[Render:trace]") return;
+        if (typeof meta?.phase === "string") phases.push(meta.phase);
+        if (typeof meta?.workerCount === "number")
+          workerCount = Math.max(workerCount, meta.workerCount);
+      },
+      warn() {},
+      error() {},
+      debug() {},
+    },
+    producerConfig: resolveConfig({ browserGpuMode: "software" }),
+  });
+  const saved = {
+    capture: process.env.HF_SEGMENTED_CAPTURE,
+    frames: process.env.HF_SEGMENT_FRAMES,
+  };
+  process.env.HF_SEGMENTED_CAPTURE = String(segmented);
+  process.env.HF_SEGMENT_FRAMES = String(SECOND_SESSION_FIRST);
+  try {
+    const output = join(root, `${name}.mp4`);
+    await executeRenderJob(job, root, output);
+    return { levels: frameLevels(output), phases, workerCount };
+  } finally {
+    if (saved.capture === undefined) delete process.env.HF_SEGMENTED_CAPTURE;
+    else process.env.HF_SEGMENTED_CAPTURE = saved.capture;
+    if (saved.frames === undefined) delete process.env.HF_SEGMENT_FRAMES;
+    else process.env.HF_SEGMENT_FRAMES = saved.frames;
+  }
+}
+
+describe.skipIf(!HAS_FFMPEG)(
+  "color-graded video at a capture session's first frame — real render (#5548)",
+  () => {
+    let reference: Render;
+    let segmented: Render;
+    let parallel: Render;
+
+    beforeAll(async () => {
+      root = mkdtempSync(join(tmpdir(), "hf-segmented-grading-"));
+      mkdirSync(join(root, "assets"), { recursive: true });
+      writeFileSync(join(root, "index.html"), COMPOSITION);
+      const clip = spawnSync(
+        FFMPEG,
+        [
+          "-v",
+          "error",
+          "-f",
+          "lavfi",
+          "-i",
+          `color=c=0x808080:size=${WIDTH}x${HEIGHT}:rate=${FPS}:duration=2`,
+          "-pix_fmt",
+          "yuv420p",
+          "-c:v",
+          "libx264",
+          "-y",
+          join(root, "assets", "clip.mp4"),
+        ],
+        { encoding: "utf-8" },
+      );
+      if (clip.status !== 0) throw new Error(`clip fixture failed: ${clip.stderr}`);
+
+      reference = await render("reference", false, 1);
+      segmented = await render("segmented", true, 2);
+      parallel = await render("parallel", false, 2);
+    }, 600_000);
+
+    afterAll(() => {
+      if (root) rmSync(root, { recursive: true, force: true });
+    });
+
+    it("renders each shape on the route it asks for", () => {
+      expect(segmented.phases).toContain("capture_segmented");
+      expect(reference.phases).not.toContain("capture_segmented");
+      expect(parallel.phases).not.toContain("capture_segmented");
+      expect(reference.workerCount).toBe(1);
+      expect(segmented.workerCount).toBe(2);
+      expect(parallel.workerCount).toBe(2);
+      for (const { levels } of [reference, segmented, parallel]) {
+        expect(levels).toHaveLength(FRAME_COUNT);
+      }
+    });
+
+    it.each([
+      ["a segment", () => segmented],
+      ["a parallel worker", () => parallel],
+    ])("renders the first frame of %s inside the clip like one session does", (_, shape) => {
+      const level = shape().levels[SECOND_SESSION_FIRST]!;
+      expect(Math.abs(level - reference.levels[SECOND_SESSION_FIRST]!)).toBeLessThan(3);
+    });
+
+    it("grades every frame of the clip, including its first", () => {
+      const graded = reference.levels[CLIP_FIRST + 5]!;
+      // Non-vacuous only if the grade visibly moves the clip off its source grey.
+      expect(CLIP_GREY - graded).toBeGreaterThan(30);
+      for (const { levels } of [reference, segmented, parallel]) {
+        for (let i = CLIP_FIRST; i <= CLIP_LAST; i++) {
+          expect(Math.abs(levels[i]! - graded), `frame ${i}`).toBeLessThan(3);
+        }
+      }
+    });
+  },
+);

scripts/test-reachability.json

@@ -61,7 +61,7 @@
       ],
       "sources": {
         "packages/producer/scripts/run-test-lane.mjs": "638885e14480d03290803389325a655f3749442167d784e07bb3af3404598af7",
-        "packages/producer/scripts/test-classification.mjs": "cdfa4035d0e2c0b1fea4d4c5b4697e0287d48c4474ab105f5e3d1ce839cca500",
+        "packages/producer/scripts/test-classification.mjs": "11102d43fdae5ab611c2f840e3d7fb4f367683fbddf9dfc770e0a623deb6aef2",
         ".github/workflows/ci.yml#job#producer-source-tests": "45da72aa6cf08cdfd5bcf5311e89f12af4842f9eebc6735df19da8c02f4d4864",
         "packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1",
         "packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",