fix(core): show a scene's starting state on a render's first frame (#5614)

* fix(core): show a scene's starting state on a render's first frame

A seek to the time a GSAP timeline already sits at renders nothing, so a
sub-composition timeline at 0 never applied its t=0 steps before frame 0
was captured. Every runtime seek now goes through seekGsapTimeline, which
redraws the timeline silently when the seek did not move it.

* test(producer): share the frame-level reader across render tests

* fix(core): name the in-place render skip instead of commenting it

* perf(core): redraw a parked scene timeline once instead of every frame

* fix(core): redraw only a scene timeline GSAP has never drawn

packages/core/src/runtime/adapters/gsap.steps.test.ts

@@ -1,7 +1,7 @@
 // @vitest-environment happy-dom
 import gsap from "gsap";
 import { describe, expect, it } from "vitest";
-import { createGsapAdapter, rerenderGsapTimelineAt } from "./gsap";
+import { createGsapAdapter, rerenderGsapTimelineAt, seekGsapTimeline } from "./gsap";
 import type { RuntimeTimelineLike } from "../types";
 
 // Every 0.1 s: hide all three frames, then show one, as frame-by-frame films do.
@@ -531,3 +531,51 @@ describe("gsap adapter just after a tween's start", () => {
     }
   });
 });
+
+describe("seekGsapTimeline", () => {
+  function hiddenAtZero() {
+    const word = document.body.appendChild(document.createElement("span"));
+    let calls = 0;
+    const timeline = gsap.timeline({ paused: true });
+    timeline.set(word, { visibility: "hidden" }, 0);
+    timeline.call(() => void calls++, [], 0);
+    timeline.set(word, { visibility: "visible" }, 0.5);
+    return { word, timeline: timeline as unknown as RuntimeTimelineLike, calls: () => calls };
+  }
+
+  it("renders a never-rendered timeline seeked to the time it sits at", () => {
+    const film = hiddenAtZero();
+    seekGsapTimeline(film.timeline, 0, false);
+    expect(film.word.style.visibility).toBe("hidden");
+  });
+
+  it("redraws a timeline parked at one time once, not on every seek", () => {
+    const film = hiddenAtZero();
+    const real = film.timeline;
+    let moves = 0;
+    const counted = {
+      ...real,
+      time: () => real.time(),
+      totalTime: (time?: number, suppressEvents?: boolean) => {
+        if (time === undefined) return real.totalTime!();
+        moves++;
+        return real.totalTime!(time, suppressEvents);
+      },
+      getChildren: real.getChildren!.bind(real),
+    } as RuntimeTimelineLike;
+    seekGsapTimeline(counted, 0, false);
+    const firstSeekMoves = moves;
+    seekGsapTimeline(counted, 0, false);
+    seekGsapTimeline(counted, 0, false);
+    expect(firstSeekMoves).toBeGreaterThan(1);
+    expect(moves - firstSeekMoves).toBe(2);
+  });
+
+  it("fires a call at that time once across the next seeks", () => {
+    const film = hiddenAtZero();
+    seekGsapTimeline(film.timeline, 0, false);
+    seekGsapTimeline(film.timeline, 0, false);
+    seekGsapTimeline(film.timeline, 0.1, false);
+    expect(film.calls()).toBe(1);
+  });
+});

packages/core/src/runtime/adapters/gsap.ts

@@ -49,6 +49,37 @@ export function rerenderGsapTimelineAt(
   }
 }
 
+type GsapRenderState = { _initted?: number };
+
+function gsapNeverRendered(timeline: RuntimeTimelineLike): boolean {
+  const [child] = (timeline.getChildren?.(false, true, true) ?? []) as { parent?: unknown }[];
+  const rendered = child?.parent as GsapRenderState | undefined;
+  return rendered !== undefined && !rendered._initted;
+}
+
+export function seekGsapTimeline(
+  timeline: RuntimeTimelineLike,
+  t: number,
+  suppressEvents: boolean,
+): boolean {
+  if (typeof timeline.totalTime !== "function") {
+    timeline.seek(t, suppressEvents);
+    return false;
+  }
+  const gsapSkipsRenderInPlace = Number(timeline.totalTime()) === t;
+  timeline.totalTime(t, suppressEvents);
+  if (!gsapSkipsRenderInPlace || !gsapNeverRendered(timeline)) return false;
+  rerenderGsapTimelineAt(
+    {
+      totalTime: timeline.totalTime.bind(timeline),
+      totalDuration: timeline.totalDuration?.bind(timeline),
+      getChildren: timeline.getChildren?.bind(timeline),
+    },
+    t,
+  );
+  return true;
+}
+
 type GsapParent = {
   _dur: number;
   _tDur: number;

packages/core/src/runtime/init.test.ts

@@ -3119,7 +3119,7 @@ describe("initSandboxRuntimeModular", () => {
     const rootTimeline = createMockTimeline(10);
     const originalTotalTime = rootTimeline.totalTime;
     rootTimeline.totalTime = (time: number, suppressEvents?: boolean) => {
-      seekCalls.push({ time, suppressEvents });
+      if (time !== undefined) seekCalls.push({ time, suppressEvents });
       return originalTotalTime?.(time, suppressEvents);
     };
 
@@ -3155,7 +3155,7 @@ describe("initSandboxRuntimeModular", () => {
     const rootTimeline = createMockTimeline(10);
     const originalTotalTime = rootTimeline.totalTime;
     rootTimeline.totalTime = (time: number, suppressEvents?: boolean) => {
-      seekCalls.push({ time, suppressEvents });
+      if (time !== undefined) seekCalls.push({ time, suppressEvents });
       return originalTotalTime?.(time, suppressEvents);
     };
     Object.assign(rootTimeline, {
@@ -3177,6 +3177,26 @@ describe("initSandboxRuntimeModular", () => {
     expect(seekCalls).toEqual([{ time: 2, suppressEvents: false }]);
   });
 
+  it("keeps a later root tween's value over a nested scene the root already drew", () => {
+    document.body.innerHTML = `<div data-composition-id="main" data-root="true" data-start="0" data-duration="2" data-width="1920" data-height="1080"><div data-composition-id="child" data-start="0" data-duration="2"><div id="box"></div></div></div>`;
+    const box = document.getElementById("box")!;
+    const options = { duration: 2, ease: "none", immediateRender: false, lazy: false };
+    const child = gsap.timeline({ smoothChildTiming: false });
+    child.fromTo(box, { opacity: 0 }, { ...options, opacity: 1, overwrite: false }, 0);
+    const main = gsap.timeline({ paused: true, smoothChildTiming: false });
+    main.add(child, 0);
+    main.fromTo(box, { opacity: 0.8 }, { ...options, opacity: 1, overwrite: false }, 0);
+    window.__timelines = { main, child };
+    initSandboxRuntimeModular();
+
+    window.__player?.renderSeek(1);
+    expect(Number(box.style.opacity)).toBeCloseTo(0.9, 5);
+
+    child.set({}, {}, 1.5);
+    window.__player?.renderSeek(1);
+    expect(Number(box.style.opacity)).toBeCloseTo(0.9, 5);
+  });
+
   it("fires a call added after the first seek exactly once", () => {
     const root = document.createElement("div");
     root.setAttribute("data-composition-id", "main");

packages/core/src/runtime/init.ts

@@ -16,7 +16,7 @@ import {
 import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics";
 import { injectCompositionCssVariables } from "./getVariables";
 import { createCssAdapter } from "./adapters/css";
-import { createGsapAdapter, rerenderGsapTimelineAt } from "./adapters/gsap";
+import { createGsapAdapter, rerenderGsapTimelineAt, seekGsapTimeline } from "./adapters/gsap";
 import { createAnimeJsAdapter } from "./adapters/animejs";
 import { createLottieAdapter } from "./adapters/lottie";
 import { createThreeAdapter } from "./adapters/three";
@@ -1663,8 +1663,7 @@ export function initSandboxRuntimeModular(): void {
       // Without this, children added after the root was created may still show initial styles.
       if (autoNestedChildren.length > 0) {
         try {
-          const currentTime = rootTimeline.time();
-          rootTimeline.seek(currentTime, false); // false = don't suppress events
+          seekGsapTimeline(rootTimeline, rootTimeline.time(), false);
         } catch {
           /* ignore */
         }
@@ -2323,7 +2322,7 @@ export function initSandboxRuntimeModular(): void {
       // abort restoring the transport time below (the catch would swallow it too).
       pauseTimelineIfPossible(state.capturedTimeline);
       if (typeof state.capturedTimeline.seek === "function") {
-        state.capturedTimeline.seek(previousTime, false);
+        seekGsapTimeline(state.capturedTimeline, previousTime, false);
       }
     } catch (err) {
       // keep runtime resilient even if a timeline implementation throws
@@ -4240,11 +4239,7 @@ export function initSandboxRuntimeModular(): void {
       const suppressEvents = options?.suppressEvents === true;
       // Guarded: a partial timeline without pause() must still get its seek.
       pauseTimelineIfPossible(timeline);
-      if (typeof timeline.totalTime === "function") {
-        timeline.totalTime(timeSeconds, suppressEvents);
-      } else {
-        timeline.seek(timeSeconds, suppressEvents);
-      }
+      seekGsapTimeline(timeline, timeSeconds, suppressEvents);
     } catch (err) {
       swallow(swallowLabel, err);
     }
@@ -4399,8 +4394,8 @@ export function initSandboxRuntimeModular(): void {
       }
       try {
         if (typeof tl.totalTime === "function") {
-          tl.totalTime(tlSeekTime, suppressEvents);
-          if (!suppressEvents && !hasZeroDurationCallbackTween(tl)) {
+          const redrawn = seekGsapTimeline(tl, tlSeekTime, suppressEvents);
+          if (!redrawn && !suppressEvents && !hasZeroDurationCallbackTween(tl)) {
             // The first seek is the only eventful one; the re-render only refreshes styles.
             rerenderGsapTimelineAt(
               {

packages/producer/scripts/test-classification.mjs

@@ -30,6 +30,7 @@ const INTEGRATION_TEST_FILES = new Set([
   "src/services/assetMediaType.test.ts",
   "src/services/htmlCompiler.mediaType.test.ts",
   "src/services/htmlCompiler.naturalDuration.test.ts",
+  "src/services/render/firstFrameStartingState.integration.test.ts",
   "src/services/render/hlsRender.integration.test.ts",
   "src/services/render/motionBlurVideo.integration.test.ts",
   "src/services/render/renderCancel.integration.test.ts",

packages/producer/src/services/render/firstFrameStartingState.integration.test.ts

@@ -0,0 +1,114 @@
+// Frame 0 of a real render shows the starting state that a sub-composition's timeline sets at
+// t=0, not the page's pre-timeline styles. Needs Chrome + ffmpeg: integration lane.
+
+import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { resolveConfig } from "@hyperframes/engine";
+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";
+import { HAS_FFMPEG, frameLevels } from "./frameLevelsTestFixtures.js";
+
+const WIDTH = 160;
+const HEIGHT = 90;
+const FPS = 30;
+const REVEAL_FRAME = 15;
+
+const INDEX = `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <script src="assets/gsap.min.js"></script>
+  <style>* { margin: 0; padding: 0; } body { overflow: hidden; background: #000; }</style>
+</head>
+<body>
+  <div id="root" data-composition-id="main" data-start="0" data-duration="1"
+    data-width="${WIDTH}" data-height="${HEIGHT}" data-fps="${FPS}">
+    <div id="s-words" data-composition-id="words" data-composition-src="compositions/words.html"
+      data-start="0" data-duration="1" data-track-index="0"
+      data-width="${WIDTH}" data-height="${HEIGHT}"></div>
+  </div>
+  <script>window.__timelines["main"] = gsap.timeline({ paused: true });</script>
+</body>
+</html>
+`;
+
+// The words have no hidden CSS state: only the timeline's t=0 steps hide them.
+const WORDS = `<!DOCTYPE html>
+<html>
+<body>
+<template id="words-template">
+  <style>
+    @font-face { font-family: "Fixture"; src: url("assets/fixture.woff2") format("woff2"); font-weight: 500; }
+    #words { position: absolute; inset: 0; background: #000; }
+    #words .w { position: absolute; top: 20px; font: 500 40px "Fixture", serif; color: #fff; }
+  </style>
+  <div id="words" data-composition-id="words" data-width="${WIDTH}" data-height="${HEIGHT}" data-duration="1">
+    <span class="w">Aa</span><span class="w">Bb</span>
+  </div>
+  <script>
+    (() => {
+      const tl = gsap.timeline({ paused: true });
+      const words = [...document.querySelectorAll("#words .w")];
+      words.forEach((w) => tl.set(w, { visibility: "hidden" }, 0));
+      words.forEach((w, i) => tl.set(w, { visibility: "visible", left: 10 + i * 70 }, ${REVEAL_FRAME / FPS}));
+      tl.set({}, {}, 1);
+      window.__timelines["words"] = tl;
+    })();
+  </script>
+</template>
+</body>
+</html>
+`;
+
+let root: string;
+
+describe.skipIf(!HAS_FFMPEG)(
+  "first frame of a real render — sub-composition starting state",
+  () => {
+    let levels: number[];
+
+    beforeAll(async () => {
+      root = mkdtempSync(join(tmpdir(), "hf-first-frame-"));
+      mkdirSync(join(root, "assets"), { recursive: true });
+      mkdirSync(join(root, "compositions"), { recursive: true });
+      copyFileSync(
+        resolve(import.meta.dirname, "../../../../core/node_modules/gsap/dist/gsap.min.js"),
+        join(root, "assets/gsap.min.js"),
+      );
+      copyFileSync(
+        resolve(
+          import.meta.dirname,
+          "../../../tests/heygen-promo-preview-assets/src/TT_Norms_Pro_Medium.woff2",
+        ),
+        join(root, "assets/fixture.woff2"),
+      );
+      writeFileSync(join(root, "index.html"), INDEX);
+      writeFileSync(join(root, "compositions/words.html"), WORDS);
+      const job = createRenderJob({
+        fps: FPS,
+        quality: "draft",
+        format: "mp4",
+        hdrMode: "force-sdr",
+        workers: 1,
+        producerConfig: resolveConfig({ browserGpuMode: "software" }),
+      });
+      const output = join(root, "out.mp4");
+      await executeRenderJob(job, root, output);
+      levels = frameLevels(output, WIDTH, HEIGHT);
+    }, 120_000);
+
+    afterAll(() => {
+      if (root) rmSync(root, { recursive: true, force: true });
+    });
+
+    it("frame 0 matches frame 1: the words are already hidden", () => {
+      expect(levels[0]).toBeCloseTo(levels[1]!, 0);
+      expect(levels[0]).toBeLessThan(1);
+    });
+
+    it("the words appear when the timeline reveals them", () => {
+      expect(levels[REVEAL_FRAME]! - levels[0]!).toBeGreaterThan(2);
+    });
+  },
+);

packages/producer/src/services/render/frameLevelsTestFixtures.ts

@@ -0,0 +1,23 @@
+import { spawnSync } from "node:child_process";
+import { getFfmpegBinary } from "@hyperframes/engine";
+
+export const FFMPEG = getFfmpegBinary();
+export const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"], { encoding: "utf-8" }).status === 0;
+
+/** Mean grey level of every frame of an encoded video, 0-255. */
+export function frameLevels(videoPath: string, width: number, height: number): number[] {
+  const raw = spawnSync(
+    FFMPEG,
+    ["-v", "error", "-i", videoPath, "-f", "rawvideo", "-pix_fmt", "gray", "-"],
+    { maxBuffer: 64 * 1024 * 1024 },
+  );
+  if (raw.status !== 0) throw new Error(`decode failed: ${raw.stderr}`);
+  const frameBytes = width * height;
+  const levels: number[] = [];
+  for (let offset = 0; offset + frameBytes <= raw.stdout.length; offset += frameBytes) {
+    let sum = 0;
+    for (let i = offset; i < offset + frameBytes; i++) sum += raw.stdout[i]!;
+    levels.push(sum / frameBytes);
+  }
+  return levels;
+}

packages/producer/src/services/render/segmentedColorGrading.integration.test.ts

@@ -7,11 +7,9 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { getFfmpegBinary, resolveConfig } from "@hyperframes/engine";
+import { resolveConfig } from "@hyperframes/engine";
 import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";
-
-const FFMPEG = getFfmpegBinary();
-const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"], { encoding: "utf-8" }).status === 0;
+import { FFMPEG, HAS_FFMPEG, frameLevels } from "./frameLevelsTestFixtures.js";
 
 const WIDTH = 160;
 const HEIGHT = 90;
@@ -55,26 +53,6 @@ const COMPOSITION = `<!DOCTYPE html>
 
 let root: string;
 
-/** Mean RGB level of every frame of an encoded video, 0-255. */
-function frameLevels(videoPath: string): number[] {
-  const raw = spawnSync(
-    FFMPEG,
-    ["-v", "error", "-i", videoPath, "-f", "rawvideo", "-pix_fmt", "gray", "-"],
-    {
-      maxBuffer: 64 * 1024 * 1024,
-    },
-  );
-  if (raw.status !== 0) throw new Error(`decode failed: ${raw.stderr}`);
-  const frameBytes = WIDTH * HEIGHT;
-  const levels: number[] = [];
-  for (let offset = 0; offset + frameBytes <= raw.stdout.length; offset += frameBytes) {
-    let sum = 0;
-    for (let i = offset; i < offset + frameBytes; i++) sum += raw.stdout[i]!;
-    levels.push(sum / frameBytes);
-  }
-  return levels;
-}
-
 interface Render {
   levels: number[];
   phases: string[];
@@ -112,7 +90,7 @@ async function render(name: string, segmented: boolean, workers: number): Promis
   try {
     const output = join(root, `${name}.mp4`);
     await executeRenderJob(job, root, output);
-    return { levels: frameLevels(output), phases, workerCount };
+    return { levels: frameLevels(output, WIDTH, HEIGHT), phases, workerCount };
   } finally {
     if (saved.capture === undefined) delete process.env.HF_SEGMENTED_CAPTURE;
     else process.env.HF_SEGMENTED_CAPTURE = saved.capture;

scripts/test-reachability.json

@@ -61,7 +61,7 @@
       ],
       "sources": {
         "packages/producer/scripts/run-test-lane.mjs": "638885e14480d03290803389325a655f3749442167d784e07bb3af3404598af7",
-        "packages/producer/scripts/test-classification.mjs": "11102d43fdae5ab611c2f840e3d7fb4f367683fbddf9dfc770e0a623deb6aef2",
+        "packages/producer/scripts/test-classification.mjs": "18ec188edb061f2a729d261a64fb22682064e4f438413fdeb4d502e709cc8684",
         ".github/workflows/ci.yml#job#producer-source-tests": "45da72aa6cf08cdfd5bcf5311e89f12af4842f9eebc6735df19da8c02f4d4864",
         "packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1",
         "packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",