Merge pull request #717 from heygen-com/refactor/producer-stages-1.1-scaffold-planhash

refactor(producer): scaffold services/render/stages/ + planHash utility

packages/producer/src/services/render/stages/freezePlan.ts

@@ -0,0 +1,106 @@
+/**
+ * freezePlan — write the meta/{composition,encoder,chunks}.json + plan.json
+ * manifest at the end of `plan()`, compute the planHash from the frozen
+ * artifacts, and return the manifest path.
+ *
+ * Phase 1 PR 1.1 ships the signature only: there are no callers yet. The
+ * function body is added in Phase 3 PR 3.1 when `services/distributed/plan.ts`
+ * lands and composes the Phase-1 stages. Keeping the skeleton in this PR
+ * means subsequent stage-extraction PRs can grow the `stages/` directory
+ * without touching the `producer/src/index.ts` export surface again.
+ *
+ * See DISTRIBUTED-RENDERING-PLAN.md §2.1 phase 6, §4.1 directory layout,
+ * §4.3 LockedRenderConfig.
+ */
+
+import type { Fps } from "@hyperframes/core";
+import type { PlanDimensions } from "./planHash.js";
+
+/**
+ * The encoder configuration locked in at plan time. Mirrors §4.3
+ * LockedRenderConfig in the design doc. Phase 1 declares the shape; Phase 2
+ * + Phase 3 populate it from the existing `renderOrchestrator` config and
+ * the new closed-GOP encoder args.
+ */
+export interface LockedRenderConfig {
+  // Capture
+  captureMode: "beginframe" | "screenshot";
+  forceScreenshot: boolean;
+  deviceScaleFactor: number;
+  useLayeredHdrComposite: boolean;
+  /** Hard-pinned to "software" in v1 distributed renders. */
+  browserGpuMode: "software";
+  warmupTicks: number;
+
+  // Encode
+  encoder: "libx264-software" | "libx265-software" | "prores-software" | "png-sequence";
+  ffmpegVersion: string;
+  preset: string;
+  crf?: number;
+  bitrate?: string;
+  /** Equal to chunkSize for closed-GOP concat-copy. */
+  gopSize: number;
+  closedGop: true;
+  forceKeyframes: "n=0";
+  pixelFormat: string;
+
+  // Chunking
+  chunkSize: number;
+  chunkCount: number;
+
+  /** Snapshot of `PRODUCER_RUNTIME_*` env vars at plan time. */
+  runtimeEnv: Record<string, string>;
+}
+
+export interface CompositionMetadataJson {
+  durationSeconds: number;
+  width: number;
+  height: number;
+  fps: Fps;
+  videoCount: number;
+  audioCount: number;
+  imageCount: number;
+}
+
+export interface ChunkSliceJson {
+  index: number;
+  startFrame: number;
+  /** Inclusive end frame for the chunk. */
+  endFrame: number;
+}
+
+/**
+ * Inputs to `freezePlan`. `planDir` already contains `compiled/`,
+ * `video-frames/`, and (optionally) `audio.aac` by the time freezePlan
+ * runs — see §2.1 phases 1-5.
+ */
+export interface FreezePlanInput {
+  /** Absolute path to the plan directory being frozen. */
+  planDir: string;
+  composition: CompositionMetadataJson;
+  encoder: LockedRenderConfig;
+  chunks: readonly ChunkSliceJson[];
+  dimensions: PlanDimensions;
+  producerVersion: string;
+  /** Hash of the deterministic-font snapshot baked into the plan. */
+  fontSnapshotSha: string;
+}
+
+export interface FreezePlanResult {
+  /** Absolute path to `plan.json`. */
+  planJsonPath: string;
+  /** Content-addressed planHash; see §4.2. */
+  planHash: string;
+}
+
+/**
+ * Freeze a plan directory: write `meta/*.json` + top-level `plan.json`, then
+ * compute `planHash` over the canonicalized contents.
+ *
+ * Skeleton only in Phase 1. Phase 3 PR 3.1 wires this up.
+ */
+export async function freezePlan(_input: FreezePlanInput): Promise<FreezePlanResult> {
+  throw new Error(
+    "freezePlan is not implemented yet — wired in Phase 3 PR 3.1 (see DISTRIBUTED-RENDERING-PLAN.md §11).",
+  );
+}

packages/producer/src/services/render/stages/planHash.test.ts

@@ -0,0 +1,253 @@
+import { describe, expect, it } from "bun:test";
+import {
+  canonicalJsonStringify,
+  computePlanHash,
+  sha256Hex,
+  type PlanAssetHash,
+  type PlanHashInput,
+} from "./planHash.js";
+
+function utf8(s: string): Uint8Array {
+  return new TextEncoder().encode(s);
+}
+
+function makeInput(overrides: Partial<PlanHashInput> = {}): PlanHashInput {
+  return {
+    compositionHtml: utf8("<!doctype html><html><body>hi</body></html>"),
+    assets: [
+      { path: "assets/a.png", sha256: "a".repeat(64) },
+      { path: "assets/b.png", sha256: "b".repeat(64) },
+    ],
+    fontSnapshotSha: "f".repeat(64),
+    encoderConfigCanonicalJson: '{"closedGop":true,"encoder":"libx264-software","gopSize":240}',
+    producerVersion: "0.5.7",
+    ffmpegVersion: "ffmpeg version 6.1.1",
+    dimensions: {
+      fpsNum: 30,
+      fpsDen: 1,
+      width: 1920,
+      height: 1080,
+      format: "mp4",
+    },
+    ...overrides,
+  };
+}
+
+describe("computePlanHash", () => {
+  it("is deterministic for identical inputs", () => {
+    const a = computePlanHash(makeInput());
+    const b = computePlanHash(makeInput());
+    expect(a).toBe(b);
+    expect(a).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  // Pin the schema-prefix-mixed-in result for one fixed input. If the
+  // framing of `computePlanHash` ever changes silently, this test must
+  // be updated (which means bumping `PLAN_HASH_SCHEMA_PREFIX` in the
+  // source too). Catches accidental drift across producer versions.
+  it("matches the known digest for a fixed reference input", () => {
+    expect(computePlanHash(makeInput())).toBe(
+      "995b4105a1a629965e85dc5d92c6aab9b888e39acdb14369d1bc781aa3247a94",
+    );
+  });
+
+  it("ignores asset order in the input array", () => {
+    const ordered = computePlanHash(
+      makeInput({
+        assets: [
+          { path: "assets/a.png", sha256: "a".repeat(64) },
+          { path: "assets/b.png", sha256: "b".repeat(64) },
+        ],
+      }),
+    );
+    const reversed = computePlanHash(
+      makeInput({
+        assets: [
+          { path: "assets/b.png", sha256: "b".repeat(64) },
+          { path: "assets/a.png", sha256: "a".repeat(64) },
+        ],
+      }),
+    );
+    expect(ordered).toBe(reversed);
+  });
+
+  it("changes when composition HTML changes", () => {
+    const a = computePlanHash(makeInput());
+    const b = computePlanHash(
+      makeInput({ compositionHtml: utf8("<!doctype html><body>bye</body>") }),
+    );
+    expect(a).not.toBe(b);
+  });
+
+  it("changes when any asset sha changes", () => {
+    const a = computePlanHash(makeInput());
+    const b = computePlanHash(
+      makeInput({
+        assets: [
+          { path: "assets/a.png", sha256: "a".repeat(64) },
+          { path: "assets/b.png", sha256: "c".repeat(64) }, // changed
+        ],
+      }),
+    );
+    expect(a).not.toBe(b);
+  });
+
+  it("changes when an asset path moves", () => {
+    const a = computePlanHash(makeInput());
+    const b = computePlanHash(
+      makeInput({
+        assets: [
+          { path: "assets/a.png", sha256: "a".repeat(64) },
+          { path: "assets/renamed.png", sha256: "b".repeat(64) },
+        ],
+      }),
+    );
+    expect(a).not.toBe(b);
+  });
+
+  it("distinguishes path-vs-sha boundary via delimiter framing", () => {
+    // Without delimiters, concatenating ("ab", "cd...") and ("abc", "d...")
+    // could hash equal because the byte stream is identical. With the
+    // 0x00 delimiter between fields the two inputs produce distinct hashes.
+    const a = computePlanHash(
+      makeInput({
+        assets: [{ path: "assets/ab", sha256: "cd" + "0".repeat(62) }],
+      }),
+    );
+    const b = computePlanHash(
+      makeInput({
+        assets: [{ path: "assets/abc", sha256: "d" + "0".repeat(63) }],
+      }),
+    );
+    expect(a).not.toBe(b);
+  });
+
+  it("changes when font snapshot changes", () => {
+    const a = computePlanHash(makeInput());
+    const b = computePlanHash(makeInput({ fontSnapshotSha: "0".repeat(64) }));
+    expect(a).not.toBe(b);
+  });
+
+  it("changes when encoder config canonical JSON changes", () => {
+    const a = computePlanHash(makeInput());
+    const b = computePlanHash(
+      makeInput({ encoderConfigCanonicalJson: '{"encoder":"libx265-software"}' }),
+    );
+    expect(a).not.toBe(b);
+  });
+
+  it("changes when producer or ffmpeg version changes", () => {
+    const base = makeInput();
+    const a = computePlanHash(base);
+    const b = computePlanHash({ ...base, producerVersion: "0.5.8" });
+    const c = computePlanHash({ ...base, ffmpegVersion: "ffmpeg version 7.0.0" });
+    expect(a).not.toBe(b);
+    expect(a).not.toBe(c);
+    expect(b).not.toBe(c);
+  });
+
+  it("changes when dimensions or fps change", () => {
+    const a = computePlanHash(makeInput());
+    const b = computePlanHash(
+      makeInput({
+        dimensions: { fpsNum: 60, fpsDen: 1, width: 1920, height: 1080, format: "mp4" },
+      }),
+    );
+    const c = computePlanHash(
+      makeInput({
+        dimensions: { fpsNum: 30, fpsDen: 1, width: 3840, height: 2160, format: "mp4" },
+      }),
+    );
+    const d = computePlanHash(
+      makeInput({
+        dimensions: { fpsNum: 30, fpsDen: 1, width: 1920, height: 1080, format: "mov" },
+      }),
+    );
+    expect(new Set([a, b, c, d]).size).toBe(4);
+  });
+
+  it("does not collide on empty assets", () => {
+    const a = computePlanHash(makeInput({ assets: [] }));
+    expect(a).toMatch(/^[0-9a-f]{64}$/);
+    const b = computePlanHash(makeInput({ assets: [{ path: "", sha256: "0".repeat(64) }] }));
+    expect(a).not.toBe(b);
+  });
+
+  it("does not mutate the input assets array", () => {
+    const assets: PlanAssetHash[] = [
+      { path: "b", sha256: "b".repeat(64) },
+      { path: "a", sha256: "a".repeat(64) },
+    ];
+    const snapshot = assets.map((a) => ({ ...a }));
+    computePlanHash(makeInput({ assets }));
+    expect(assets).toEqual(snapshot);
+  });
+});
+
+describe("canonicalJsonStringify", () => {
+  it("sorts object keys byte-wise", () => {
+    expect(canonicalJsonStringify({ b: 1, a: 2 })).toBe('{"a":2,"b":1}');
+  });
+
+  it("preserves array order", () => {
+    expect(canonicalJsonStringify([3, 1, 2])).toBe("[3,1,2]");
+  });
+
+  it("recurses into nested structures", () => {
+    expect(
+      canonicalJsonStringify({
+        z: [{ b: 1, a: 2 }, "x"],
+        a: { y: true, x: null },
+      }),
+    ).toBe('{"a":{"x":null,"y":true},"z":[{"a":2,"b":1},"x"]}');
+  });
+
+  it("escapes strings via JSON.stringify", () => {
+    expect(canonicalJsonStringify('he said "hi"')).toBe('"he said \\"hi\\""');
+  });
+
+  it("rejects non-finite numbers", () => {
+    expect(() => canonicalJsonStringify(Number.NaN)).toThrow(TypeError);
+    expect(() => canonicalJsonStringify(Number.POSITIVE_INFINITY)).toThrow(TypeError);
+  });
+
+  it("rejects unsupported value types", () => {
+    expect(() => canonicalJsonStringify(() => 1)).toThrow(TypeError);
+    expect(() => canonicalJsonStringify(Symbol("s"))).toThrow(TypeError);
+  });
+
+  it("rejects undefined at the top level", () => {
+    expect(() => canonicalJsonStringify(undefined)).toThrow(TypeError);
+  });
+
+  it("produces equal output for semantically equal objects with different key order", () => {
+    const a = canonicalJsonStringify({
+      encoder: "libx264-software",
+      gopSize: 240,
+      closedGop: true,
+    });
+    const b = canonicalJsonStringify({
+      gopSize: 240,
+      closedGop: true,
+      encoder: "libx264-software",
+    });
+    expect(a).toBe(b);
+  });
+});
+
+describe("sha256Hex", () => {
+  it("matches the well-known empty-string digest", () => {
+    expect(sha256Hex("")).toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+  });
+
+  it("matches the well-known abc digest", () => {
+    expect(sha256Hex("abc")).toBe(
+      "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+    );
+  });
+
+  it("treats string and equivalent Uint8Array the same", () => {
+    const s = "hyperframes";
+    expect(sha256Hex(s)).toBe(sha256Hex(new TextEncoder().encode(s)));
+  });
+});

packages/producer/src/services/render/stages/planHash.ts

@@ -0,0 +1,172 @@
+/**
+ * planHash — content-addressed hash for distributed render plans.
+ *
+ * See DISTRIBUTED-RENDERING-PLAN.md §4.2 for the contract:
+ *
+ *   planHash = sha256(
+ *     SCHEMA_PREFIX
+ *     ⊕ composition_html_bytes
+ *     ⊕ asset_shas (sorted by relative path)
+ *     ⊕ font_snapshot_sha
+ *     ⊕ encoder_config_canonical_json
+ *     ⊕ producer_version
+ *     ⊕ ffmpeg_version
+ *     ⊕ fps ⊕ width ⊕ height ⊕ format
+ *   )
+ *
+ * Two invocations with identical inputs MUST produce the same hash. Adapters
+ * use this to short-circuit `plan()` on workflow replay and to detect
+ * cross-version mismatches (§9.3 PLAN_HASH_MISMATCH).
+ *
+ * Pure utility; no caller exists yet — the distributed-render
+ * `services/distributed/plan.ts` will compose it.
+ *
+ * ## Encoding contract
+ *
+ * Every string-typed component (`fontSnapshotSha`,
+ * `encoderConfigCanonicalJson`, `producerVersion`, `ffmpegVersion`, asset
+ * paths and shas, the dimensions tuple) is hashed as UTF-8. External
+ * verifiers must encode the same way. Binary fields (`compositionHtml`)
+ * are hashed verbatim.
+ */
+
+import { createHash } from "node:crypto";
+
+/**
+ * Schema-version prefix mixed into every digest. Bump the trailing version
+ * integer whenever the framing of `computePlanHash` changes (new fields,
+ * new delimiter, new field order, etc.) so every cached plan from an older
+ * producer is forced to mismatch and re-plan. This is impossible to
+ * backfill, so a deliberate bump is the only correct action.
+ */
+const PLAN_HASH_SCHEMA_PREFIX = "hyperframes-plan-hash-v1\x00";
+
+/**
+ * 0x00 byte used to frame each `hash.update()` call. Hoisted to module
+ * scope so it's not reallocated on every `computePlanHash` invocation.
+ */
+const FIELD_DELIMITER = Buffer.from([0x00]);
+
+/**
+ * SHA-256 hex digest of an asset, paired with its plan-relative path. Sort
+ * order across an asset list is by `path` (byte-wise ascending) to keep the
+ * digest deterministic regardless of filesystem walk order.
+ */
+export interface PlanAssetHash {
+  /** Plan-relative path. Stable across machines (no absolute paths). */
+  path: string;
+  /** Hex-encoded sha256 of the asset bytes. */
+  sha256: string;
+}
+
+/**
+ * Render dimensions + frame rate that affect the encoded output. Kept as a
+ * separate type so callers can reuse it for log lines and adapter payloads.
+ */
+export interface PlanDimensions {
+  /** Frame rate numerator (e.g. 30 or 30000 for NTSC). */
+  fpsNum: number;
+  /** Frame rate denominator (e.g. 1 or 1001 for NTSC). */
+  fpsDen: number;
+  width: number;
+  height: number;
+  format: "mp4" | "mov" | "png-sequence" | "webm";
+}
+
+export interface PlanHashInput {
+  /** Raw bytes of `compiled/index.html` after recompile. */
+  compositionHtml: Uint8Array;
+  /** All non-HTML assets referenced from the composition, in any order. */
+  assets: readonly PlanAssetHash[];
+  /** Hash of the deterministic-font snapshot used to render. */
+  fontSnapshotSha: string;
+  /** Canonical-JSON serialization of `meta/encoder.json` (LockedRenderConfig). */
+  encoderConfigCanonicalJson: string;
+  /** `@hyperframes/producer` package version that produced the plan. */
+  producerVersion: string;
+  /** ffmpeg `--version` line (e.g. "ffmpeg version 6.1.1"). */
+  ffmpegVersion: string;
+  dimensions: PlanDimensions;
+}
+
+/**
+ * Compute the content-addressed planHash for a frozen plan.
+ *
+ * The hash incorporates each component as a separate `update()` call after a
+ * fixed delimiter byte; that prevents two distinct inputs from accidentally
+ * sharing a hash if their concatenation happens to collide (e.g. asset count
+ * vs. asset bytes).
+ */
+export function computePlanHash(input: PlanHashInput): string {
+  const hash = createHash("sha256");
+
+  hash.update(PLAN_HASH_SCHEMA_PREFIX, "utf8");
+
+  hash.update(input.compositionHtml);
+  hash.update(FIELD_DELIMITER);
+
+  const sortedAssets = [...input.assets].sort((a, b) =>
+    a.path < b.path ? -1 : a.path > b.path ? 1 : 0,
+  );
+  for (const asset of sortedAssets) {
+    hash.update(asset.path, "utf8");
+    hash.update(FIELD_DELIMITER);
+    hash.update(asset.sha256, "utf8");
+    hash.update(FIELD_DELIMITER);
+  }
+
+  hash.update(input.fontSnapshotSha, "utf8");
+  hash.update(FIELD_DELIMITER);
+  hash.update(input.encoderConfigCanonicalJson, "utf8");
+  hash.update(FIELD_DELIMITER);
+  hash.update(input.producerVersion, "utf8");
+  hash.update(FIELD_DELIMITER);
+  hash.update(input.ffmpegVersion, "utf8");
+  hash.update(FIELD_DELIMITER);
+
+  const d = input.dimensions;
+  hash.update(`${d.fpsNum}/${d.fpsDen}x${d.width}x${d.height}x${d.format}`, "utf8");
+
+  return hash.digest("hex");
+}
+
+/**
+ * Canonical-JSON serialization helper. JSON keys are emitted in
+ * byte-wise-sorted order recursively, with no whitespace. Used to feed the
+ * encoder config into `computePlanHash` such that semantically-equal configs
+ * produce equal hashes regardless of source key ordering.
+ *
+ * Supports the subset that LockedRenderConfig values use: primitives, plain
+ * objects, and arrays. Throws on functions, symbols, BigInts, and Maps.
+ */
+export function canonicalJsonStringify(value: unknown): string {
+  if (value === null) return "null";
+  if (typeof value === "boolean") return value ? "true" : "false";
+  if (typeof value === "number") {
+    if (!Number.isFinite(value)) {
+      throw new TypeError(`canonicalJsonStringify: non-finite number ${value}`);
+    }
+    return JSON.stringify(value);
+  }
+  if (typeof value === "string") return JSON.stringify(value);
+  if (Array.isArray(value)) {
+    return `[${value.map(canonicalJsonStringify).join(",")}]`;
+  }
+  if (typeof value === "object") {
+    const obj = value as Record<string, unknown>;
+    const keys = Object.keys(obj).sort();
+    const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalJsonStringify(obj[k])}`);
+    return `{${parts.join(",")}}`;
+  }
+  throw new TypeError(`canonicalJsonStringify: unsupported value type ${typeof value}`);
+}
+
+/**
+ * Convenience helper: sha256 a file path or buffer, return hex digest. Used
+ * by the eventual `freezePlan` to hash assets on disk.
+ */
+export function sha256Hex(bytes: Uint8Array | string): string {
+  const h = createHash("sha256");
+  h.update(bytes);
+  return h.digest("hex");
+}