mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 06:30:03 +00:00
refactor(producer): share plan execution builder (#2906)
## What Refactor distributed planning around one shared local execution-plan builder: - `buildLocalExecutionPlan()` now owns compile/probe/extract/audio/freeze. - Legacy `plan()` remains a deprecated v1 transport wrapper. - Plan v2 calls the shared builder directly and publishes through the existing manifest/CAS contract. - Add neutral `createPlanV2FromExecutionPlan()`, `publishPlanV2FromExecutionPlan()`, `getPlanV2ExecutionPlanHash()`, and `PLAN_PROTOCOL_V1` names. - Retain deprecated v1-named exports and wire aliases. - Recommend explicit Plan v2 opt-in for new producer, Lambda, and Cloud Run integrations. ## Why Plan v2 previously looked like it invoked a v1 planner even though v1 and v2 share the same frozen local execution representation. This removes that migration-era coupling while preserving the public minor-version compatibility contract. ## How The shared builder returns neutral internal execution-plan fields. The v1 wrapper maps those fields back to the existing `PlanResult`; the v2 publisher consumes them directly. Compatibility is intentional and covered by exact shape tests: - omitted `planProtocol` still serializes/selects `"v1"`; - v1 layouts, descriptor-less decoding, event unions, workflow branches, and exports remain; - the v1 descriptor JSON is byte-identical and `CURRENT_PLAN_PROTOCOL` is an identity-preserving alias; - v2 manifest bytes, key order, hash framing, and `sourcePlanV1Hash` wire key remain unchanged; - no enumerable neutral hash field was added to manifests or returned result objects; - v1/v2 result objects, cloud event payloads, and SDK handle key sets remain unchanged. ## Test plan - Focused Plan v1/v2/protocol/export/size compatibility: 141 passed - `@hyperframes/core`: 1,419 passed - `@hyperframes/producer` unit lane: 990 passed - `@hyperframes/aws-lambda`: 140 passed - `@hyperframes/gcp-cloud-run`: 101 passed - Producer, Lambda, and Cloud Run typechecks - Repository-wide lint, format check, workspace/package-subpath checks - Full workspace build - `git diff --check` - [x] Unit tests added/updated - [ ] Manual testing performed - [x] Documentation updated (if applicable)
This commit is contained in:
James Russo
@@ -1,29 +1,29 @@
|
||||
/**
|
||||
* `@hyperframes/producer/distributed` — the distributed render primitives.
|
||||
*
|
||||
* The three activities (`plan` → `renderChunk` × N → `assemble`) are pure
|
||||
* functions over local file paths; networking + orchestration live in
|
||||
* adapters.
|
||||
* The distributed activities are pure functions over local file paths;
|
||||
* networking + orchestration live in adapters. New integrations should use
|
||||
* Plan v2; the v1 functions remain available for compatibility.
|
||||
*
|
||||
* Adopters (AWS Lambda, Cloud Run Jobs, Temporal, K8s Jobs, plain SSH):
|
||||
*
|
||||
* ```ts
|
||||
* import {
|
||||
* plan,
|
||||
* renderChunk,
|
||||
* assemble,
|
||||
* planV2,
|
||||
* renderChunkV2,
|
||||
* assembleV2,
|
||||
* } from "@hyperframes/producer/distributed";
|
||||
*
|
||||
* // Controller-side: produce a self-contained planDir + content-addressed planHash.
|
||||
* const planResult = await plan(projectDir, config, planDir);
|
||||
* // Controller-side: publish a content-addressed Plan v2 manifest + CAS.
|
||||
* const planResult = await planV2(projectDir, config, planV2Dir);
|
||||
*
|
||||
* // Worker-side: render one chunk. Byte-identical retries on the same
|
||||
* // (planDir, chunkIndex) — Temporal / Step Functions retry policies are
|
||||
* // (planV2Dir, chunkIndex) — Temporal / Step Functions retry policies are
|
||||
* // safe to point at this.
|
||||
* const chunk = await renderChunk(planDir, chunkIndex, outputChunkPath);
|
||||
* const chunk = await renderChunkV2(planV2Dir, chunkIndex, outputChunkPath);
|
||||
*
|
||||
* // Controller-side: stitch chunks into the final deliverable.
|
||||
* await assemble(planDir, chunkPaths, audioPath, outputPath);
|
||||
* await assembleV2(planV2Dir, chunkPaths, outputPath);
|
||||
* ```
|
||||
*
|
||||
* No networking, no AWS SDK, no Temporal SDK — those live in adapter
|
||||
@@ -56,11 +56,14 @@ export {
|
||||
|
||||
// ── Plan v2 content-addressed transport ────────────────────────────────────
|
||||
export {
|
||||
createPlanV2FromExecutionPlan,
|
||||
createPlanV2FromV1,
|
||||
getPlanV2ExecutionPlanHash,
|
||||
listPlanV2ArtifactsForTarget,
|
||||
materializePlanV2Target,
|
||||
planV2,
|
||||
planV2WithPublisher,
|
||||
publishPlanV2FromExecutionPlan,
|
||||
publishPlanV2FromV1,
|
||||
readPlanV2Manifest,
|
||||
validatePlanV2MaterializedTarget,
|
||||
@@ -123,6 +126,7 @@ export {
|
||||
getDistributedRenderCapabilities,
|
||||
PLAN_ARTIFACT_LAYOUT,
|
||||
PLAN_HASH_SCHEMA,
|
||||
PLAN_PROTOCOL_V1,
|
||||
PLAN_PROTOCOL_V2,
|
||||
PLAN_PROTOCOL_UNSUPPORTED,
|
||||
PLAN_SCHEMA_VERSION,
|
||||
|
||||
@@ -145,6 +145,7 @@ export {
|
||||
getDistributedRenderCapabilities,
|
||||
PLAN_ARTIFACT_LAYOUT,
|
||||
PLAN_HASH_SCHEMA,
|
||||
PLAN_PROTOCOL_V1,
|
||||
PLAN_PROTOCOL_V2,
|
||||
PLAN_PROTOCOL_UNSUPPORTED,
|
||||
PLAN_SCHEMA_VERSION,
|
||||
@@ -153,7 +154,9 @@ export {
|
||||
PLAN_V2_INTEGRITY_UNRECOVERABLE,
|
||||
PLAN_V2_MATERIALIZATION_MARKER,
|
||||
PLAN_V2_SCHEMA_VERSION,
|
||||
createPlanV2FromExecutionPlan,
|
||||
createPlanV2FromV1,
|
||||
getPlanV2ExecutionPlanHash,
|
||||
listPlanV2ArtifactsForTarget,
|
||||
materializePlanV2Target,
|
||||
plan,
|
||||
@@ -164,6 +167,7 @@ export {
|
||||
readPlanProtocol,
|
||||
readPlanProtocolV1,
|
||||
readPlanV2Manifest,
|
||||
publishPlanV2FromExecutionPlan,
|
||||
publishPlanV2FromV1,
|
||||
renderChunk,
|
||||
renderChunkV2,
|
||||
|
||||
@@ -25,6 +25,7 @@ import { RenderQualityError } from "../renderOrchestrator.js";
|
||||
import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";
|
||||
import {
|
||||
applyDistributedAudioWarningPolicy,
|
||||
buildLocalExecutionPlan,
|
||||
buildChunkSlices,
|
||||
DEFAULT_CHUNK_SIZE,
|
||||
DEFAULT_MAX_PARALLEL_CHUNKS,
|
||||
@@ -569,7 +570,7 @@ describe("plan() — golden planDir + planHash determinism", () => {
|
||||
);
|
||||
|
||||
it(
|
||||
"produces a byte-identical planHash on a second invocation",
|
||||
"shares one byte-identical execution plan between the builder and legacy v1 wrapper",
|
||||
async () => {
|
||||
const planDirA = join(runRoot, "plan-determinism-a");
|
||||
const planDirB = join(runRoot, "plan-determinism-b");
|
||||
@@ -577,12 +578,26 @@ describe("plan() — golden planDir + planHash determinism", () => {
|
||||
mkdirSync(planDirB, { recursive: true });
|
||||
|
||||
const config = { fps: 30 as const, width: 320, height: 240, format: "mp4" as const };
|
||||
const a = await plan(projectDir, config, planDirA);
|
||||
const a = await buildLocalExecutionPlan(projectDir, config, planDirA);
|
||||
const b = await plan(projectDir, config, planDirB);
|
||||
|
||||
expect(a.planHash).toBe(b.planHash);
|
||||
expect(a.executionPlanDir).toBe(planDirA);
|
||||
expect(a.executionPlanHash).toBe(b.planHash);
|
||||
expect(a.chunkCount).toBe(b.chunkCount);
|
||||
expect(a.totalFrames).toBe(b.totalFrames);
|
||||
expect(Object.keys(b)).toEqual([
|
||||
"planDir",
|
||||
"planProtocol",
|
||||
"planHash",
|
||||
"chunkCount",
|
||||
"totalFrames",
|
||||
"fps",
|
||||
"width",
|
||||
"height",
|
||||
"format",
|
||||
"ffmpegVersion",
|
||||
"producerVersion",
|
||||
]);
|
||||
|
||||
// Encoder JSON must be byte-identical — its bytes feed planHash, so any
|
||||
// drift here would silently change the hash framing.
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
/**
|
||||
* Activity A of the distributed render pipeline.
|
||||
*
|
||||
* `plan(projectDir, config, planDir)` composes the existing render stages
|
||||
* (compile → probe → extract videos → audio → freeze) into a self-contained
|
||||
* `<planDir>/` directory tree that downstream chunk workers consume:
|
||||
* `buildLocalExecutionPlan(projectDir, config, executionPlanDir)` composes the
|
||||
* existing render stages (compile → probe → extract videos → audio → freeze)
|
||||
* into a self-contained local execution directory that downstream chunk
|
||||
* workers consume:
|
||||
*
|
||||
* <planDir>/
|
||||
* ├── plan.json
|
||||
@@ -16,8 +17,9 @@
|
||||
* └── chunks.json
|
||||
*
|
||||
* Pure function over local paths. No networking. Two invocations with the
|
||||
* same inputs produce the same `planHash` — adapters use that contract to
|
||||
* short-circuit `plan()` on workflow replay.
|
||||
* same inputs produce the same execution-plan hash. Transport adapters use
|
||||
* that representation either through the legacy v1 `plan()` wrapper or the
|
||||
* v2 manifest/CAS publisher.
|
||||
*
|
||||
* Banned configurations (GPU encode, hardware browser GL, system primary
|
||||
* fonts) are rejected at plan time via `planValidation.ts` so chunk workers
|
||||
@@ -77,7 +79,7 @@ import {
|
||||
readFfmpegVersion,
|
||||
readProducerVersion,
|
||||
} from "./shared.js";
|
||||
import { CURRENT_PLAN_PROTOCOL, type PlanProtocolV1Descriptor } from "./planProtocol.js";
|
||||
import { PLAN_PROTOCOL_V1, type PlanProtocolV1Descriptor } from "./planProtocol.js";
|
||||
import {
|
||||
measurePlanSizeBreakdown,
|
||||
type PlanSizeBreakdown,
|
||||
@@ -253,9 +255,35 @@ export interface DistributedRenderConfig {
|
||||
variables?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Shared local representation consumed by both distributed plan transports. */
|
||||
export interface LocalExecutionPlan {
|
||||
executionPlanDir: string;
|
||||
executionPlanHash: string;
|
||||
chunkCount: number;
|
||||
totalFrames: number;
|
||||
fps: 24 | 30 | 60;
|
||||
width: number;
|
||||
height: number;
|
||||
format: DistributedFormat;
|
||||
ffmpegVersion: string;
|
||||
producerVersion: string;
|
||||
}
|
||||
|
||||
export interface BuildLocalExecutionPlanOptions {
|
||||
/**
|
||||
* Transport-specific size ceiling for the local execution representation.
|
||||
* Legacy v1 uses the monolithic plan-directory limit; v2 disables that
|
||||
* transport cap because it publishes role-scoped content-addressed objects.
|
||||
*/
|
||||
readonly executionPlanSizeLimitBytes?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of {@link plan}. The `planHash` is the content-addressed identifier
|
||||
* that adapters key replay short-circuits off of.
|
||||
* Result of the legacy v1 {@link plan} wrapper. The `planHash` is the
|
||||
* content-addressed identifier that adapters key replay short-circuits off of.
|
||||
*
|
||||
* @deprecated Use `planV2()` or `planV2WithPublisher()` for new integrations.
|
||||
* The v1 result remains supported for existing transports.
|
||||
*/
|
||||
export interface PlanResult {
|
||||
planDir: string;
|
||||
@@ -798,15 +826,16 @@ export function resolveDistributedEngineConfig(config: DistributedRenderConfig):
|
||||
}
|
||||
|
||||
/**
|
||||
* Activity A of the distributed render pipeline. Produces a self-contained
|
||||
* `<planDir>/` from a project + config. See module docstring for the
|
||||
* directory layout.
|
||||
* Build the shared local execution representation used by both transport
|
||||
* protocols. See the module docstring for the directory layout.
|
||||
*/
|
||||
export async function plan(
|
||||
export async function buildLocalExecutionPlan(
|
||||
projectDir: string,
|
||||
config: DistributedRenderConfig,
|
||||
planDir: string,
|
||||
): Promise<PlanResult> {
|
||||
executionPlanDir: string,
|
||||
options: Readonly<BuildLocalExecutionPlanOptions> = {},
|
||||
): Promise<LocalExecutionPlan> {
|
||||
const planDir = executionPlanDir;
|
||||
// Plan-time validation. Rejections here surface as typed errors with
|
||||
// non-retryable codes so workflow adapters don't waste retry budget on
|
||||
// banned configs. Runs BEFORE any directory creation so a banned input
|
||||
@@ -820,7 +849,10 @@ export async function plan(
|
||||
if (!existsSync(planDir)) mkdirSync(planDir, { recursive: true });
|
||||
|
||||
const log = config.logger ?? defaultLogger;
|
||||
const sizeLimitBytes = config.planDirSizeLimitBytes ?? PLAN_DIR_SIZE_LIMIT_BYTES;
|
||||
const sizeLimitBytes =
|
||||
options.executionPlanSizeLimitBytes ??
|
||||
config.planDirSizeLimitBytes ??
|
||||
PLAN_DIR_SIZE_LIMIT_BYTES;
|
||||
const abortSignal = config.abortSignal;
|
||||
const assertNotAborted = (): void => {
|
||||
if (abortSignal?.aborted) {
|
||||
@@ -1193,9 +1225,8 @@ export async function plan(
|
||||
});
|
||||
|
||||
return {
|
||||
planDir,
|
||||
planProtocol: CURRENT_PLAN_PROTOCOL,
|
||||
planHash,
|
||||
executionPlanDir: planDir,
|
||||
executionPlanHash: planHash,
|
||||
chunkCount,
|
||||
totalFrames,
|
||||
fps: config.fps,
|
||||
@@ -1206,3 +1237,30 @@ export async function plan(
|
||||
producerVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy v1 transport wrapper around {@link buildLocalExecutionPlan}.
|
||||
*
|
||||
* @deprecated Use `planV2()` or `planV2WithPublisher()` for new integrations.
|
||||
* This wrapper, its layout, and its result shape remain supported.
|
||||
*/
|
||||
export async function plan(
|
||||
projectDir: string,
|
||||
config: DistributedRenderConfig,
|
||||
planDir: string,
|
||||
): Promise<PlanResult> {
|
||||
const executionPlan = await buildLocalExecutionPlan(projectDir, config, planDir);
|
||||
return {
|
||||
planDir: executionPlan.executionPlanDir,
|
||||
planProtocol: PLAN_PROTOCOL_V1,
|
||||
planHash: executionPlan.executionPlanHash,
|
||||
chunkCount: executionPlan.chunkCount,
|
||||
totalFrames: executionPlan.totalFrames,
|
||||
fps: executionPlan.fps,
|
||||
width: executionPlan.width,
|
||||
height: executionPlan.height,
|
||||
format: executionPlan.format,
|
||||
ffmpegVersion: executionPlan.ffmpegVersion,
|
||||
producerVersion: executionPlan.producerVersion,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
getDistributedRenderCapabilities,
|
||||
PLAN_ARTIFACT_LAYOUT,
|
||||
PLAN_HASH_SCHEMA,
|
||||
PLAN_PROTOCOL_V1,
|
||||
PLAN_PROTOCOL_V2,
|
||||
PLAN_PROTOCOL_UNSUPPORTED,
|
||||
PLAN_SCHEMA_VERSION,
|
||||
@@ -95,6 +96,13 @@ function createReaderPlan(options: {
|
||||
}
|
||||
|
||||
describe("readPlanProtocol()", () => {
|
||||
it("keeps the v1 descriptor byte-for-byte and identity-compatible", () => {
|
||||
expect(PLAN_PROTOCOL_V1).toBe(CURRENT_PLAN_PROTOCOL);
|
||||
expect(JSON.stringify(PLAN_PROTOCOL_V1)).toBe(
|
||||
'{"schemaVersion":1,"artifactLayout":"plan-dir-v1","hashSchema":"hyperframes-plan-hash-v1"}',
|
||||
);
|
||||
});
|
||||
|
||||
it("treats an absent descriptor as legacy v1", () => {
|
||||
expect(readPlanProtocol({ planHash: "legacy" })).toBe(CURRENT_PLAN_PROTOCOL);
|
||||
});
|
||||
|
||||
@@ -36,13 +36,21 @@ export interface PlanProtocolV2Descriptor extends PlanProtocolDescriptor {
|
||||
|
||||
export type SupportedPlanProtocolDescriptor = PlanProtocolV1Descriptor | PlanProtocolV2Descriptor;
|
||||
|
||||
/** Descriptor written by the current producer and accepted by v1 workers. */
|
||||
export const CURRENT_PLAN_PROTOCOL: Readonly<PlanProtocolV1Descriptor> = Object.freeze({
|
||||
/** Descriptor for the legacy v1 execution-directory transport. */
|
||||
export const PLAN_PROTOCOL_V1: Readonly<PlanProtocolV1Descriptor> = Object.freeze({
|
||||
schemaVersion: PLAN_SCHEMA_VERSION,
|
||||
artifactLayout: PLAN_ARTIFACT_LAYOUT,
|
||||
hashSchema: PLAN_HASH_SCHEMA,
|
||||
});
|
||||
|
||||
/**
|
||||
* Descriptor written by the legacy v1 planner and accepted by v1 workers.
|
||||
*
|
||||
* @deprecated Use {@link PLAN_PROTOCOL_V1}. Kept as an identity-preserving
|
||||
* alias for existing integrations.
|
||||
*/
|
||||
export const CURRENT_PLAN_PROTOCOL: Readonly<PlanProtocolV1Descriptor> = PLAN_PROTOCOL_V1;
|
||||
|
||||
/** Explicit opt-in descriptor for the content-addressed v2 transport layout. */
|
||||
export const PLAN_PROTOCOL_V2: Readonly<PlanProtocolV2Descriptor> = Object.freeze({
|
||||
schemaVersion: PLAN_V2_SCHEMA_VERSION,
|
||||
@@ -70,14 +78,14 @@ export const DISTRIBUTED_RENDER_CAPABILITIES: Readonly<DistributedRenderCapabili
|
||||
Object.freeze({
|
||||
roles: Object.freeze({
|
||||
planner: Object.freeze({
|
||||
produces: Object.freeze([CURRENT_PLAN_PROTOCOL, PLAN_PROTOCOL_V2]),
|
||||
produces: Object.freeze([PLAN_PROTOCOL_V1, PLAN_PROTOCOL_V2]),
|
||||
}),
|
||||
chunk: Object.freeze({
|
||||
accepts: Object.freeze([CURRENT_PLAN_PROTOCOL, PLAN_PROTOCOL_V2]),
|
||||
accepts: Object.freeze([PLAN_PROTOCOL_V1, PLAN_PROTOCOL_V2]),
|
||||
acceptsLegacyV1WithoutDescriptor: true,
|
||||
}),
|
||||
assembler: Object.freeze({
|
||||
accepts: Object.freeze([CURRENT_PLAN_PROTOCOL, PLAN_PROTOCOL_V2]),
|
||||
accepts: Object.freeze([PLAN_PROTOCOL_V1, PLAN_PROTOCOL_V2]),
|
||||
acceptsLegacyV1WithoutDescriptor: true,
|
||||
}),
|
||||
}),
|
||||
@@ -143,13 +151,13 @@ export function readPlanProtocol(
|
||||
if (!Object.prototype.hasOwnProperty.call(planJson, "protocol")) {
|
||||
if (
|
||||
!capabilities.acceptsLegacyV1WithoutDescriptor ||
|
||||
!capabilitiesAccept(capabilities, CURRENT_PLAN_PROTOCOL)
|
||||
!capabilitiesAccept(capabilities, PLAN_PROTOCOL_V1)
|
||||
) {
|
||||
throw new PlanProtocolUnsupportedError(
|
||||
"legacy v1 plan without a protocol descriptor is not accepted by this worker",
|
||||
);
|
||||
}
|
||||
return CURRENT_PLAN_PROTOCOL;
|
||||
return PLAN_PROTOCOL_V1;
|
||||
}
|
||||
|
||||
const descriptor = planJson.protocol;
|
||||
@@ -163,8 +171,8 @@ export function readPlanProtocol(
|
||||
}
|
||||
}
|
||||
|
||||
const protocol = protocolMatches(descriptor, CURRENT_PLAN_PROTOCOL)
|
||||
? CURRENT_PLAN_PROTOCOL
|
||||
const protocol = protocolMatches(descriptor, PLAN_PROTOCOL_V1)
|
||||
? PLAN_PROTOCOL_V1
|
||||
: protocolMatches(descriptor, PLAN_PROTOCOL_V2)
|
||||
? PLAN_PROTOCOL_V2
|
||||
: null;
|
||||
@@ -185,10 +193,10 @@ export function readPlanProtocolV1(
|
||||
.chunk,
|
||||
): Readonly<PlanProtocolV1Descriptor> {
|
||||
const protocol = readPlanProtocol(planJson, capabilities);
|
||||
if (protocol !== CURRENT_PLAN_PROTOCOL) {
|
||||
if (protocol !== PLAN_PROTOCOL_V1) {
|
||||
throw new PlanProtocolUnsupportedError(
|
||||
"content-addressed v2 plan must be materialized before v1 layout access",
|
||||
);
|
||||
}
|
||||
return CURRENT_PLAN_PROTOCOL;
|
||||
return PLAN_PROTOCOL_V1;
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ import {
|
||||
PlanTooLargeError,
|
||||
plan,
|
||||
} from "./plan.js";
|
||||
import { planV2, readPlanV2Manifest } from "./planV2.js";
|
||||
import { getPlanV2ExecutionPlanHash, planV2, readPlanV2Manifest } from "./planV2.js";
|
||||
import { measurePlanSizeBreakdown } from "./planSize.js";
|
||||
import { DISTRIBUTED_DURATION_OUT_OF_RANGE } from "../render/planValidation.js";
|
||||
|
||||
@@ -262,7 +262,8 @@ describe("plan() under size cap", () => {
|
||||
const manifest = readPlanV2Manifest(v2.planDir);
|
||||
expect(v2.planProtocol.schemaVersion).toBe(2);
|
||||
expect(v2.planHash).toBe(manifest.planHash);
|
||||
expect(v2.sourcePlanV1Hash).toBe(manifest.sourcePlanV1Hash);
|
||||
expect(getPlanV2ExecutionPlanHash(v2)).toBe(manifest.sourcePlanV1Hash);
|
||||
expect(Object.hasOwn(v2, "executionPlanHash")).toBe(false);
|
||||
expect(manifest.artifacts.length).toBeGreaterThan(0);
|
||||
},
|
||||
TIMEOUT_MS,
|
||||
|
||||
@@ -18,11 +18,14 @@ import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";
|
||||
import { FFMPEG_VERSION_MISMATCH, renderChunk, RenderChunkValidationError } from "./renderChunk.js";
|
||||
import {
|
||||
createPlanV2FromV1,
|
||||
createPlanV2FromExecutionPlan,
|
||||
getPlanV2ExecutionPlanHash,
|
||||
listPlanV2ArtifactsForTarget,
|
||||
materializePlanV2Target,
|
||||
PLAN_V2_INTEGRITY_UNRECOVERABLE,
|
||||
PlanV2IntegrityError,
|
||||
publishPlanV2FromV1,
|
||||
publishPlanV2FromExecutionPlan,
|
||||
readPlanV2Manifest,
|
||||
validatePlanV2MaterializedTarget,
|
||||
} from "./planV2.js";
|
||||
@@ -164,19 +167,67 @@ function createV1Plan(
|
||||
}
|
||||
|
||||
describe("Plan v2 manifest", () => {
|
||||
it("is deterministic and keeps the v1 transport opt-in", () => {
|
||||
it("is deterministic without changing the manifest or result wire shapes", () => {
|
||||
const root = tempPath("hf-plan-v2-determinism-");
|
||||
const v1 = createV1Plan(root, { audio: true });
|
||||
const first = createPlanV2FromV1(v1, join(root, "v2-a"));
|
||||
const second = createPlanV2FromV1(v1, join(root, "v2-b"));
|
||||
const first = createPlanV2FromExecutionPlan(v1, join(root, "v2-a"));
|
||||
const second = createPlanV2FromExecutionPlan(v1, join(root, "v2-b"));
|
||||
const serializedManifest = readFileSync(first.manifestPath, "utf-8");
|
||||
const manifest = JSON.parse(serializedManifest) as Record<string, unknown>;
|
||||
|
||||
expect(first.planHash).toBe(second.planHash);
|
||||
expect(readFileSync(first.manifestPath, "utf-8")).toBe(
|
||||
readFileSync(second.manifestPath, "utf-8"),
|
||||
);
|
||||
expect(serializedManifest).toBe(readFileSync(second.manifestPath, "utf-8"));
|
||||
expect(first.planHash).not.toBe(first.sourcePlanV1Hash);
|
||||
expect(getPlanV2ExecutionPlanHash(first)).toBe(first.sourcePlanV1Hash);
|
||||
expect(first.planProtocol.schemaVersion).toBe(2);
|
||||
expect(first.limitations.videoDependencyMode).toBe("exact-rendered-frames");
|
||||
expect(Object.keys(first)).toEqual([
|
||||
"planDir",
|
||||
"manifestPath",
|
||||
"planProtocol",
|
||||
"planHash",
|
||||
"sourcePlanV1Hash",
|
||||
"chunkCount",
|
||||
"totalFrames",
|
||||
"fps",
|
||||
"width",
|
||||
"height",
|
||||
"format",
|
||||
"ffmpegVersion",
|
||||
"producerVersion",
|
||||
"limitations",
|
||||
]);
|
||||
expect(Object.keys(manifest)).toEqual([
|
||||
"artifacts",
|
||||
"chunkCount",
|
||||
"ffmpegVersion",
|
||||
"format",
|
||||
"fps",
|
||||
"height",
|
||||
"limitations",
|
||||
"planHash",
|
||||
"producerVersion",
|
||||
"protocol",
|
||||
"sourcePlanV1Hash",
|
||||
"totalFrames",
|
||||
"width",
|
||||
]);
|
||||
expect(Object.hasOwn(first, "executionPlanHash")).toBe(false);
|
||||
expect(Object.hasOwn(manifest, "executionPlanHash")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps legacy conversion names as byte-identical compatibility aliases", async () => {
|
||||
const root = tempPath("hf-plan-v2-compat-aliases-");
|
||||
const executionPlanDir = createV1Plan(root, { audio: true });
|
||||
const canonical = createPlanV2FromExecutionPlan(executionPlanDir, join(root, "canonical"));
|
||||
const compatibility = createPlanV2FromV1(executionPlanDir, join(root, "compatibility"));
|
||||
const publisher = new LocalPlanV2ArtifactPublisher(join(root, "published"));
|
||||
const published = await publishPlanV2FromExecutionPlan(executionPlanDir, publisher);
|
||||
|
||||
expect(readFileSync(canonical.manifestPath)).toEqual(readFileSync(compatibility.manifestPath));
|
||||
expect(getPlanV2ExecutionPlanHash(published)).toBe(getPlanV2ExecutionPlanHash(canonical));
|
||||
expect(published.sourcePlanV1Hash).toBe(canonical.sourcePlanV1Hash);
|
||||
expect(Object.hasOwn(published, "executionPlanHash")).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts and materializes the same bounded timing produced for v1", () => {
|
||||
@@ -220,7 +271,7 @@ describe("Plan v2 manifest", () => {
|
||||
expect(caught).toHaveProperty("code", PLAN_V2_INTEGRITY_UNRECOVERABLE);
|
||||
expect(caught).toHaveProperty(
|
||||
"message",
|
||||
expect.stringMatching(/v1 plan content fingerprint does not match/),
|
||||
expect.stringMatching(/execution plan content fingerprint does not match/),
|
||||
);
|
||||
expect(existsSync(destination)).toBe(false);
|
||||
});
|
||||
@@ -584,6 +635,16 @@ describe("Plan v2 manifest", () => {
|
||||
result.planHash,
|
||||
);
|
||||
expect(chunk.sourcePlanV1Hash).toBe(result.sourcePlanV1Hash);
|
||||
expect(Object.keys(chunk)).toEqual([
|
||||
"planDir",
|
||||
"target",
|
||||
"planHash",
|
||||
"sourcePlanV1Hash",
|
||||
"artifactCount",
|
||||
"sizeBytes",
|
||||
"audioPath",
|
||||
]);
|
||||
expect(Object.hasOwn(chunk, "executionPlanHash")).toBe(false);
|
||||
});
|
||||
|
||||
it("uses v2 subset integrity instead of the whole-v1 plan hash", async () => {
|
||||
@@ -689,7 +750,7 @@ describe("Plan v2 artifact publisher", () => {
|
||||
const v1 = createV1Plan(root, { audio: true });
|
||||
const destination = join(root, "v2");
|
||||
const publisher = new LocalPlanV2ArtifactPublisher(destination);
|
||||
const manifest = await publishPlanV2FromV1(v1, publisher);
|
||||
const manifest = await publishPlanV2FromExecutionPlan(v1, publisher);
|
||||
const artifact = manifest.artifacts.find(
|
||||
(candidate) => candidate.path === "compiled/asset.txt",
|
||||
);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* V2 deliberately separates transport from execution. The transport root is
|
||||
* a small immutable `plan.json` manifest plus sha256-addressed blobs. Workers
|
||||
* select and materialize only the dependencies for their role, then invoke
|
||||
* the existing v1 execution functions on the verified local layout.
|
||||
* the shared execution functions on the verified local layout.
|
||||
*
|
||||
* Video-frame dependencies are derived by evaluating the engine's own
|
||||
* FrameLookupTable at every captured global frame. If legacy video metadata
|
||||
@@ -33,7 +33,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { createFrameLookupTable, type ExtractedFrames } from "@hyperframes/engine";
|
||||
import { recomputePlanHashFromPlanDir, type ChunkSliceJson } from "../render/stages/freezePlan.js";
|
||||
import { canonicalJsonStringify, sha256Hex } from "../render/stages/planHash.js";
|
||||
import { type DistributedRenderConfig, plan } from "./plan.js";
|
||||
import { buildLocalExecutionPlan, type DistributedRenderConfig } from "./plan.js";
|
||||
import {
|
||||
PLAN_PROTOCOL_V2,
|
||||
readPlanProtocolV1,
|
||||
@@ -69,7 +69,7 @@ export type PlanV2MaterializationTarget =
|
||||
| Readonly<{ role: "assembler" }>;
|
||||
|
||||
export interface PlanV2Artifact {
|
||||
/** POSIX path in the materialized v1 execution directory. */
|
||||
/** POSIX path in the materialized local execution directory. */
|
||||
readonly path: string;
|
||||
readonly sha256: string;
|
||||
readonly sizeBytes: number;
|
||||
@@ -80,9 +80,14 @@ export interface PlanV2Artifact {
|
||||
|
||||
export interface PlanV2Manifest {
|
||||
readonly protocol: Readonly<PlanProtocolV2Descriptor>;
|
||||
/** V2 manifest digest; intentionally distinct from the v1 execution hash. */
|
||||
/** V2 manifest digest; intentionally distinct from the local execution-plan hash. */
|
||||
readonly planHash: string;
|
||||
/** Original execution-plan hash retained for output/replay correlation. */
|
||||
/**
|
||||
* Local execution-plan hash retained for output/replay correlation.
|
||||
*
|
||||
* @deprecated This is the compatibility wire name. Use
|
||||
* {@link getPlanV2ExecutionPlanHash} in new code.
|
||||
*/
|
||||
readonly sourcePlanV1Hash: string;
|
||||
readonly chunkCount: number;
|
||||
readonly totalFrames: number;
|
||||
@@ -105,6 +110,12 @@ export interface PlanV2Result {
|
||||
readonly manifestPath: string;
|
||||
readonly planProtocol: Readonly<PlanProtocolV2Descriptor>;
|
||||
readonly planHash: string;
|
||||
/**
|
||||
* Hash of the shared local execution representation.
|
||||
*
|
||||
* @deprecated This is the compatibility wire name. Use
|
||||
* {@link getPlanV2ExecutionPlanHash} in new code.
|
||||
*/
|
||||
readonly sourcePlanV1Hash: string;
|
||||
readonly chunkCount: number;
|
||||
readonly totalFrames: number;
|
||||
@@ -119,7 +130,7 @@ export interface PlanV2Result {
|
||||
|
||||
export interface PlanV2WithPublisherOptions {
|
||||
/**
|
||||
* Parent for the planner-private v1 staging directory. Remote adapters
|
||||
* Parent for the planner-private local execution directory. Remote adapters
|
||||
* normally leave this unset and use the OS temp directory. The local
|
||||
* compatibility wrapper keeps staging beside its destination so hard links
|
||||
* can avoid a second set of data blocks.
|
||||
@@ -131,6 +142,12 @@ export interface PlanV2MaterializationResult {
|
||||
readonly planDir: string;
|
||||
readonly target: PlanV2MaterializationTarget;
|
||||
readonly planHash: string;
|
||||
/**
|
||||
* Hash of the shared local execution representation.
|
||||
*
|
||||
* @deprecated This is the compatibility wire name. Use
|
||||
* {@link getPlanV2ExecutionPlanHash} in new code.
|
||||
*/
|
||||
readonly sourcePlanV1Hash: string;
|
||||
readonly artifactCount: number;
|
||||
readonly sizeBytes: number;
|
||||
@@ -198,7 +215,7 @@ function listFiles(root: string): Array<{ path: string; absolutePath: string }>
|
||||
return files.sort((a, b) => a.path.localeCompare(b.path));
|
||||
}
|
||||
|
||||
/** Hash large plan artifacts with bounded memory (important above the v1 2 GiB cap). */
|
||||
/** Hash large plan artifacts with bounded memory (important above the legacy 2 GiB cap). */
|
||||
function sha256File(path: string): string {
|
||||
const hash = createHash("sha256");
|
||||
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
||||
@@ -226,8 +243,8 @@ function assertValidExtractionCacheCompleteSentinel(path: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function validateExtractionCacheCompleteSentinels(planV1Dir: string): void {
|
||||
const videoRoot = join(planV1Dir, "video-frames");
|
||||
function validateExtractionCacheCompleteSentinels(executionPlanDir: string): void {
|
||||
const videoRoot = join(executionPlanDir, "video-frames");
|
||||
if (!existsSync(videoRoot)) return;
|
||||
|
||||
for (const videoEntry of readdirSync(videoRoot, { withFileTypes: true })) {
|
||||
@@ -259,7 +276,7 @@ function resolveExtractedVideoOutputDir(planDir: string, videoId: string): strin
|
||||
return outputDir;
|
||||
}
|
||||
|
||||
// This is the fail-safe policy table for every v1 artifact class. Keeping the
|
||||
// This is the fail-safe policy table for every local execution artifact class. Keeping the
|
||||
// branches together makes new artifact classes visibly fall through to both roles.
|
||||
// fallow-ignore-next-line complexity
|
||||
function artifactTargets(
|
||||
@@ -283,14 +300,14 @@ function artifactTargets(
|
||||
assembler: false,
|
||||
};
|
||||
}
|
||||
// Unknown future v1 files go to both roles. Over-including is safe;
|
||||
// Unknown future execution files go to both roles. Over-including is safe;
|
||||
// silently omitting a new execution dependency is not.
|
||||
return { chunks: "all", assembler: true };
|
||||
}
|
||||
|
||||
function listVideoFramePaths(planV1Dir: string, videos: PlanVideosJson): ExtractedFrames[] {
|
||||
function listVideoFramePaths(executionPlanDir: string, videos: PlanVideosJson): ExtractedFrames[] {
|
||||
return videos.extracted.map((video) => {
|
||||
const outputDir = resolveExtractedVideoOutputDir(planV1Dir, video.videoId);
|
||||
const outputDir = resolveExtractedVideoOutputDir(executionPlanDir, video.videoId);
|
||||
const frameNames = readdirSync(outputDir).sort();
|
||||
const framePaths = new Map<number, string>();
|
||||
for (const frameName of frameNames) {
|
||||
@@ -370,26 +387,26 @@ function parseChunkSlices(value: unknown): ChunkSliceJson[] {
|
||||
}
|
||||
|
||||
function buildVideoChunkDependencies(
|
||||
planV1Dir: string,
|
||||
executionPlanDir: string,
|
||||
dimensions: Record<string, unknown>,
|
||||
): {
|
||||
mode: "exact-rendered-frames" | "full-source-pack";
|
||||
dependencies: ReadonlyMap<string, readonly number[]> | null;
|
||||
} {
|
||||
const videoRoot = join(planV1Dir, "video-frames");
|
||||
const videoRoot = join(executionPlanDir, "video-frames");
|
||||
const hasExtractedFrames = existsSync(videoRoot) && listFiles(videoRoot).length > 0;
|
||||
const videosPath = join(planV1Dir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
||||
const videosPath = join(executionPlanDir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
||||
if (!existsSync(videosPath)) {
|
||||
return hasExtractedFrames
|
||||
? { mode: "full-source-pack", dependencies: null }
|
||||
: { mode: "exact-rendered-frames", dependencies: new Map() };
|
||||
}
|
||||
const chunksPath = join(planV1Dir, "meta", "chunks.json");
|
||||
const chunksPath = join(executionPlanDir, "meta", "chunks.json");
|
||||
const videos = readJsonFile(videosPath, PLAN_VIDEOS_META_RELATIVE_PATH);
|
||||
const chunks = readJsonFile(chunksPath, "meta/chunks.json");
|
||||
const parsedVideos = parsePlanVideosJson(videos);
|
||||
const parsedChunks = parseChunkSlices(chunks);
|
||||
const extracted = listVideoFramePaths(planV1Dir, parsedVideos);
|
||||
const extracted = listVideoFramePaths(executionPlanDir, parsedVideos);
|
||||
const table = createFrameLookupTable(parsedVideos.videos, extracted);
|
||||
const fpsNum = readPositiveInteger(dimensions.fpsNum, "dimensions.fpsNum");
|
||||
const fpsDen = readPositiveInteger(dimensions.fpsDen, "dimensions.fpsDen");
|
||||
@@ -399,7 +416,7 @@ function buildVideoChunkDependencies(
|
||||
for (let frame = chunk.startFrame; frame < chunk.endFrame; frame++) {
|
||||
const globalTime = (frame * fpsDen) / fpsNum;
|
||||
for (const payload of table.getActiveFramePayloads(globalTime).values()) {
|
||||
const path = relative(resolve(planV1Dir), payload.framePath).split(sep).join("/");
|
||||
const path = relative(resolve(executionPlanDir), payload.framePath).split(sep).join("/");
|
||||
assertSafeRelativePath(path);
|
||||
const indexes = mutable.get(path) ?? new Set<number>();
|
||||
indexes.add(chunk.index);
|
||||
@@ -445,38 +462,40 @@ interface PlanV2Publication {
|
||||
|
||||
// Artifact classification intentionally keeps every fail-safe branch together.
|
||||
// fallow-ignore-next-line complexity
|
||||
function buildPlanV2Publication(planV1Dir: string): PlanV2Publication {
|
||||
const v1PlanPath = join(planV1Dir, "plan.json");
|
||||
if (!existsSync(v1PlanPath)) {
|
||||
throw new PlanV2IntegrityError(`v1 plan is missing plan.json: ${v1PlanPath}`);
|
||||
function buildPlanV2Publication(executionPlanDir: string): PlanV2Publication {
|
||||
const executionPlanPath = join(executionPlanDir, "plan.json");
|
||||
if (!existsSync(executionPlanPath)) {
|
||||
throw new PlanV2IntegrityError(`execution plan is missing plan.json: ${executionPlanPath}`);
|
||||
}
|
||||
const v1PlanValue = readJsonFile(v1PlanPath, "v1 plan.json");
|
||||
if (!isRecord(v1PlanValue)) {
|
||||
throw new PlanV2IntegrityError("v1 plan.json must be an object");
|
||||
const executionPlanValue = readJsonFile(executionPlanPath, "execution plan.json");
|
||||
if (!isRecord(executionPlanValue)) {
|
||||
throw new PlanV2IntegrityError("execution plan.json must be an object");
|
||||
}
|
||||
const v1Plan = v1PlanValue;
|
||||
readPlanProtocolV1(v1Plan);
|
||||
const sourcePlanV1Hash = v1Plan.planHash;
|
||||
if (!isSha256(sourcePlanV1Hash)) {
|
||||
throw new PlanV2IntegrityError("v1 plan.json.planHash must be a sha256 digest");
|
||||
const executionPlan = executionPlanValue;
|
||||
// The shared local representation intentionally retains the v1-compatible
|
||||
// descriptor while both legacy readers and v2 materialization are supported.
|
||||
readPlanProtocolV1(executionPlan);
|
||||
const executionPlanHash = executionPlan.planHash;
|
||||
if (!isSha256(executionPlanHash)) {
|
||||
throw new PlanV2IntegrityError("execution plan.json.planHash must be a sha256 digest");
|
||||
}
|
||||
const recomputedSourcePlanV1Hash = recomputePlanHashFromPlanDir(planV1Dir);
|
||||
if (recomputedSourcePlanV1Hash !== sourcePlanV1Hash) {
|
||||
const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir);
|
||||
if (recomputedExecutionPlanHash !== executionPlanHash) {
|
||||
throw new PlanV2IntegrityError(
|
||||
`v1 plan content fingerprint does not match plan.json.planHash: ` +
|
||||
`expected ${sourcePlanV1Hash}, recomputed ${recomputedSourcePlanV1Hash}`,
|
||||
`execution plan content fingerprint does not match plan.json.planHash: ` +
|
||||
`expected ${executionPlanHash}, recomputed ${recomputedExecutionPlanHash}`,
|
||||
);
|
||||
}
|
||||
|
||||
const artifacts: PlanV2Artifact[] = [];
|
||||
const blobs = new Map<string, PlanV2PublishBlob>();
|
||||
const dimensions = v1Plan.dimensions;
|
||||
const dimensions = executionPlan.dimensions;
|
||||
if (!isRecord(dimensions)) {
|
||||
throw new PlanV2IntegrityError("v1 plan.json.dimensions must be an object");
|
||||
throw new PlanV2IntegrityError("execution plan.json.dimensions must be an object");
|
||||
}
|
||||
validateExtractionCacheCompleteSentinels(planV1Dir);
|
||||
const videoDependencyPlan = buildVideoChunkDependencies(planV1Dir, dimensions);
|
||||
for (const file of listFiles(planV1Dir)) {
|
||||
validateExtractionCacheCompleteSentinels(executionPlanDir);
|
||||
const videoDependencyPlan = buildVideoChunkDependencies(executionPlanDir, dimensions);
|
||||
for (const file of listFiles(executionPlanDir)) {
|
||||
if (isExtractionCacheCompleteSentinelPath(file.path)) continue;
|
||||
const targets = artifactTargets(file.path, videoDependencyPlan.dependencies);
|
||||
if (
|
||||
@@ -502,15 +521,17 @@ function buildPlanV2Publication(planV1Dir: string): PlanV2Publication {
|
||||
|
||||
const base: Omit<PlanV2Manifest, "planHash"> = {
|
||||
protocol: PLAN_PROTOCOL_V2,
|
||||
sourcePlanV1Hash,
|
||||
chunkCount: readPositiveInteger(v1Plan.chunkCount, "chunkCount"),
|
||||
totalFrames: readPositiveInteger(v1Plan.totalFrames, "totalFrames"),
|
||||
fps: readV1PlanFps(dimensions),
|
||||
// Retain the established manifest key byte-for-byte. It now acts as the
|
||||
// wire alias for the neutral local execution-plan hash.
|
||||
sourcePlanV1Hash: executionPlanHash,
|
||||
chunkCount: readPositiveInteger(executionPlan.chunkCount, "chunkCount"),
|
||||
totalFrames: readPositiveInteger(executionPlan.totalFrames, "totalFrames"),
|
||||
fps: readExecutionPlanFps(dimensions),
|
||||
width: readPositiveInteger(dimensions.width, "dimensions.width"),
|
||||
height: readPositiveInteger(dimensions.height, "dimensions.height"),
|
||||
format: readDistributedFormat(dimensions.format),
|
||||
ffmpegVersion: readString(v1Plan.ffmpegVersion, "ffmpegVersion"),
|
||||
producerVersion: readString(v1Plan.producerVersion, "producerVersion"),
|
||||
ffmpegVersion: readString(executionPlan.ffmpegVersion, "ffmpegVersion"),
|
||||
producerVersion: readString(executionPlan.producerVersion, "producerVersion"),
|
||||
limitations: { videoDependencyMode: videoDependencyPlan.mode },
|
||||
artifacts,
|
||||
};
|
||||
@@ -523,12 +544,15 @@ function buildPlanV2Publication(planV1Dir: string): PlanV2Publication {
|
||||
};
|
||||
}
|
||||
|
||||
/** Convert a frozen v1 execution directory into the immutable v2 transport. */
|
||||
export function createPlanV2FromV1(planV1Dir: string, planV2Dir: string): PlanV2Result {
|
||||
/** Publish a frozen local execution directory into an immutable local v2 transport. */
|
||||
export function createPlanV2FromExecutionPlan(
|
||||
executionPlanDir: string,
|
||||
planV2Dir: string,
|
||||
): PlanV2Result {
|
||||
if (existsSync(planV2Dir)) {
|
||||
throw new PlanV2IntegrityError(`output directory already exists: ${planV2Dir}`);
|
||||
}
|
||||
const publication = buildPlanV2Publication(planV1Dir);
|
||||
const publication = buildPlanV2Publication(executionPlanDir);
|
||||
mkdirSync(dirname(planV2Dir), { recursive: true });
|
||||
const tempDir = mkdtempSync(join(dirname(planV2Dir), ".plan-v2-build-"));
|
||||
try {
|
||||
@@ -548,12 +572,22 @@ export function createPlanV2FromV1(planV1Dir: string, planV2Dir: string): PlanV2
|
||||
}
|
||||
}
|
||||
|
||||
export async function publishPlanV2FromV1(
|
||||
planV1Dir: string,
|
||||
/**
|
||||
* Compatibility alias for {@link createPlanV2FromExecutionPlan}.
|
||||
*
|
||||
* @deprecated Use `createPlanV2FromExecutionPlan`.
|
||||
*/
|
||||
export function createPlanV2FromV1(executionPlanDir: string, planV2Dir: string): PlanV2Result {
|
||||
return createPlanV2FromExecutionPlan(executionPlanDir, planV2Dir);
|
||||
}
|
||||
|
||||
/** Publish a frozen local execution directory through a storage-neutral v2 publisher. */
|
||||
export async function publishPlanV2FromExecutionPlan(
|
||||
executionPlanDir: string,
|
||||
publisher: PlanV2ArtifactPublisher,
|
||||
): Promise<PlanV2Manifest> {
|
||||
try {
|
||||
const publication = buildPlanV2Publication(planV1Dir);
|
||||
const publication = buildPlanV2Publication(executionPlanDir);
|
||||
const concurrency = 16;
|
||||
for (let offset = 0; offset < publication.blobs.length; offset += concurrency) {
|
||||
const batch = publication.blobs.slice(offset, offset + concurrency);
|
||||
@@ -574,6 +608,18 @@ export async function publishPlanV2FromV1(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compatibility alias for {@link publishPlanV2FromExecutionPlan}.
|
||||
*
|
||||
* @deprecated Use `publishPlanV2FromExecutionPlan`.
|
||||
*/
|
||||
export async function publishPlanV2FromV1(
|
||||
executionPlanDir: string,
|
||||
publisher: PlanV2ArtifactPublisher,
|
||||
): Promise<PlanV2Manifest> {
|
||||
return publishPlanV2FromExecutionPlan(executionPlanDir, publisher);
|
||||
}
|
||||
|
||||
/**
|
||||
* Plan into a storage-neutral publisher. Implementations may write to a local
|
||||
* directory, S3, GCS, or another durable CAS. Only the planner's private
|
||||
@@ -586,14 +632,12 @@ export async function planV2WithPublisher(
|
||||
publisher: PlanV2ArtifactPublisher,
|
||||
options: Readonly<PlanV2WithPublisherOptions> = {},
|
||||
): Promise<PlanV2Manifest> {
|
||||
const stagingRoot = mkdtempSync(join(options.stagingParentDir ?? tmpdir(), ".plan-v2-source-"));
|
||||
const stagingRoot = mkdtempSync(join(options.stagingParentDir ?? tmpdir(), ".execution-plan-"));
|
||||
try {
|
||||
await plan(
|
||||
projectDir,
|
||||
{ ...config, planDirSizeLimitBytes: Number.MAX_SAFE_INTEGER },
|
||||
stagingRoot,
|
||||
);
|
||||
return await publishPlanV2FromV1(stagingRoot, publisher);
|
||||
await buildLocalExecutionPlan(projectDir, config, stagingRoot, {
|
||||
executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,
|
||||
});
|
||||
return await publishPlanV2FromExecutionPlan(stagingRoot, publisher);
|
||||
} catch (error) {
|
||||
try {
|
||||
await publisher.abort();
|
||||
@@ -607,9 +651,9 @@ export async function planV2WithPublisher(
|
||||
}
|
||||
|
||||
/**
|
||||
* Plan directly into v2. The large v1 directory exists only as local staging;
|
||||
* its historical 2 GiB transport cap is disabled because no monolithic
|
||||
* archive is emitted.
|
||||
* Plan directly into v2. The shared local execution representation exists
|
||||
* only as planner-private staging; the legacy 2 GiB transport cap is disabled
|
||||
* because no monolithic archive is emitted.
|
||||
*/
|
||||
export async function planV2(
|
||||
projectDir: string,
|
||||
@@ -632,7 +676,7 @@ function resultFromManifest(planV2Dir: string, manifest: PlanV2Manifest): PlanV2
|
||||
manifestPath: join(planV2Dir, "plan.json"),
|
||||
planProtocol: PLAN_PROTOCOL_V2,
|
||||
planHash: manifest.planHash,
|
||||
sourcePlanV1Hash: manifest.sourcePlanV1Hash,
|
||||
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
||||
chunkCount: manifest.chunkCount,
|
||||
totalFrames: manifest.totalFrames,
|
||||
fps: manifest.fps,
|
||||
@@ -645,6 +689,16 @@ function resultFromManifest(planV2Dir: string, manifest: PlanV2Manifest): PlanV2
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the shared local execution-plan hash from a v2 manifest while
|
||||
* preserving the established `sourcePlanV1Hash` wire key.
|
||||
*/
|
||||
export function getPlanV2ExecutionPlanHash(
|
||||
plan: Readonly<Pick<PlanV2Manifest, "sourcePlanV1Hash">>,
|
||||
): string {
|
||||
return plan.sourcePlanV1Hash;
|
||||
}
|
||||
|
||||
function readString(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || value.length === 0) {
|
||||
throw new PlanV2IntegrityError(`${field} must be a non-empty string`);
|
||||
@@ -673,7 +727,7 @@ function readSupportedFps(value: unknown): 24 | 30 | 60 {
|
||||
return value;
|
||||
}
|
||||
|
||||
function readV1PlanFps(dimensions: Record<string, unknown>): 24 | 30 | 60 {
|
||||
function readExecutionPlanFps(dimensions: Record<string, unknown>): 24 | 30 | 60 {
|
||||
const fpsDen = readPositiveInteger(dimensions.fpsDen, "dimensions.fpsDen");
|
||||
if (fpsDen !== 1) {
|
||||
throw new PlanV2IntegrityError("dimensions.fpsDen must be 1 for plan v2");
|
||||
@@ -744,7 +798,7 @@ function parsePlanV2Manifest(value: unknown): Readonly<PlanV2Manifest> {
|
||||
paths.add(artifact.path);
|
||||
}
|
||||
if (!paths.has("plan.json"))
|
||||
throw new PlanV2IntegrityError("manifest must include the v1 plan.json artifact");
|
||||
throw new PlanV2IntegrityError("manifest must include the local execution plan.json artifact");
|
||||
if (
|
||||
!isRecord(value.limitations) ||
|
||||
(value.limitations.videoDependencyMode !== "exact-rendered-frames" &&
|
||||
@@ -830,7 +884,7 @@ function verifyBlob(planV2Dir: string, artifact: Readonly<PlanV2Artifact>): stri
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify every selected blob first, then atomically publish a v1-compatible
|
||||
* Verify every selected blob first, then atomically publish the shared local
|
||||
* execution directory. The marker lets execution functions revalidate the
|
||||
* selected subset without requiring assembler-only audio in chunk workers.
|
||||
*/
|
||||
@@ -858,7 +912,7 @@ export function materializePlanV2Target(
|
||||
}
|
||||
// Plan v2 transports only files, so a chunk where a video is inactive has
|
||||
// no selected frame artifact from which to create its per-video directory.
|
||||
// renderChunk consumes a v1-compatible layout and intentionally validates
|
||||
// renderChunk consumes the shared local layout and intentionally validates
|
||||
// every extracted-video directory from meta/videos.json. Recreate those
|
||||
// zero-byte structural directories without downloading unused frame data.
|
||||
if (target.role === "chunk") materializeExtractedVideoDirectories(tempDir);
|
||||
@@ -876,7 +930,7 @@ export function materializePlanV2Target(
|
||||
planDir: destinationDir,
|
||||
target,
|
||||
planHash: manifest.planHash,
|
||||
sourcePlanV1Hash: manifest.sourcePlanV1Hash,
|
||||
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
||||
artifactCount: artifacts.length,
|
||||
sizeBytes: artifacts.reduce((sum, artifact) => sum + artifact.sizeBytes, 0),
|
||||
audioPath:
|
||||
|
||||
@@ -83,6 +83,7 @@ describe("@hyperframes/producer/distributed (subpath)", () => {
|
||||
artifactLayout: "plan-dir-v1",
|
||||
hashSchema: "hyperframes-plan-hash-v1",
|
||||
});
|
||||
expect(distributedSubpath.PLAN_PROTOCOL_V1).toBe(distributedSubpath.CURRENT_PLAN_PROTOCOL);
|
||||
expect(distributedSubpath.DISTRIBUTED_RENDER_CAPABILITIES.roles).toEqual({
|
||||
planner: {
|
||||
produces: [distributedSubpath.CURRENT_PLAN_PROTOCOL, distributedSubpath.PLAN_PROTOCOL_V2],
|
||||
@@ -100,6 +101,11 @@ describe("@hyperframes/producer/distributed (subpath)", () => {
|
||||
expect(typeof distributedSubpath.readPlanProtocol).toBe("function");
|
||||
expect(typeof distributedSubpath.planV2).toBe("function");
|
||||
expect(typeof distributedSubpath.planV2WithPublisher).toBe("function");
|
||||
expect(typeof distributedSubpath.createPlanV2FromExecutionPlan).toBe("function");
|
||||
expect(typeof distributedSubpath.publishPlanV2FromExecutionPlan).toBe("function");
|
||||
expect(typeof distributedSubpath.getPlanV2ExecutionPlanHash).toBe("function");
|
||||
// Deprecated compatibility aliases remain available.
|
||||
expect(typeof distributedSubpath.createPlanV2FromV1).toBe("function");
|
||||
expect(typeof distributedSubpath.publishPlanV2FromV1).toBe("function");
|
||||
expect(typeof distributedSubpath.LocalPlanV2ArtifactPublisher).toBe("function");
|
||||
expect(typeof distributedSubpath.renderChunkV2).toBe("function");
|
||||
@@ -119,6 +125,7 @@ describe("@hyperframes/producer (main entry)", () => {
|
||||
|
||||
it("re-exports the plan protocol contract", () => {
|
||||
expect(producerIndex.CURRENT_PLAN_PROTOCOL).toBe(distributedSubpath.CURRENT_PLAN_PROTOCOL);
|
||||
expect(producerIndex.PLAN_PROTOCOL_V1).toBe(distributedSubpath.PLAN_PROTOCOL_V1);
|
||||
expect(producerIndex.DISTRIBUTED_RENDER_CAPABILITIES).toBe(
|
||||
distributedSubpath.DISTRIBUTED_RENDER_CAPABILITIES,
|
||||
);
|
||||
@@ -127,6 +134,11 @@ describe("@hyperframes/producer (main entry)", () => {
|
||||
expect(producerIndex.PLAN_V2_INTEGRITY_UNRECOVERABLE).toBe("PLAN_V2_INTEGRITY_UNRECOVERABLE");
|
||||
expect(typeof producerIndex.readPlanProtocol).toBe("function");
|
||||
expect(typeof producerIndex.planV2WithPublisher).toBe("function");
|
||||
expect(typeof producerIndex.createPlanV2FromExecutionPlan).toBe("function");
|
||||
expect(typeof producerIndex.publishPlanV2FromExecutionPlan).toBe("function");
|
||||
expect(typeof producerIndex.getPlanV2ExecutionPlanHash).toBe("function");
|
||||
// Deprecated compatibility aliases remain available.
|
||||
expect(typeof producerIndex.createPlanV2FromV1).toBe("function");
|
||||
expect(typeof producerIndex.publishPlanV2FromV1).toBe("function");
|
||||
expect(typeof producerIndex.PlanV2IntegrityError).toBe("function");
|
||||
expect(typeof producerIndex.PlanProtocolUnsupportedError).toBe("function");
|
||||
|
||||
@@ -14,7 +14,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
|
||||
import { join, relative, resolve } from "node:path";
|
||||
|
||||
import type { Fps } from "@hyperframes/core";
|
||||
import { CURRENT_PLAN_PROTOCOL } from "../../distributed/planProtocol.js";
|
||||
import { PLAN_PROTOCOL_V1 } from "../../distributed/planProtocol.js";
|
||||
import {
|
||||
canonicalJsonStringify,
|
||||
computePlanHash,
|
||||
@@ -356,7 +356,7 @@ export async function freezePlan(input: FreezePlanInput): Promise<FreezePlanResu
|
||||
});
|
||||
|
||||
const planJson = {
|
||||
protocol: CURRENT_PLAN_PROTOCOL,
|
||||
protocol: PLAN_PROTOCOL_V1,
|
||||
planHash,
|
||||
producerVersion,
|
||||
ffmpegVersion: encoder.ffmpegVersion,
|
||||
|
||||
Reference in New Issue
Block a user