mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
fix(render): keep segment checkpoints in the local cache (#8137)
* fix(render): keep segment checkpoints in the local cache * fix(producer): read checkpoints through a validated descriptor * fix(render): enforce cache and audio sidecar input policies * fix(render): recapture unreadable segment checkpoints * chore(parsers): remove redundant concat comment
This commit is contained in:
+11
-6
@@ -2146,12 +2146,17 @@ against a single-encoder streaming render, and buys three things:
|
||||
segments a previous attempt finished, after re-validating each one; a
|
||||
resumed render is byte-identical to an uninterrupted one.
|
||||
|
||||
Segments live in `renders/.hf-segments/<hash>/`, where the hash covers
|
||||
everything that determines segment bytes, so changing the composition, fps,
|
||||
resolution, quality or segment size starts a fresh set instead of splicing
|
||||
mismatched encodes. The directory is deleted after a successful render unless
|
||||
`--keep-segments`, and deliberately kept after a failure, because that is what
|
||||
`--resume` reads next time.
|
||||
Segments live in the local HyperFrames cache under `segments/<project-key>/<plan-hash>/`.
|
||||
Run `--resume` from the same project location with the same render settings.
|
||||
Moving a project, changing its settings, or upgrading from a version that stored
|
||||
segments in the project starts a fresh capture. A failed render keeps its
|
||||
checkpoints. A successful render removes them unless you pass `--keep-segments`.
|
||||
After a successful render with the new version, you can delete the old
|
||||
`renders/.hf-segments/` directory to reclaim disk space. If you set
|
||||
`HYPERFRAMES_CACHE_DIR`, choose a private directory outside the project.
|
||||
Segmented and multi-chunk joins require FFmpeg 5.0 or later for per-file
|
||||
options in concat lists. If FFmpeg reports an unknown `option` directive,
|
||||
update the binary used for rendering.
|
||||
|
||||
`--resume` reuses only segments you finished on this machine. Their record is
|
||||
signed with a key derived from your per-user preview token, a private file in
|
||||
|
||||
@@ -462,14 +462,14 @@ export default defineCommand({
|
||||
description:
|
||||
"Segmented capture only (HF_SEGMENTED_CAPTURE=true): reuse segments a " +
|
||||
"previous run of the same composition and settings already finished, " +
|
||||
"recorded in renders/.hf-segments/<hash>/segments.json. Each reused " +
|
||||
"recorded in the local HyperFrames cache. Each reused " +
|
||||
"segment is re-validated before it is skipped.",
|
||||
default: false,
|
||||
},
|
||||
"keep-segments": {
|
||||
type: "boolean",
|
||||
description:
|
||||
"Segmented capture only: keep renders/.hf-segments/<hash> after a " +
|
||||
"Segmented capture only: keep locally cached segments after a " +
|
||||
"successful render instead of deleting it.",
|
||||
default: false,
|
||||
},
|
||||
|
||||
@@ -203,6 +203,7 @@ export {
|
||||
buildEncoderArgs,
|
||||
encodeFramesFromDir,
|
||||
buildConcatArgs,
|
||||
writeVideoConcatList,
|
||||
concatVideoFiles,
|
||||
encodeFramesChunkedConcat,
|
||||
muxVideoWithAudio,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
* so long audio is encoded as frame-aligned segments in parallel and their frames joined.
|
||||
*/
|
||||
|
||||
import { localContainerInputArgs } from "@hyperframes/parsers/ff-binaries";
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { removeScratch } from "@hyperframes/core/remove-scratch";
|
||||
@@ -84,10 +85,11 @@ function supportsSettsDuration(binary = getFfmpegBinary()): Promise<boolean> {
|
||||
function encodeInOnePass(
|
||||
input: string,
|
||||
outputPath: string,
|
||||
inputArgs: string[],
|
||||
{ signal, timeout, filter }: { signal?: AbortSignal; timeout: number; filter?: string },
|
||||
): Promise<RunFfmpegResult> {
|
||||
const filters = [filter, `aresample=${SAMPLE_RATE}`].filter(Boolean).join(",");
|
||||
return runFfmpeg(["-i", input, "-af", filters, ...AAC_ARGS, "-y", outputPath], {
|
||||
return runFfmpeg([...inputArgs, "-i", input, "-af", filters, ...AAC_ARGS, "-y", outputPath], {
|
||||
signal,
|
||||
timeout,
|
||||
});
|
||||
@@ -102,11 +104,19 @@ export async function encodeAac(
|
||||
signal,
|
||||
timeout,
|
||||
filter,
|
||||
inputContainer,
|
||||
segments: count = aacSegmentCount(totalDuration),
|
||||
}: { signal?: AbortSignal; timeout: number; filter?: string; segments?: number },
|
||||
}: {
|
||||
signal?: AbortSignal;
|
||||
timeout: number;
|
||||
filter?: string;
|
||||
segments?: number;
|
||||
inputContainer?: "mov";
|
||||
},
|
||||
): Promise<RunFfmpegResult> {
|
||||
const inputArgs = localContainerInputArgs(inputContainer);
|
||||
if (!(await supportsSettsDuration())) {
|
||||
return encodeInOnePass(input, outputPath, { signal, timeout, filter });
|
||||
return encodeInOnePass(input, outputPath, inputArgs, { signal, timeout, filter });
|
||||
}
|
||||
const dir = createStagingDir(dirname(outputPath), ".aac-segments-");
|
||||
try {
|
||||
@@ -120,6 +130,7 @@ export async function encodeAac(
|
||||
[
|
||||
"-cpuflags",
|
||||
"0",
|
||||
...inputArgs,
|
||||
"-i",
|
||||
input,
|
||||
"-af",
|
||||
|
||||
@@ -4,7 +4,12 @@ import { tmpdir } from "node:os";
|
||||
import { join, relative } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js";
|
||||
import { concatVideoFiles, encodeFramesChunkedConcat } from "./chunkEncoder.js";
|
||||
import {
|
||||
buildConcatArgs,
|
||||
writeVideoConcatList,
|
||||
concatVideoFiles,
|
||||
encodeFramesChunkedConcat,
|
||||
} from "./chunkEncoder.js";
|
||||
import { validJpeg } from "./__fixtures__/jpeg.js";
|
||||
|
||||
const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"]).status === 0;
|
||||
@@ -58,6 +63,34 @@ describe.skipIf(!HAS_FFMPEG)("concat input paths", () => {
|
||||
expect(probe.stdout.trim()).toBe("64,64,30/1,40");
|
||||
}, 30_000);
|
||||
|
||||
it("rejects a non-video demuxer even after a valid first segment", async () => {
|
||||
const clip = join(root, "clip.mp4");
|
||||
const made = spawnSync(getFfmpegBinary(), [
|
||||
"-v",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=size=32x32",
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
clip,
|
||||
]);
|
||||
expect(made.status, made.stderr.toString()).toBe(0);
|
||||
const other = join(root, "second.mp4");
|
||||
writeFileSync(other, "ffconcat version 1.0\n");
|
||||
const list = writeVideoConcatList(root, [other]);
|
||||
const rejected = spawnSync(getFfmpegBinary(), buildConcatArgs(list, join(root, "unused.mp4")), {
|
||||
encoding: "utf8",
|
||||
});
|
||||
expect(rejected.status).not.toBe(0);
|
||||
expect(rejected.stderr).toContain("not on whitelist");
|
||||
const result = await concatVideoFiles([clip, other], join(root, "joined.mp4"));
|
||||
expect(result).toMatchObject({ success: false, externalInterruption: false });
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
"preserves frames and order with mixed input paths, relative first: %s",
|
||||
async (relativeFirst) => {
|
||||
@@ -201,9 +234,7 @@ describe.skipIf(!HAS_FFMPEG)("concat input formats", () => {
|
||||
expect(result.success).toBe(false);
|
||||
}, 30_000);
|
||||
|
||||
it("follows a listed file that is itself an ffconcat list, but only to relative names", async () => {
|
||||
// `concat` stays in the format whitelist because the list itself opens through it, so a nested
|
||||
// list is followed. It opens with the default `-safe 1`, which refuses absolute paths.
|
||||
it("refuses nested lists with relative or absolute names", async () => {
|
||||
const sub = join(root, "sub");
|
||||
mkdirSync(sub);
|
||||
makeClip(join(root, "a.mp4"), 2);
|
||||
@@ -212,10 +243,8 @@ describe.skipIf(!HAS_FFMPEG)("concat input formats", () => {
|
||||
const joined = join(root, "joined.mp4");
|
||||
|
||||
writeFileSync(nested, "ffconcat version 1.0\nfile 'beside.mp4'\n");
|
||||
expect(await concatVideoFiles([join(root, "a.mp4"), nested], joined)).toEqual({
|
||||
success: true,
|
||||
});
|
||||
expect(frameCount(joined)).toBe("5");
|
||||
const relative = await concatVideoFiles([join(root, "a.mp4"), nested], joined);
|
||||
expect(relative.success).toBe(false);
|
||||
|
||||
writeFileSync(nested, `ffconcat version 1.0\nfile '${join(sub, "beside.mp4")}'\n`);
|
||||
const absolute = await concatVideoFiles([join(root, "a.mp4"), nested], joined);
|
||||
|
||||
@@ -314,7 +314,9 @@ describe("encodeFramesChunkedConcat ffmpegEncodeTimeout", () => {
|
||||
expect(dirname(chunkPath)).not.toBe(legacyDir);
|
||||
expect(dirname(list)).toBe(dirname(chunkPath));
|
||||
expect(dirname(dirname(list))).toBe(root);
|
||||
expect(readFileSync(list, "utf8")).toBe(`file '${chunkPath.replace(/'/g, "'\\''")}'\n`);
|
||||
expect(readFileSync(list, "utf8")).toBe(
|
||||
`file '${chunkPath.replace(/'/g, "'\\''")}'\noption format_whitelist mov,matroska\n`,
|
||||
);
|
||||
}
|
||||
expect(readFileSync(legacyList, "utf8")).toBe("another render's concat list");
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
* Supports CPU (libx264) and GPU encoding.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { frameSequencePath } from "./extractedFrameIndex.js";
|
||||
import {
|
||||
closeSync,
|
||||
@@ -50,7 +51,11 @@ import { appendVp9CpuUsedArg } from "./vp9Options.js";
|
||||
import { appendSoftwareEncoderThreadsArg } from "./softwareEncoderThreads.js";
|
||||
import { appendRenderProvenanceArgs } from "../utils/renderProvenance.js";
|
||||
import { createStagingDir } from "@hyperframes/core/atomic-file";
|
||||
import { concatListInputArgs, concatListText } from "@hyperframes/parsers/ff-binaries";
|
||||
import {
|
||||
concatListInputArgs,
|
||||
concatListText,
|
||||
localContainerInputArgs,
|
||||
} from "@hyperframes/parsers/ff-binaries";
|
||||
|
||||
export type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
|
||||
|
||||
@@ -102,6 +107,7 @@ export interface MuxVideoWithAudioOptions extends Partial<
|
||||
* depend on the file extension alone.
|
||||
*/
|
||||
audioCodec?: "aac";
|
||||
audioContainer?: "mov";
|
||||
/** Replaces `-i <videoPath>` (which is then unused), e.g. with a concat-demuxer input. */
|
||||
videoInputArgs?: readonly string[];
|
||||
/**
|
||||
@@ -124,7 +130,9 @@ async function shouldCopyAacSidecar(
|
||||
if (KNOWN_NON_AAC_AUDIO_EXTENSIONS.has(audioExtension)) return false;
|
||||
|
||||
try {
|
||||
const metadata = await extractAudioMetadata(audioPath);
|
||||
const metadata = options?.audioContainer
|
||||
? await extractAudioMetadata(audioPath, { inputContainer: options.audioContainer })
|
||||
: await extractAudioMetadata(audioPath);
|
||||
return metadata.audioCodec === "aac";
|
||||
} catch {
|
||||
// Preserve the pre-existing fallback for invalid or unprobeable sidecars:
|
||||
@@ -683,18 +691,13 @@ export function buildConcatArgs(concatListPath: string, outputPath: string): str
|
||||
return args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sequence rather than a timestamp: two lists written in the same millisecond
|
||||
* into one directory would otherwise collide, and a concat that reads another
|
||||
* render's list produces a silently wrong video rather than an error.
|
||||
*/
|
||||
let concatListSeq = 0;
|
||||
|
||||
function writeConcatList(dir: string, inputPaths: readonly string[]): string {
|
||||
concatListSeq += 1;
|
||||
const listPath = join(dir, `concat-list-${process.pid}-${concatListSeq}.txt`);
|
||||
const body = concatListText(inputPaths.map((p) => resolve(p)));
|
||||
writeFileSync(listPath, body, "utf-8");
|
||||
export function writeVideoConcatList(dir: string, inputPaths: readonly string[]): string {
|
||||
const listPath = join(dir, `concat-list-${randomUUID()}.txt`);
|
||||
const body = concatListText(
|
||||
inputPaths.map((path) => resolve(path)),
|
||||
{ videoContainersOnly: true },
|
||||
);
|
||||
writeFileSync(listPath, body, { flag: "wx", mode: 0o600 });
|
||||
return listPath;
|
||||
}
|
||||
|
||||
@@ -723,7 +726,7 @@ export async function concatVideoFiles(
|
||||
// and `--debug` keeps both so a bad concat can be reproduced from its list.
|
||||
let listPath: string;
|
||||
try {
|
||||
listPath = writeConcatList(dirname(firstInput), inputPaths);
|
||||
listPath = writeVideoConcatList(dirname(firstInput), inputPaths);
|
||||
} catch (error) {
|
||||
return { success: false, error: (error as Error).message, externalInterruption: false };
|
||||
}
|
||||
@@ -891,7 +894,14 @@ export async function muxVideoWithAudio(
|
||||
const isWebm = outputExtension === ".webm";
|
||||
const isMov = outputExtension === ".mov";
|
||||
const shouldCopyAudio = isWebm ? false : await shouldCopyAacSidecar(audioPath, config);
|
||||
const args = [...(config?.videoInputArgs ?? ["-i", videoPath]), "-i", audioPath, "-c:v", "copy"];
|
||||
const args = [
|
||||
...(config?.videoInputArgs ?? ["-i", videoPath]),
|
||||
...localContainerInputArgs(config?.audioContainer),
|
||||
"-i",
|
||||
audioPath,
|
||||
"-c:v",
|
||||
"copy",
|
||||
];
|
||||
|
||||
if (isWebm) {
|
||||
args.push("-c:a", "libopus", "-b:a", "128k");
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// fallow-ignore-file code-duplication complexity
|
||||
import { localContainerInputArgs } from "@hyperframes/parsers/ff-binaries";
|
||||
import { spawn } from "child_process";
|
||||
import { createReadStream, mkdtempSync, readFileSync, rmSync, statSync } from "fs";
|
||||
import { chunkCrc32 } from "./crc32.js";
|
||||
@@ -1333,10 +1334,12 @@ export const extractVideoMetadata = extractMediaMetadata;
|
||||
|
||||
export async function extractAudioMetadata(
|
||||
filePath: string,
|
||||
options?: { signal?: AbortSignal },
|
||||
options?: { signal?: AbortSignal; inputContainer?: "mov" },
|
||||
): Promise<AudioMetadata> {
|
||||
// A probe bound to a caller's abort signal is shared only among callers of that same signal,
|
||||
// and only when the file has an identity to key it by.
|
||||
if (options?.inputContainer)
|
||||
return probeAudioMetadata(filePath, options.signal, options.inputContainer);
|
||||
const identity = mediaFileIdentity(filePath);
|
||||
const signal = options?.signal;
|
||||
if (signal && identity === null) return probeAudioMetadata(filePath, signal);
|
||||
@@ -1352,8 +1355,17 @@ export async function extractAudioMetadata(
|
||||
return probePromise;
|
||||
}
|
||||
|
||||
async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promise<AudioMetadata> {
|
||||
const output = await probeMediaOutput(filePath, signal);
|
||||
async function probeAudioMetadata(
|
||||
filePath: string,
|
||||
signal?: AbortSignal,
|
||||
inputContainer?: "mov",
|
||||
): Promise<AudioMetadata> {
|
||||
const inputArgs = localContainerInputArgs(inputContainer);
|
||||
const output = inputContainer
|
||||
? refuseTextArt(
|
||||
parseProbeJson(await runFfprobe(filePath, [...PROBE_JSON_ARGS, ...inputArgs], signal)),
|
||||
)
|
||||
: await probeMediaOutput(filePath, signal);
|
||||
const audioStreams = output.streams.filter((s) => s.codec_type === "audio");
|
||||
const audioStreamIndex = decodableAudioStream(output.streams);
|
||||
const audioStream = audioStreamIndex === null ? undefined : audioStreams[audioStreamIndex];
|
||||
@@ -1373,6 +1385,7 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi
|
||||
audioStartSeconds(audioStream, output.format),
|
||||
durationSeconds,
|
||||
signal,
|
||||
inputArgs,
|
||||
);
|
||||
if (decoded !== null) {
|
||||
durationSeconds = reconcileAudioDuration(durationSeconds, decoded, output.format);
|
||||
@@ -1455,6 +1468,7 @@ async function probeDecodedAudioDuration(
|
||||
startSeconds: number,
|
||||
containerDurationSeconds: number,
|
||||
signal?: AbortSignal,
|
||||
inputArgs: string[] = [],
|
||||
): Promise<number | null> {
|
||||
const probe = async (
|
||||
readInterval?: string,
|
||||
@@ -1471,7 +1485,7 @@ async function probeDecodedAudioDuration(
|
||||
try {
|
||||
const stdout = await runFfprobe(
|
||||
filePath,
|
||||
args,
|
||||
[...args, ...inputArgs],
|
||||
signal,
|
||||
// CSV, not JSON: a tail truncated under the size cap still parses.
|
||||
{ retainTail: true, maxChars: 64 * 1024 },
|
||||
|
||||
@@ -252,7 +252,11 @@ export function clearFfBinaryLookupCache(): void {
|
||||
const endsAListLine = (text: string) => ["\n", "\r", "\u0000"].some((char) => text.includes(char));
|
||||
|
||||
/** One `file` entry per path; a path holding what ends a list line (LF, CR, NUL) is refused. */
|
||||
export function concatListText(paths: readonly string[]): string {
|
||||
export function concatListText(
|
||||
paths: readonly string[],
|
||||
options: { videoContainersOnly?: boolean } = {},
|
||||
): string {
|
||||
const entryOptions = options.videoContainersOnly ? "option format_whitelist mov,matroska\n" : "";
|
||||
return paths
|
||||
.map((path) => {
|
||||
if (endsAListLine(path)) {
|
||||
@@ -260,7 +264,7 @@ export function concatListText(paths: readonly string[]): string {
|
||||
`A file to join holds a line break or NUL in its path: ${JSON.stringify(path)}`,
|
||||
);
|
||||
}
|
||||
return `file '${path.replace(/'/g, "'\\''")}'\n`;
|
||||
return `file '${path.replace(/'/g, "'\\''")}'\n${entryOptions}`;
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
@@ -332,6 +336,10 @@ export function _setDemuxerListingForTests(listing?: string): void {
|
||||
demuxerLists.clear();
|
||||
}
|
||||
|
||||
export function localContainerInputArgs(container: "mov" | "matroska" | undefined): string[] {
|
||||
return container ? ["-protocol_whitelist", "file", "-format_whitelist", container] : [];
|
||||
}
|
||||
|
||||
export { assertLiteralMediaInput };
|
||||
|
||||
/** Every demuxer `listingBinary` reports that opens nothing by itself and is not text art, for reading `input`; one that cannot be asked gets the self-contained list, and is asked again. */
|
||||
|
||||
@@ -265,6 +265,43 @@ function diskKb(path: string): number {
|
||||
describe("assemble()", () => {
|
||||
const TIMEOUT_MS = 30_000;
|
||||
|
||||
it("rejects a WAV file in place of the planned M4A sidecar", async () => {
|
||||
if (!hasFfmpeg) return;
|
||||
const { planDir, chunkPaths, audioPath } = buildAudioFixture();
|
||||
const wav = spawnSync("ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:duration=1",
|
||||
"-f",
|
||||
"wav",
|
||||
audioPath,
|
||||
]);
|
||||
expect(wav.status).toBe(0);
|
||||
const output = join(planDir, "audio-check.mp4");
|
||||
await expect(assemble(planDir, chunkPaths, audioPath, output)).rejects.toThrow();
|
||||
expect(existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects an input list in place of an encoded chunk", async () => {
|
||||
if (!hasFfmpeg) return;
|
||||
const chunks = [
|
||||
{ index: 0, startFrame: 0, endFrame: 5 },
|
||||
{ index: 1, startFrame: 5, endFrame: 10 },
|
||||
];
|
||||
const planDir = buildPlanDir("mp4", chunks, 10, false);
|
||||
const video = join(planDir, "video.mp4");
|
||||
const list = join(planDir, "list.mp4");
|
||||
makeMp4Chunk(video, 5);
|
||||
writeFileSync(list, "ffconcat version 1.0\n");
|
||||
const output = join(planDir, "output.mp4");
|
||||
await expect(assemble(planDir, [video, list], null, output)).rejects.toThrow();
|
||||
expect(existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it(
|
||||
"concat-copies two mp4 chunks and applies faststart",
|
||||
async () => {
|
||||
|
||||
@@ -32,14 +32,14 @@ import {
|
||||
readdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { localContainerInputArgs } from "@hyperframes/parsers/ff-binaries";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import {
|
||||
appendRateCapArgs,
|
||||
writeVideoConcatList,
|
||||
appendRenderProvenanceArgs,
|
||||
concatListInputArgs,
|
||||
concatListText,
|
||||
describeFfmpegFailure,
|
||||
getEncoderPreset,
|
||||
MIXED_AUDIO_FILENAME,
|
||||
@@ -282,8 +282,12 @@ interface JoinContext {
|
||||
// container), so a single file is read directly; every write also sets output `-r`.
|
||||
function videoInputArgs(ctx: JoinContext, videoSources: readonly string[]): string[] {
|
||||
return videoSources.length === 1
|
||||
? ["-i", videoSources[0]!]
|
||||
: concatDemuxerInput(videoSources, ctx.workDir, ctx.fpsArg);
|
||||
? [
|
||||
...localContainerInputArgs(ctx.format === "webm" ? "matroska" : "mov"),
|
||||
"-i",
|
||||
videoSources[0]!,
|
||||
]
|
||||
: ["-r", ctx.fpsArg, ...concatListInputArgs(writeVideoConcatList(ctx.workDir, videoSources))];
|
||||
}
|
||||
|
||||
export function cfrEncodeArgs(
|
||||
@@ -354,8 +358,9 @@ async function writeJoined(
|
||||
const padTrimResult = await ctx.timed("audioMs", () =>
|
||||
padOrTrimAudioToVideoFrameCount({
|
||||
videoPath: videoSources[0]!,
|
||||
probeVideoFrameInfo: () => probeJoinedFrameInfo(videoSources, ctx.fps, ctx.signal),
|
||||
probeVideoFrameInfo: () => probeJoinedFrameInfo(videoSources, ctx),
|
||||
audioPath,
|
||||
audioContainer: "mov",
|
||||
outputPath: paddedAudioPath,
|
||||
signal: ctx.signal,
|
||||
timeoutMs: ctx.timeout,
|
||||
@@ -377,6 +382,7 @@ async function writeJoined(
|
||||
ctx.signal,
|
||||
{
|
||||
audioCodec: "aac",
|
||||
audioContainer: "mov",
|
||||
ffmpegProcessTimeout: ctx.timeout,
|
||||
videoInputArgs: videoInputArgs(ctx, videoSources),
|
||||
},
|
||||
@@ -388,30 +394,20 @@ async function writeJoined(
|
||||
}
|
||||
}
|
||||
|
||||
/** Concat-demuxer input args; `-r` keeps the demuxer from PTS-averaging the rate. */
|
||||
function concatDemuxerInput(
|
||||
chunkPaths: readonly string[],
|
||||
workDir: string,
|
||||
fpsArg: string,
|
||||
): string[] {
|
||||
const listPath = join(workDir, "concat-list.txt");
|
||||
writeFileSync(listPath, concatListText(chunkPaths), "utf-8");
|
||||
return ["-r", fpsArg, ...concatListInputArgs(listPath)];
|
||||
}
|
||||
|
||||
/** What a probe of the joined file would report: the chunks' frames at the plan's rate. */
|
||||
async function probeJoinedFrameInfo(
|
||||
videoPaths: readonly string[],
|
||||
fps: { num: number; den: number },
|
||||
signal: AbortSignal | undefined,
|
||||
ctx: JoinContext,
|
||||
): Promise<ProbeVideoFrameInfo> {
|
||||
const infos = await Promise.all(
|
||||
videoPaths.map((path) => defaultProbeVideoFrameInfo(path, signal)),
|
||||
videoPaths.map((path) =>
|
||||
defaultProbeVideoFrameInfo(path, ctx.signal, ctx.format === "webm" ? "matroska" : "mov"),
|
||||
),
|
||||
);
|
||||
return {
|
||||
frameCount: infos.reduce((sum, info) => sum + info.frameCount, 0),
|
||||
fpsNum: fps.num,
|
||||
fpsDen: fps.den,
|
||||
fpsNum: ctx.fps.num,
|
||||
fpsDen: ctx.fps.den,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
* can stretch the terminal packet in the final MP4.
|
||||
*/
|
||||
|
||||
import { localContainerInputArgs } from "@hyperframes/parsers/ff-binaries";
|
||||
import { spawn } from "node:child_process";
|
||||
import { dirname, join } from "node:path";
|
||||
import {
|
||||
@@ -73,6 +74,7 @@ export interface AudioProbeInfo {
|
||||
}
|
||||
|
||||
export interface PadTrimAudioInput {
|
||||
audioContainer?: "mov";
|
||||
/** Path to the assembled video. Used to derive `frameCount / fps`. */
|
||||
videoPath: string;
|
||||
/** Path to the pre-mixed audio (typically `<planDir>/audio.m4a`). */
|
||||
@@ -236,11 +238,16 @@ export async function padOrTrimAudioToVideoFrameCount(
|
||||
const probeVideo =
|
||||
input.probeVideoFrameInfo ??
|
||||
((videoPath: string) => defaultProbeVideoFrameInfo(videoPath, input.signal));
|
||||
const probeAudio = input.probeAudioInfo ?? defaultProbeAudioInfo;
|
||||
const probeAudio =
|
||||
input.probeAudioInfo ??
|
||||
((path: string, signal?: AbortSignal) =>
|
||||
defaultProbeAudioInfo(path, signal, input.audioContainer));
|
||||
const timeoutMs = input.timeoutMs ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
|
||||
const runner =
|
||||
input.runFfmpeg ?? ((args: string[]) => defaultRunFfmpeg(args, timeoutMs, input.signal));
|
||||
const encode = input.encodeAac ?? defaultAacEncoder(timeoutMs, input.signal, input.sourcePath);
|
||||
const encode =
|
||||
input.encodeAac ??
|
||||
defaultAacEncoder(timeoutMs, input.signal, input.sourcePath, input.audioContainer);
|
||||
// Injected runners usually do not materialize media. Tests that exercise
|
||||
// peak correction opt in with a matching probe; production always uses the
|
||||
// real post-codec measurement.
|
||||
@@ -313,7 +320,7 @@ export async function padOrTrimAudioToVideoFrameCount(
|
||||
for (const step of plan.steps) {
|
||||
const ffmpegResult =
|
||||
step.kind === "copy"
|
||||
? await runner(step.args)
|
||||
? await runner([...localContainerInputArgs(input.audioContainer), ...step.args])
|
||||
: await encode(
|
||||
input.sourcePath ?? input.audioPath,
|
||||
input.outputPath,
|
||||
@@ -569,11 +576,13 @@ interface FfprobeOutput {
|
||||
export async function defaultProbeVideoFrameInfo(
|
||||
videoPath: string,
|
||||
signal?: AbortSignal,
|
||||
container?: "mov" | "matroska",
|
||||
): Promise<ProbeVideoFrameInfo> {
|
||||
// Try the container header (`nb_frames`) first — single moov atom read,
|
||||
// no decode. Closed-GOP, B-frame-free streams (the only ones we'll ever
|
||||
// ask to pad/trim) reliably set it. Fall back to `-count_packets` which
|
||||
// walks the packet stream when the header doesn't carry the count.
|
||||
const inputArgs = localContainerInputArgs(container);
|
||||
const fastInfo = await runFfprobeJson<FfprobeOutput>(
|
||||
[
|
||||
"-v",
|
||||
@@ -584,6 +593,7 @@ export async function defaultProbeVideoFrameInfo(
|
||||
"stream=nb_frames,r_frame_rate",
|
||||
"-of",
|
||||
"json",
|
||||
...inputArgs,
|
||||
"--",
|
||||
videoPath,
|
||||
],
|
||||
@@ -605,6 +615,7 @@ export async function defaultProbeVideoFrameInfo(
|
||||
"stream=nb_read_packets,r_frame_rate",
|
||||
"-of",
|
||||
"json",
|
||||
...inputArgs,
|
||||
"--",
|
||||
videoPath,
|
||||
],
|
||||
@@ -632,10 +643,11 @@ function parseFrameRate(rate: string): { fpsNum: number; fpsDen: number } {
|
||||
async function defaultProbeAudioInfo(
|
||||
audioPath: string,
|
||||
signal?: AbortSignal,
|
||||
inputContainer?: "mov",
|
||||
): Promise<AudioProbeInfo> {
|
||||
// The shared ffprobe wrapper corrects for a lying container duration so
|
||||
// every consumer sees the same VBR-safe metadata while preserving cancellation.
|
||||
const metadata: AudioMetadata = await extractAudioMetadata(audioPath, { signal });
|
||||
const metadata: AudioMetadata = await extractAudioMetadata(audioPath, { signal, inputContainer });
|
||||
return {
|
||||
durationSeconds: metadata.durationSeconds,
|
||||
sampleRate: metadata.sampleRate,
|
||||
@@ -656,12 +668,13 @@ function defaultAacEncoder(
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal,
|
||||
pcmSourcePath?: string,
|
||||
inputContainer?: "mov",
|
||||
): AacEncoder {
|
||||
return async (inputPath, outputPath, durationSeconds, filter) => {
|
||||
// Each segment decodes its input from the start: cheap from PCM, a second lossy pass from AAC.
|
||||
const options = pcmSourcePath
|
||||
? { signal, timeout: timeoutMs, filter }
|
||||
: { signal, timeout: timeoutMs, filter, segments: 1 };
|
||||
: { signal, timeout: timeoutMs, filter, segments: 1, inputContainer };
|
||||
return toStepResult(
|
||||
await encodeAac(pcmSourcePath ?? inputPath, outputPath, durationSeconds, options),
|
||||
timeoutMs,
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
chmodSync,
|
||||
cpSync,
|
||||
symlinkSync,
|
||||
linkSync,
|
||||
mkdtempSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
mkdirSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { previewToken } from "@hyperframes/core/preview-proof";
|
||||
@@ -17,7 +26,7 @@ import {
|
||||
type SegmentManifestEntry,
|
||||
type SegmentPlanHashInput,
|
||||
} from "./segmentManifest.js";
|
||||
import { planSegments, segmentOutputPath } from "./segmentPlan.js";
|
||||
import { segmentOutputPath } from "./segmentPlan.js";
|
||||
|
||||
const hashInput: SegmentPlanHashInput = {
|
||||
compositionHash: "abc",
|
||||
@@ -123,6 +132,29 @@ describe("manifest io", () => {
|
||||
expect(readSegmentManifest(dir)).toBeNull();
|
||||
});
|
||||
|
||||
it("skips linked manifests even when they contain a valid checkpoint", () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
const copy = join(dir, "copy");
|
||||
mkdirSync(copy, { mode: 0o700 });
|
||||
const manifest: SegmentManifest = {
|
||||
version: 1,
|
||||
planHash: "abc",
|
||||
totalFrames: 3,
|
||||
segmentFrames: 3,
|
||||
completed: [],
|
||||
};
|
||||
writeSegmentManifest(dir, manifest, null);
|
||||
const source = join(dir, "segments.json");
|
||||
const path = join(copy, "segments.json");
|
||||
symlinkSync(source, path);
|
||||
expect(readSegmentManifest(copy)).toBeNull();
|
||||
rmSync(path);
|
||||
linkSync(source, path);
|
||||
expect(readSegmentManifest(copy)).toBeNull();
|
||||
rmSync(path);
|
||||
expect(readSegmentManifest(dir)).toEqual(manifest);
|
||||
});
|
||||
|
||||
it("rejects a manifest whose shape is wrong rather than resuming from it", () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
writeFileSync(join(dir, "segments.json"), JSON.stringify({ version: 2, completed: [] }));
|
||||
@@ -140,9 +172,48 @@ describe("manifest io", () => {
|
||||
expect(readSegmentManifest(dir)).toBeNull();
|
||||
});
|
||||
|
||||
it("segmentDirFor nests under renders/.hf-segments", () => {
|
||||
it.skipIf(!process.getuid)("reports an explicitly configured writable cache", () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
expect(segmentDirFor("/p/renders", "abc")).toBe(join("/p/renders", ".hf-segments", "abc"));
|
||||
const project = join(dir, "project");
|
||||
const cache = join(dir, "shared-cache");
|
||||
mkdirSync(project, { mode: 0o700 });
|
||||
mkdirSync(cache);
|
||||
chmodSync(cache, 0o770);
|
||||
vi.stubEnv("HYPERFRAMES_CACHE_DIR", cache);
|
||||
try {
|
||||
expect(() => segmentDirFor(project, "abc")).toThrow("can be changed by another user account");
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("only finds checkpoints from the local cache for the same project", () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
const project = join(dir, "project");
|
||||
const copy = join(dir, "copy");
|
||||
mkdirSync(project, { mode: 0o700 });
|
||||
mkdirSync(copy, { mode: 0o700 });
|
||||
vi.stubEnv("HYPERFRAMES_CACHE_DIR", join(dir, "cache-a"));
|
||||
try {
|
||||
const checkpoint = segmentDirFor(project, "abc");
|
||||
const manifest: SegmentManifest = {
|
||||
version: 1,
|
||||
planHash: "abc",
|
||||
totalFrames: 3,
|
||||
segmentFrames: 3,
|
||||
completed: [],
|
||||
};
|
||||
writeSegmentManifest(checkpoint, manifest, null);
|
||||
expect(readSegmentManifest(segmentDirFor(project, "abc"))).toEqual(manifest);
|
||||
expect(readSegmentManifest(segmentDirFor(copy, "abc"))).toBeNull();
|
||||
cpSync(checkpoint, join(project, "renders", ".hf-segments", "abc"), { recursive: true });
|
||||
vi.stubEnv("HYPERFRAMES_CACHE_DIR", join(dir, "cache-b"));
|
||||
expect(readSegmentManifest(segmentDirFor(project, "abc"))).toBeNull();
|
||||
vi.stubEnv("HYPERFRAMES_CACHE_DIR", join(project, "cache"));
|
||||
expect(() => segmentDirFor(project, "abc")).toThrow("outside the project");
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -183,7 +254,8 @@ describe("validateCompletedSegments", () => {
|
||||
) => ({
|
||||
expectedHash,
|
||||
segmentDir: dir,
|
||||
plan: planSegments(totalFrames, 3),
|
||||
totalFrames,
|
||||
segmentFrames: 3,
|
||||
key: KEY,
|
||||
probeFrames,
|
||||
});
|
||||
@@ -246,6 +318,7 @@ describe("validateCompletedSegments", () => {
|
||||
const elsewhere = join(dir, "elsewhere.mp4");
|
||||
writeFileSync(elsewhere, "xx");
|
||||
const named = [elsewhere, "/etc/hosts", "\\\\host\\share\\segment.mp4"];
|
||||
for (const index of named.keys()) writeFileSync(segmentOutputPath(dir, index), "xx");
|
||||
const manifest = manifestOf(named.map((path, index) => ({ ...entry(index), path })));
|
||||
const probed: string[] = [];
|
||||
const probe = async (path: string) => {
|
||||
@@ -265,6 +338,17 @@ describe("validateCompletedSegments", () => {
|
||||
expect(await reuses(manifest, 3)).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects a hard-linked segment before probing it", async () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
const path = segmentOutputPath(dir, 0);
|
||||
writeFileSync(path, "xx");
|
||||
const manifest = manifestOf([entry(0)]);
|
||||
linkSync(path, join(dir, "second-name.mp4"));
|
||||
const probe = vi.fn(async () => 3);
|
||||
expect((await validateCompletedSegments(manifest, check(3, probe))).size).toBe(0);
|
||||
expect(probe).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores entries for segments this plan does not have", async () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
for (const index of [0, 1, 5]) writeFileSync(segmentOutputPath(dir, index), "xx");
|
||||
@@ -306,6 +390,22 @@ describe("validateCompletedSegments", () => {
|
||||
expect(probed).toBe(0);
|
||||
});
|
||||
|
||||
it.skipIf(!process.getuid || process.getuid() === 0)(
|
||||
"recaptures an unreadable segment and still validates the next checkpoint",
|
||||
async () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
const unreadable = segmentOutputPath(dir, 0);
|
||||
const readable = segmentOutputPath(dir, 1);
|
||||
writeFileSync(unreadable, "xx");
|
||||
writeFileSync(readable, "xx");
|
||||
const manifest = manifestOf([entry(0), entry(1)]);
|
||||
chmodSync(unreadable, 0o000);
|
||||
const probe = vi.fn(async (_path: string) => 3);
|
||||
expect([...(await validateCompletedSegments(manifest, check(6, probe)))]).toEqual([1]);
|
||||
expect(probe.mock.calls).toEqual([[readable]]);
|
||||
},
|
||||
);
|
||||
|
||||
it("hashes a segment file's contents", async () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
writeFileSync(segmentOutputPath(dir, 0), "xx");
|
||||
|
||||
@@ -8,17 +8,26 @@
|
||||
*/
|
||||
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
|
||||
import {
|
||||
closeSync,
|
||||
constants,
|
||||
createReadStream,
|
||||
existsSync,
|
||||
fstatSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
assertPrivateCachePath,
|
||||
ensurePrivateCacheDir,
|
||||
hyperframesCacheDir,
|
||||
} from "@hyperframes/core/cache-dir";
|
||||
import { isSafePath, realpath } from "@hyperframes/core/safe-path";
|
||||
import { isPathInside } from "@hyperframes/parsers/asset-paths";
|
||||
import { previewToken } from "@hyperframes/core/preview-proof";
|
||||
import { renameWithRetry } from "@hyperframes/engine";
|
||||
import { segmentOutputPath, type SegmentSlice } from "./segmentPlan.js";
|
||||
import { planSegments, segmentOutputPath, type SegmentSlice } from "./segmentPlan.js";
|
||||
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
|
||||
export interface SegmentManifestEntry {
|
||||
index: number;
|
||||
@@ -97,8 +106,25 @@ export function computeSegmentPlanHash(input: SegmentPlanHashInput): string {
|
||||
return createHash("sha256").update(ordered.join(" ")).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
export function segmentDirFor(projectRendersDir: string, planHash: string): string {
|
||||
return join(projectRendersDir, ".hf-segments", planHash);
|
||||
export function segmentDirFor(projectDir: string, planHash: string): string {
|
||||
const root = hyperframesCacheDir();
|
||||
const projectKey = createHash("sha256").update(realpath(projectDir)).digest("hex");
|
||||
const parent = join(root, "segments", projectKey);
|
||||
const dir = join(parent, planHash);
|
||||
if (
|
||||
isPathInside(root, projectDir) ||
|
||||
isSafePath(projectDir, root) ||
|
||||
isSafePath(projectDir, parent) ||
|
||||
isSafePath(projectDir, dir)
|
||||
) {
|
||||
throw new Error("Segmented capture requires HYPERFRAMES_CACHE_DIR outside the project.");
|
||||
}
|
||||
assertPrivateCachePath(root, root);
|
||||
ensurePrivateCacheDir(parent, root);
|
||||
if (!isSafePath(parent, dir) || lstatSync(dir, { throwIfNoEntry: false })?.isSymbolicLink()) {
|
||||
throw new Error("Invalid segment cache directory");
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
function hasNumber(value: object, key: string): boolean {
|
||||
@@ -137,14 +163,25 @@ function isManifest(value: unknown): value is SegmentManifest {
|
||||
|
||||
export function readSegmentManifest(segmentDir: string): SegmentManifest | null {
|
||||
const path = join(segmentDir, MANIFEST_FILENAME);
|
||||
if (!existsSync(path)) return null;
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(readFileSync(path, "utf-8"));
|
||||
fd = openSync(
|
||||
path,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0),
|
||||
);
|
||||
const opened = fstatSync(fd);
|
||||
if (!opened.isFile() || opened.nlink !== 1) return null;
|
||||
// Windows lacks O_NOFOLLOW. Verify the directory entry still names the opened file.
|
||||
const named = lstatSync(path);
|
||||
if (!named.isFile() || named.dev !== opened.dev || named.ino !== opened.ino) return null;
|
||||
const parsed: unknown = JSON.parse(readFileSync(fd, "utf-8"));
|
||||
return isManifest(parsed) ? parsed : null;
|
||||
} catch {
|
||||
// A truncated or hand-edited manifest means "no resume", never a crash:
|
||||
// the render still has every frame it needs to capture from scratch.
|
||||
return null;
|
||||
} finally {
|
||||
if (fd !== undefined) closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,9 +229,11 @@ export function writeSegmentManifest(
|
||||
key: Buffer | null,
|
||||
): void {
|
||||
mkdirSync(segmentDir, { recursive: true });
|
||||
const tmp = join(segmentDir, `${MANIFEST_FILENAME}.${process.pid}.tmp`);
|
||||
writeFileSync(tmp, JSON.stringify(signSegmentManifest(manifest, key), null, 2), "utf-8");
|
||||
renameWithRetry(tmp, join(segmentDir, MANIFEST_FILENAME));
|
||||
replaceFileAtomically(
|
||||
join(segmentDir, MANIFEST_FILENAME),
|
||||
JSON.stringify(signSegmentManifest(manifest, key), null, 2),
|
||||
0o600,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -224,8 +263,8 @@ export async function probeSegmentFrameCount(
|
||||
export interface SegmentResumeCheck {
|
||||
expectedHash: string;
|
||||
segmentDir: string;
|
||||
/** This render's segments; manifest entries are trusted only where they match one. */
|
||||
plan: readonly SegmentSlice[];
|
||||
totalFrames: number;
|
||||
segmentFrames: number;
|
||||
key: Buffer | null;
|
||||
probeFrames: (path: string) => Promise<number | null>;
|
||||
}
|
||||
@@ -241,9 +280,17 @@ function plannedSlice(
|
||||
return slice;
|
||||
}
|
||||
|
||||
function isPlainFileOfSize(path: string, bytes: number): boolean {
|
||||
const stat = lstatSync(path, { throwIfNoEntry: false });
|
||||
return stat !== undefined && stat.isFile() && stat.size > 0 && stat.size === bytes;
|
||||
async function matchesSegmentFile(path: string, entry: SegmentManifestEntry): Promise<boolean> {
|
||||
try {
|
||||
const stat = lstatSync(path, { throwIfNoEntry: false });
|
||||
if (!stat?.isFile() || stat.nlink !== 1 || stat.size <= 0 || stat.size !== entry.bytes) {
|
||||
return false;
|
||||
}
|
||||
return (await hashSegmentFile(path)) === entry.sha256;
|
||||
} catch {
|
||||
// A checkpoint can disappear or become unreadable before hashing finishes.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -261,13 +308,19 @@ export async function validateCompletedSegments(
|
||||
check: SegmentResumeCheck,
|
||||
): Promise<Set<number>> {
|
||||
const ok = new Set<number>();
|
||||
if (manifest.planHash !== check.expectedHash || !hasValidMac(manifest, check.key)) return ok;
|
||||
if (
|
||||
manifest.totalFrames !== check.totalFrames ||
|
||||
manifest.segmentFrames !== check.segmentFrames ||
|
||||
manifest.planHash !== check.expectedHash ||
|
||||
!hasValidMac(manifest, check.key)
|
||||
)
|
||||
return ok;
|
||||
const plan = planSegments(check.totalFrames, check.segmentFrames);
|
||||
for (const entry of manifest.completed) {
|
||||
const slice = plannedSlice(entry, check.plan);
|
||||
const slice = plannedSlice(entry, plan);
|
||||
if (!slice) continue;
|
||||
const path = segmentOutputPath(check.segmentDir, slice.index);
|
||||
if (!isPlainFileOfSize(path, entry.bytes)) continue;
|
||||
if ((await hashSegmentFile(path)) !== entry.sha256) continue;
|
||||
if (entry.path !== path || !(await matchesSegmentFile(path, entry))) continue;
|
||||
const frames = await check.probeFrames(path);
|
||||
if (frames !== slice.endFrame - slice.startFrame) continue;
|
||||
ok.add(slice.index);
|
||||
|
||||
@@ -269,6 +269,7 @@ async function captureSegmentToEncoder(
|
||||
segment: SegmentSlice,
|
||||
): Promise<StreamingEncoderResult> {
|
||||
const segmentPath = segmentOutputPath(ctx.segmentDir, segment.index);
|
||||
rmSync(segmentPath, { force: true });
|
||||
let encoder: StreamingEncoder;
|
||||
try {
|
||||
encoder = await ctx.spawnEncoder(
|
||||
|
||||
@@ -185,6 +185,9 @@ afterEach(() => {
|
||||
function render(resumeSegments = false): Promise<void> {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-segmented-de-retry-"));
|
||||
dirs.push(root);
|
||||
const cache = mkdtempSync(join(tmpdir(), "hf-segment-cache-"));
|
||||
dirs.push(cache);
|
||||
vi.stubEnv("HYPERFRAMES_CACHE_DIR", cache);
|
||||
writeFileSync(join(root, "index.html"), "<div></div>");
|
||||
const job = createRenderJob({
|
||||
fps: 30,
|
||||
|
||||
@@ -195,7 +195,7 @@ import {
|
||||
startRenderEtaGuard,
|
||||
} from "./render/renderEtaGuard.js";
|
||||
import { checkPageNeverReady } from "./render/pageNeverReady.js";
|
||||
import { planSegments, resolveSegmentFrames } from "./render/segmentPlan.js";
|
||||
import { resolveSegmentFrames } from "./render/segmentPlan.js";
|
||||
import { extractAllDeferredFrames } from "./render/deferredVideoFrames.js";
|
||||
import { measureLocalShownBoxes, oversizedVideos } from "./render/videoShownBoxes.js";
|
||||
import { createSegmentVideoFrames } from "./render/segmentVideoFrames.js";
|
||||
@@ -388,7 +388,7 @@ export interface RenderConfig {
|
||||
quality: "draft" | "standard" | "high";
|
||||
/** Segmented capture: reuse segments recorded in a matching manifest (Phase 2b). */
|
||||
resumeSegments?: boolean;
|
||||
/** Segmented capture: keep renders/.hf-segments/<planHash> after a successful render. */
|
||||
/** Segmented capture: keep the local segment cache after a successful render. */
|
||||
keepSegments?: boolean;
|
||||
/**
|
||||
* Output container format. Defaults to `"mp4"`; set explicitly for other formats.
|
||||
@@ -5111,31 +5111,8 @@ async function executeRenderPipeline(input: {
|
||||
pageMotionBlur: job.config.pageMotionBlur,
|
||||
captureQuality: captureOptions.quality,
|
||||
});
|
||||
const segmentDir = segmentDirFor(join(projectDir, "renders"), segmentPlanHash);
|
||||
const segmentDir = segmentDirFor(projectDir, segmentPlanHash);
|
||||
const manifestKey = loadResumeKeyOrWarn(job.config.resumeSegments === true, log);
|
||||
let completedSegments: ReadonlySet<number> = new Set<number>();
|
||||
if (job.config.resumeSegments === true) {
|
||||
const existing = readSegmentManifest(segmentDir);
|
||||
if (existing) {
|
||||
completedSegments = await validateCompletedSegments(existing, {
|
||||
expectedHash: segmentPlanHash,
|
||||
segmentDir,
|
||||
plan: planSegments(totalFrames, segmentFrames),
|
||||
key: manifestKey,
|
||||
probeFrames: (path) => probeSegmentFrameCount(path, extractMediaMetadata),
|
||||
});
|
||||
log.info(`[Render] resuming: ${completedSegments.size} segments complete`, {
|
||||
segmentDir,
|
||||
});
|
||||
}
|
||||
if (completedSegments.size === 0) {
|
||||
rmSync(segmentDir, { recursive: true, force: true });
|
||||
}
|
||||
} else {
|
||||
// Without --resume the directory is stale by definition: a previous
|
||||
// run's segments must never be spliced into this output.
|
||||
rmSync(segmentDir, { recursive: true, force: true });
|
||||
}
|
||||
const emptySegmentManifest = (): SegmentManifest => ({
|
||||
version: 1,
|
||||
planHash: segmentPlanHash,
|
||||
@@ -5143,7 +5120,26 @@ async function executeRenderPipeline(input: {
|
||||
segmentFrames,
|
||||
completed: [],
|
||||
});
|
||||
let segmentManifest = readSegmentManifest(segmentDir) ?? emptySegmentManifest();
|
||||
let segmentManifest = emptySegmentManifest();
|
||||
let completedSegments: ReadonlySet<number> = new Set<number>();
|
||||
const existing = job.config.resumeSegments ? readSegmentManifest(segmentDir) : null;
|
||||
if (existing) {
|
||||
completedSegments = await validateCompletedSegments(existing, {
|
||||
expectedHash: segmentPlanHash,
|
||||
segmentDir,
|
||||
totalFrames,
|
||||
segmentFrames,
|
||||
key: manifestKey,
|
||||
probeFrames: (path) => probeSegmentFrameCount(path, extractMediaMetadata),
|
||||
});
|
||||
segmentManifest.completed = existing.completed.filter((entry) =>
|
||||
completedSegments.has(entry.index),
|
||||
);
|
||||
log.info(`[Render] resuming: ${completedSegments.size} segments complete`, {
|
||||
segmentDir,
|
||||
});
|
||||
}
|
||||
if (completedSegments.size === 0) rmSync(segmentDir, { recursive: true, force: true });
|
||||
const invokeSegmented = () => {
|
||||
if (capturePlan.kind !== "sdr_segmented") {
|
||||
throw new Error(`Cannot invoke segmented stage with ${capturePlan.kind} plan`);
|
||||
|
||||
@@ -41,12 +41,13 @@ The Phase 0 mutation check is the same render with `PRODUCER_STREAMING_ENCODE_DU
|
||||
|
||||
### Interrupting the 2b run
|
||||
|
||||
Kill on the manifest, not on a log line — the manifest is the resume contract:
|
||||
Use a fresh local cache for this gate and keep it set for the resumed run:
|
||||
|
||||
```sh
|
||||
export HYPERFRAMES_CACHE_DIR="$(mktemp -d)"
|
||||
HF_SEGMENTED_CAPTURE=true HF_SEGMENT_FRAMES=1500 node <repo>/packages/cli/dist/cli.js \
|
||||
render a-single --fps 30 -w 1 --quality draft -o a-single/renders/resumed.mp4 & RPID=$!
|
||||
until [ "$(python3 -c "import json,glob;f=glob.glob('a-single/renders/.hf-segments/*/segments.json');print(len(json.load(open(f[0]))['completed']) if f else 0)")" -ge 2 ]; do sleep 1; done
|
||||
until [ "$(python3 -c "import json,glob,os;f=glob.glob(os.path.join(os.environ['HYPERFRAMES_CACHE_DIR'],'segments/*/*/segments.json'));print(len(json.load(open(f[0]))['completed']) if f else 0)")" -ge 2 ]; do sleep 1; done
|
||||
kill -9 $RPID
|
||||
```
|
||||
|
||||
@@ -143,14 +144,12 @@ have re-run it on a fresh screenshot session. Opting a specific composition into
|
||||
scratch and resume. Wiring the streaming path's failure classification into the
|
||||
segmented branch is the other follow-up before the R4 flip.
|
||||
|
||||
### Operational note: `--resume` state lives in the project directory
|
||||
### Local resume cache
|
||||
|
||||
Segments are written to `<project>/renders/.hf-segments/<planHash>/` rather than
|
||||
the ephemeral work directory, and they are deliberately left in place after a
|
||||
failed render because that is what `--resume` reads. A project that iterates on
|
||||
a long render grows by one segment directory per distinct plan hash until a run
|
||||
succeeds without `--keep-segments`, which removes its own hash directory. Stale
|
||||
hash directories from earlier settings are not swept.
|
||||
Each project path and render plan has its own directory under the local
|
||||
HyperFrames cache. Failed runs retain their checkpoints. A successful run
|
||||
removes its own directory unless you pass `--keep-segments`. Checkpoints from
|
||||
earlier settings stay in the cache until you remove them.
|
||||
|
||||
## PRINFRA-694 gate for the R3 flip
|
||||
|
||||
|
||||
Reference in New Issue
Block a user