diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 156cf18810..a872da1918 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2155,6 +2155,16 @@ mismatched encodes. The directory is deleted after a successful render unless `--keep-segments`, and deliberately kept after a failure, because that is what `--resume` reads next time. +`--resume` reuses only segments you finished on this machine. Their record is +signed with a key derived from your per-user preview token, a private file in +your home folder (`~/.local/state/hyperframes/auth`, or +`%LOCALAPPDATA%\hyperframes` on Windows), so a `.hf-segments` folder copied +from another machine or received inside a project is captured again. If that +token cannot be read or created, for example in a read-only home folder, +`--resume` warns and captures every segment again. Each reused segment must +also match the size and SHA-256 recorded when it finished. Segments recorded by +an older HyperFrames version are captured again once. + mp4 and mov only; png-sequence, gif, webm, HDR and shader-transition renders keep their existing routes. diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 4d9a755445..93c667f044 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -198,6 +198,7 @@ export { } from "./services/screenshotService.js"; // ── Encoding ─────────────────────────────────────────────────────────────────── +export { concatListInputArgs } from "@hyperframes/parsers/ff-binaries"; export { buildEncoderArgs, encodeFramesFromDir, diff --git a/packages/engine/src/services/chunkEncoder.relativePaths.test.ts b/packages/engine/src/services/chunkEncoder.relativePaths.test.ts index 70b1c961b3..db2d78c3c8 100644 --- a/packages/engine/src/services/chunkEncoder.relativePaths.test.ts +++ b/packages/engine/src/services/chunkEncoder.relativePaths.test.ts @@ -114,3 +114,94 @@ describe.skipIf(!HAS_FFMPEG)("concat input paths", () => { 30_000, ); }); + +describe.skipIf(!HAS_FFMPEG)("concat input formats", () => { + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "hf-concat-formats-")); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + function makeClip(path: string, frames: number): void { + const codec = path.endsWith(".webm") ? ["libvpx-vp9"] : ["libx264", "-bf", "0"]; + const made = spawnSync(getFfmpegBinary(), [ + ...["-v", "error", "-f", "lavfi", "-i", "color=red:size=32x32:rate=30"], + ...["-frames:v", String(frames), "-c:v", ...codec, "-y", path], + ]); + expect(made.status, made.stderr.toString()).toBe(0); + } + + function frameCount(path: string): string { + const probe = spawnSync( + getFfprobeBinary(), + [ + ...["-v", "error", "-count_frames", "-select_streams", "v:0"], + ...["-show_entries", "stream=nb_read_frames", "-of", "csv=p=0", "--", path], + ], + { encoding: "utf8" }, + ); + expect(probe.status, probe.stderr).toBe(0); + return probe.stdout.trim(); + } + + it.each(["mp4", "mov", "webm"])( + "joins %s files the encoders write", + async (ext) => { + const inputs = [join(root, `a.${ext}`), join(root, `b.${ext}`)]; + makeClip(inputs[0]!, 2); + makeClip(inputs[1]!, 3); + const outputPath = join(root, `joined.${ext}`); + expect(await concatVideoFiles(inputs, outputPath)).toEqual({ success: true }); + expect(frameCount(outputPath)).toBe("5"); + }, + 30_000, + ); + + function writePlaylist(): string { + makeClip(join(root, "elsewhere.mp4"), 4); + const playlist = join(root, "segment.m3u8"); + writeFileSync( + playlist, + "#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXTINF:1,\nelsewhere.mp4\n#EXT-X-ENDLIST\n", + ); + return playlist; + } + + it("refuses a listed file that is a playlist of other files", async () => { + const result = await concatVideoFiles([writePlaylist()], join(root, "joined.mp4")); + expect(result.success).toBe(false); + }, 30_000); + + it("follows a listed file that is itself an ffconcat list, but only to relative names", async () => { + // `concat` stays in the format whitelist because the list itself opens through it, so a nested + // list is followed. It opens with the default `-safe 1`, which refuses absolute paths. + const sub = join(root, "sub"); + mkdirSync(sub); + makeClip(join(root, "a.mp4"), 2); + makeClip(join(sub, "beside.mp4"), 3); + const nested = join(sub, "nested.mp4"); + const joined = join(root, "joined.mp4"); + + writeFileSync(nested, "ffconcat version 1.0\nfile 'beside.mp4'\n"); + expect(await concatVideoFiles([join(root, "a.mp4"), nested], joined)).toEqual({ + success: true, + }); + expect(frameCount(joined)).toBe("5"); + + writeFileSync(nested, `ffconcat version 1.0\nfile '${join(sub, "beside.mp4")}'\n`); + const absolute = await concatVideoFiles([join(root, "a.mp4"), nested], joined); + expect(absolute.success).toBe(false); + }, 30_000); + + it("fails the join when a refused file sits between good ones", async () => { + const good = [join(root, "a.mp4"), join(root, "b.mp4")]; + for (const path of good) makeClip(path, 2); + const inputs = [good[0]!, writePlaylist(), good[1]!]; + const result = await concatVideoFiles(inputs, join(root, "joined.mp4")); + expect(result.success).toBe(false); + }, 30_000); +}); diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.ts index 3fb64b0cfd..09eaf9b36f 100644 --- a/packages/engine/src/services/chunkEncoder.test.ts +++ b/packages/engine/src/services/chunkEncoder.test.ts @@ -1919,11 +1919,16 @@ describe("buildEncoderArgs HDR color space", () => { describe("buildConcatArgs", () => { it("stream-copies the concat list and writes provenance before the output", () => { const args = buildConcatArgs("/w/concat-list.txt", "/w/video-only.mp4"); - expect(args.slice(0, 8)).toEqual([ + expect(args.slice(0, 13)).toEqual([ + "-xerror", + "-protocol_whitelist", + "file", "-f", "concat", "-safe", "0", + "-format_whitelist", + "concat,mov,matroska", "-i", "/w/concat-list.txt", "-c", diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts index f38f718ee2..0a939ff4eb 100644 --- a/packages/engine/src/services/chunkEncoder.ts +++ b/packages/engine/src/services/chunkEncoder.ts @@ -50,6 +50,7 @@ import { appendVp9CpuUsedArg } from "./vp9Options.js"; import { appendSoftwareEncoderThreadsArg } from "./softwareEncoderThreads.js"; import { appendRenderProvenanceArgs } from "../utils/renderProvenance.js"; import { createStagingDir } from "@hyperframes/core/atomic-file"; +import { concatListInputArgs } from "@hyperframes/parsers/ff-binaries"; export type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js"; @@ -674,7 +675,7 @@ export async function encodeFramesFromDir( } export function buildConcatArgs(concatListPath: string, outputPath: string): string[] { - const args = ["-f", "concat", "-safe", "0", "-i", concatListPath, "-c", "copy"]; + const args = [...concatListInputArgs(concatListPath), "-c", "copy"]; // The concat demuxer does not carry per-input container metadata into the // output, so provenance is re-asserted on the concatenated file. appendRenderProvenanceArgs(args, outputPath); diff --git a/packages/parsers/src/ffBinaries.test.ts b/packages/parsers/src/ffBinaries.test.ts index 7ec28ec7e4..9817f91787 100644 --- a/packages/parsers/src/ffBinaries.test.ts +++ b/packages/parsers/src/ffBinaries.test.ts @@ -624,3 +624,20 @@ describe("assertLiteralMediaInput", () => { expect(() => mediaInputFormatArgs(ffmpeg, pattern)).toThrow(/pattern of other files/); }); }); + +describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)("concatListInputArgs", () => { + it("refuses a listed entry that is not a local file", async () => { + const { concatListInputArgs } = await importFresh(); + const dir = mkdtempSync(join(tmpdir(), "hf-concat-list-")); + onTestFinished(() => rmSync(dir, { recursive: true, force: true })); + const list = join(dir, "list.txt"); + writeFileSync(list, "file 'data:video/mp4;base64,AAAA'\n"); + const run = spawnSync( + "ffmpeg", + ["-v", "error", ...concatListInputArgs(list), "-c", "copy", "-y", join(dir, "out.mp4")], + { encoding: "utf8" }, + ); + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("not on whitelist 'file'"); + }); +}); diff --git a/packages/parsers/src/ffBinaries.ts b/packages/parsers/src/ffBinaries.ts index 3b1cd30762..9f2da0b89c 100644 --- a/packages/parsers/src/ffBinaries.ts +++ b/packages/parsers/src/ffBinaries.ts @@ -249,6 +249,26 @@ export function clearFfBinaryLookupCache(): void { skippedX86Binaries.clear(); } +/** + * Concat input for a list HyperFrames wrote: entries open only as local mov, matroska or concat + * files (an ffconcat entry is followed in safe mode), and `-xerror` fails at a refused entry. + */ +export function concatListInputArgs(concatListPath: string): string[] { + return [ + "-xerror", + "-protocol_whitelist", + "file", + "-f", + "concat", + "-safe", + "0", + "-format_whitelist", + "concat,mov,matroska", + "-i", + concatListPath, + ]; +} + /** What a render service lets a media input open: containers that name no other file. */ export const SELF_CONTAINED_FORMATS = "mov,matroska,mxf,avi,mpeg,mpegts,mpegvideo,flv,asf,ogg,wav,mp3,aac,flac,aiff,gif,image2,png_pipe,jpeg_pipe,webp_pipe,bmp_pipe,tiff_pipe"; diff --git a/packages/producer/src/services/distributed/assemble.test.ts b/packages/producer/src/services/distributed/assemble.test.ts index 4f654757f1..40ef14d112 100644 --- a/packages/producer/src/services/distributed/assemble.test.ts +++ b/packages/producer/src/services/distributed/assemble.test.ts @@ -935,6 +935,31 @@ describe("assemble()", () => { TIMEOUT_MS, ); + it( + "refuses a chunk that is a playlist of other files", + async () => { + if (!hasFfmpeg) return; + const chunks: ChunkSliceJson[] = [ + { index: 0, startFrame: 0, endFrame: 5 }, + { index: 1, startFrame: 5, endFrame: 10 }, + ]; + const planDir = buildPlanDir("mp4", chunks, 10, false); + makeMp4Chunk(join(planDir, "elsewhere.mp4"), 5); + const playlist = join(planDir, "chunk-0.m3u8"); + writeFileSync( + playlist, + "#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXTINF:1,\nelsewhere.mp4\n#EXT-X-ENDLIST\n", + ); + const chunkB = join(planDir, "chunk-1.mp4"); + makeMp4Chunk(chunkB, 5); + + await expect( + assemble(planDir, [playlist, chunkB], null, join(planDir, "output.mp4")), + ).rejects.toThrow(); + }, + TIMEOUT_MS, + ); + it( "stops ffmpeg at FFMPEG_PROCESS_TIMEOUT_MS and says it timed out", async () => { diff --git a/packages/producer/src/services/distributed/assemble.ts b/packages/producer/src/services/distributed/assemble.ts index 7a69af3133..4f700bc528 100644 --- a/packages/producer/src/services/distributed/assemble.ts +++ b/packages/producer/src/services/distributed/assemble.ts @@ -38,6 +38,7 @@ import { basename, dirname, join } from "node:path"; import { appendRateCapArgs, appendRenderProvenanceArgs, + concatListInputArgs, describeFfmpegFailure, getEncoderPreset, MIXED_AUDIO_FILENAME, @@ -396,7 +397,7 @@ function concatDemuxerInput( const listPath = join(workDir, "concat-list.txt"); const body = chunkPaths.map((path) => `file '${path.replace(/'/g, "'\\''")}'`).join("\n"); writeFileSync(listPath, `${body}\n`, "utf-8"); - return ["-r", fpsArg, "-f", "concat", "-safe", "0", "-i", listPath]; + return ["-r", fpsArg, ...concatListInputArgs(listPath)]; } /** What a probe of the joined file would report: the chunks' frames at the plan's rate. */ diff --git a/packages/producer/src/services/render/segmentManifest.test.ts b/packages/producer/src/services/render/segmentManifest.test.ts index a1dae608d0..e0341c9f0a 100644 --- a/packages/producer/src/services/render/segmentManifest.test.ts +++ b/packages/producer/src/services/render/segmentManifest.test.ts @@ -1,17 +1,23 @@ -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { previewToken } from "@hyperframes/core/preview-proof"; +import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { computeSegmentPlanHash, + hashSegmentFile, probeSegmentFrameCount, readSegmentManifest, segmentDirFor, + segmentManifestKey, validateCompletedSegments, writeSegmentManifest, type SegmentManifest, + type SegmentManifestEntry, type SegmentPlanHashInput, } from "./segmentManifest.js"; +import { planSegments, segmentOutputPath } from "./segmentPlan.js"; const hashInput: SegmentPlanHashInput = { compositionHash: "abc", @@ -106,11 +112,12 @@ describe("manifest io", () => { endFrame: 3, path: join(dir, "segment_00000.mp4"), bytes: 10, + sha256: "0".repeat(64), completedAt: "2026-09-17T00:00:00.000Z", }, ], }; - writeSegmentManifest(dir, manifest); + writeSegmentManifest(dir, manifest, null); expect(readSegmentManifest(dir)).toEqual(manifest); writeFileSync(join(dir, "segments.json"), "{not json"); expect(readSegmentManifest(dir)).toBeNull(); @@ -143,77 +150,81 @@ describe("validateCompletedSegments", () => { let dir = ""; afterEach(() => rmSync(dir, { recursive: true, force: true })); + const XX_SHA256 = createHash("sha256").update("xx").digest("hex"); + const entry = (index: number, bytes = 2): SegmentManifestEntry => ({ + index, + startFrame: index * 3, + endFrame: index * 3 + 3, + path: segmentOutputPath(dir, index), + bytes, + sha256: XX_SHA256, + completedAt: "", + }); + const KEY = Buffer.alloc(32, 1); + /** Written and read back the way a render does, so it carries a real signature. */ + const manifestOf = ( + completed: SegmentManifestEntry[], + options: { totalFrames?: number; planHash?: string; key?: Buffer | null } = {}, + ): SegmentManifest => { + const { totalFrames = completed.length * 3, planHash = "h", key = KEY } = options; + writeSegmentManifest( + dir, + { version: 1, planHash, totalFrames, segmentFrames: 3, completed }, + key, + ); + const written = readSegmentManifest(dir); + if (!written) throw new Error("manifest did not round-trip"); + return written; + }; + const check = ( + totalFrames: number, + probeFrames: (path: string) => Promise, + expectedHash = "h", + ) => ({ + expectedHash, + segmentDir: dir, + plan: planSegments(totalFrames, 3), + key: KEY, + probeFrames, + }); + const reuses = async (manifest: SegmentManifest, totalFrames: number) => + ( + await validateCompletedSegments( + manifest, + check(totalFrames, async () => 3), + ) + ).size; + it("keeps only segments whose file exists and whose frame count matches", async () => { dir = mkdtempSync(join(tmpdir(), "hf-seg-")); - mkdirSync(dir, { recursive: true }); - const ok = join(dir, "segment_00000.mp4"); - const short = join(dir, "segment_00001.mp4"); + const ok = segmentOutputPath(dir, 0); + const short = segmentOutputPath(dir, 1); writeFileSync(ok, "xx"); writeFileSync(short, "xx"); - const manifest: SegmentManifest = { - version: 1, - planHash: "h", - totalFrames: 9, - segmentFrames: 3, - completed: [ - { index: 0, startFrame: 0, endFrame: 3, path: ok, bytes: 2, completedAt: "" }, - { index: 1, startFrame: 3, endFrame: 6, path: short, bytes: 2, completedAt: "" }, - { - index: 2, - startFrame: 6, - endFrame: 9, - path: join(dir, "missing.mp4"), - bytes: 2, - completedAt: "", - }, - ], - }; + const manifest = manifestOf([entry(0), entry(1), entry(2)]); const probe = async (p: string) => (p === ok ? 3 : p === short ? 2 : null); - expect([...(await validateCompletedSegments(manifest, "h", probe))]).toEqual([0]); + expect([...(await validateCompletedSegments(manifest, check(9, probe)))]).toEqual([0]); }); it("rejects a segment whose size no longer matches the manifest", async () => { // A half-written file from a kill can still exist and still probe; the // recorded size is what proves it is the same file that was finished. dir = mkdtempSync(join(tmpdir(), "hf-seg-")); - const truncated = join(dir, "segment_00000.mp4"); - writeFileSync(truncated, "x"); - const manifest: SegmentManifest = { - version: 1, - planHash: "h", - totalFrames: 3, - segmentFrames: 3, - completed: [ - { index: 0, startFrame: 0, endFrame: 3, path: truncated, bytes: 999, completedAt: "" }, - ], - }; - expect((await validateCompletedSegments(manifest, "h", async () => 3)).size).toBe(0); + writeFileSync(segmentOutputPath(dir, 0), "x"); + const manifest = manifestOf([entry(0, 999)]); + expect(await reuses(manifest, 3)).toBe(0); }); it("a resume whose segment files were removed after a join captures them again rather than failing", async () => { dir = mkdtempSync(join(tmpdir(), "hf-seg-")); - const joined = [0, 1].map((index) => join(dir, `segment_0000${index}.mp4`)); - const manifest: SegmentManifest = { - version: 1, - planHash: "h", - totalFrames: 6, - segmentFrames: 3, - completed: joined.map((path, index) => ({ - index, - startFrame: index * 3, - endFrame: index * 3 + 3, - path, - bytes: 2, - completedAt: "", - })), - }; // The manifest still lists them, but the join deleted the files. + const manifest = manifestOf([entry(0), entry(1)]); let probed = 0; const probe = async () => { probed += 1; return 3; }; - expect((await validateCompletedSegments(manifest, "h", probe)).size).toBe(0); + expect((await validateCompletedSegments(manifest, check(6, probe))).size).toBe(0); expect(probed).toBe(0); }); @@ -222,17 +233,111 @@ describe("validateCompletedSegments", () => { // frame count — so only the hash check can reject it. With an empty // `completed` list this assertion would pass with the check deleted. dir = mkdtempSync(join(tmpdir(), "hf-seg-")); - const good = join(dir, "segment_00000.mp4"); - writeFileSync(good, "xx"); - const manifest: SegmentManifest = { - version: 1, - planHash: "old", - totalFrames: 3, - segmentFrames: 3, - completed: [{ index: 0, startFrame: 0, endFrame: 3, path: good, bytes: 2, completedAt: "" }], + writeFileSync(segmentOutputPath(dir, 0), "xx"); + const manifest = manifestOf([entry(0)], { planHash: "old" }); + const probe = async () => 3; + expect((await validateCompletedSegments(manifest, check(3, probe, "old"))).size).toBe(1); + expect((await validateCompletedSegments(manifest, check(3, probe, "new"))).size).toBe(0); + }); + + it("never opens the paths a manifest names, only the segment files the join reads", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + // A file that would pass every check if the manifest's own path were trusted. + const elsewhere = join(dir, "elsewhere.mp4"); + writeFileSync(elsewhere, "xx"); + const named = [elsewhere, "/etc/hosts", "\\\\host\\share\\segment.mp4"]; + const manifest = manifestOf(named.map((path, index) => ({ ...entry(index), path }))); + const probed: string[] = []; + const probe = async (path: string) => { + probed.push(path); + return 3; }; - expect((await validateCompletedSegments(manifest, "old", async () => 3)).size).toBe(1); - expect((await validateCompletedSegments(manifest, "new", async () => 3)).size).toBe(0); + expect((await validateCompletedSegments(manifest, check(9, probe))).size).toBe(0); + expect(probed).toEqual([]); + }); + + it("re-captures a segment file that is a link, even to a finished segment", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + const target = join(dir, "real.mp4"); + writeFileSync(target, "xx"); + symlinkSync(target, segmentOutputPath(dir, 0)); + const manifest = manifestOf([entry(0)]); + expect(await reuses(manifest, 3)).toBe(0); + }); + + it("ignores entries for segments this plan does not have", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + for (const index of [0, 1, 5]) writeFileSync(segmentOutputPath(dir, index), "xx"); + const shifted = { ...entry(1), startFrame: 2, endFrame: 5 }; + const manifest = manifestOf([entry(0), shifted, entry(5), { ...entry(0), index: 0.5 }], { + totalFrames: 6, + }); + const probe = async () => 3; + expect([...(await validateCompletedSegments(manifest, check(6, probe)))]).toEqual([0]); + }); + it("reuses segments only from a manifest this user signed", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + writeFileSync(segmentOutputPath(dir, 0), "xx"); + expect(await reuses(manifestOf([entry(0)]), 3)).toBe(1); + // Copied from another user or machine, or written by hand without a signature. + expect(await reuses(manifestOf([entry(0)], { key: Buffer.alloc(32, 2) }), 3)).toBe(0); + expect(await reuses(manifestOf([entry(0)], { key: null }), 3)).toBe(0); + }); + + it("rejects a signed manifest whose entries were edited afterwards", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + writeFileSync(segmentOutputPath(dir, 0), "xx"); + const signed = manifestOf([entry(0, 3)]); + expect(await reuses({ ...signed, completed: [entry(0, 2)] }, 3)).toBe(0); + }); + + it("re-captures a segment replaced by a different file of the same size, without probing it", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + writeFileSync(segmentOutputPath(dir, 0), "xx"); + const manifest = manifestOf([entry(0)]); + expect(await reuses(manifest, 3)).toBe(1); + writeFileSync(segmentOutputPath(dir, 0), "yy"); + let probed = 0; + const probe = async () => { + probed += 1; + return 3; + }; + expect((await validateCompletedSegments(manifest, check(3, probe))).size).toBe(0); + expect(probed).toBe(0); + }); + + it("hashes a segment file's contents", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + writeFileSync(segmentOutputPath(dir, 0), "xx"); + expect(await hashSegmentFile(segmentOutputPath(dir, 0))).toBe(XX_SHA256); + }); + + it("reuses nothing when this user has no key", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + writeFileSync(segmentOutputPath(dir, 0), "xx"); + const noKey = { ...check(3, async () => 3), key: null }; + expect((await validateCompletedSegments(manifestOf([entry(0)]), noKey)).size).toBe(0); + }); +}); + +describe("segmentManifestKey", () => { + it("derives one stable key from this user's preview token, distinct from the token", () => { + const home = mkdtempSync(join(tmpdir(), "hf-seg-home-")); + onTestFinished(() => rmSync(home, { recursive: true, force: true })); + for (const name of ["HOME", "USERPROFILE", "LOCALAPPDATA"]) vi.stubEnv(name, home); + onTestFinished(() => vi.unstubAllEnvs()); + + const key = segmentManifestKey(); + const token = previewToken(); + expect(token).not.toBeNull(); + expect(key?.length).toBe(32); + expect(segmentManifestKey()).toEqual(key); + expect(key?.toString("hex")).not.toBe(token); + expect(segmentManifestKey("ab".repeat(32))).not.toEqual(key); + }); + + it("is null without a preview token", () => { + expect(segmentManifestKey(null)).toBeNull(); }); }); diff --git a/packages/producer/src/services/render/segmentManifest.ts b/packages/producer/src/services/render/segmentManifest.ts index 862d7cca89..f37d522474 100644 --- a/packages/producer/src/services/render/segmentManifest.ts +++ b/packages/producer/src/services/render/segmentManifest.ts @@ -6,10 +6,19 @@ * guard that: the plan hash covers everything that determines segment bytes, * and every reused segment is re-validated on disk before it is skipped. */ -import { createHash } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { + createReadStream, + existsSync, + lstatSync, + mkdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; import { join } from "node:path"; +import { previewToken } from "@hyperframes/core/preview-proof"; import { renameWithRetry } from "@hyperframes/engine"; +import { segmentOutputPath, type SegmentSlice } from "./segmentPlan.js"; export interface SegmentManifestEntry { index: number; @@ -17,6 +26,8 @@ export interface SegmentManifestEntry { endFrame: number; path: string; bytes: number; + /** Hex SHA-256 of the segment file, taken when it finished. */ + sha256: string; completedAt: string; } @@ -26,6 +37,8 @@ export interface SegmentManifest { totalFrames: number; segmentFrames: number; completed: SegmentManifestEntry[]; + /** HMAC by this user's manifest key; resume trusts no manifest without a valid one. */ + mac?: string; } export interface SegmentPlanHashInput { @@ -104,6 +117,7 @@ function isEntry(value: unknown): value is SegmentManifestEntry { hasNumber(value, "endFrame") && hasString(value, "path") && hasNumber(value, "bytes") && + hasString(value, "sha256") && hasString(value, "completedAt") ); } @@ -134,11 +148,52 @@ export function readSegmentManifest(segmentDir: string): SegmentManifest | null } } +/** + * The key that signs segment manifests, derived from this user's preview token, which lives in the + * user's home outside every project. Null when no token can be read or made; resume then reuses nothing. + */ +export function segmentManifestKey(token: string | null = previewToken()): Buffer | null { + return token ? createHmac("sha256", token).update("segment-manifest").digest() : null; +} + +function manifestMac(manifest: SegmentManifest, key: Buffer): string { + const signed = [ + manifest.planHash, + manifest.totalFrames, + manifest.segmentFrames, + manifest.completed.map((e) => [e.index, e.startFrame, e.endFrame, e.bytes, e.sha256]), + ]; + return createHmac("sha256", key).update(JSON.stringify(signed)).digest("hex"); +} + +function signSegmentManifest(manifest: SegmentManifest, key: Buffer | null): SegmentManifest { + const { version, planHash, totalFrames, segmentFrames, completed } = manifest; + const unsigned: SegmentManifest = { version, planHash, totalFrames, segmentFrames, completed }; + return key ? { ...unsigned, mac: manifestMac(unsigned, key) } : unsigned; +} + +function hasValidMac(manifest: SegmentManifest, key: Buffer | null): boolean { + if (!key || typeof manifest.mac !== "string") return false; + const expected = Buffer.from(manifestMac(manifest, key), "hex"); + const actual = Buffer.from(manifest.mac, "hex"); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} + +export async function hashSegmentFile(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +} + /** Atomic: write to a temp name, then rename over the manifest. */ -export function writeSegmentManifest(segmentDir: string, manifest: SegmentManifest): void { +export function writeSegmentManifest( + segmentDir: string, + manifest: SegmentManifest, + key: Buffer | null, +): void { mkdirSync(segmentDir, { recursive: true }); const tmp = join(segmentDir, `${MANIFEST_FILENAME}.${process.pid}.tmp`); - writeFileSync(tmp, JSON.stringify(manifest, null, 2), "utf-8"); + writeFileSync(tmp, JSON.stringify(signSegmentManifest(manifest, key), null, 2), "utf-8"); renameWithRetry(tmp, join(segmentDir, MANIFEST_FILENAME)); } @@ -166,26 +221,56 @@ export async function probeSegmentFrameCount( } } +export interface SegmentResumeCheck { + expectedHash: string; + segmentDir: string; + /** This render's segments; manifest entries are trusted only where they match one. */ + plan: readonly SegmentSlice[]; + key: Buffer | null; + probeFrames: (path: string) => Promise; +} + +function plannedSlice( + entry: SegmentManifestEntry, + plan: readonly SegmentSlice[], +): SegmentSlice | undefined { + const slice = Number.isInteger(entry.index) ? plan[entry.index] : undefined; + if (!slice || slice.startFrame !== entry.startFrame || slice.endFrame !== entry.endFrame) { + return undefined; + } + return slice; +} + +function isPlainFileOfSize(path: string, bytes: number): boolean { + const stat = lstatSync(path, { throwIfNoEntry: false }); + return stat !== undefined && stat.isFile() && stat.size > 0 && stat.size === bytes; +} + /** - * Indices safe to skip on resume. A segment counts only if the hash matches, - * the file exists at the recorded size, and its frame count equals the slice - * length. Anything else re-captures — the cost of re-capturing a good segment - * is minutes; the cost of reusing a bad one is a silently wrong video. + * Indices safe to skip on resume. A segment counts only if this user's key + * signed the manifest, the plan hash matches, the entry matches a slice of + * this plan, and the file the join will read (the slice's own path in + * `segmentDir`, never the manifest's `path`) is a plain file at the recorded + * size and hash whose frame count equals the slice length. The hash is + * checked before the prober opens the file. Anything else re-captures: + * re-capturing a good segment costs minutes, reusing a bad one silently + * produces a wrong video. */ export async function validateCompletedSegments( manifest: SegmentManifest, - expectedHash: string, - probeFrames: (path: string) => Promise, + check: SegmentResumeCheck, ): Promise> { const ok = new Set(); - if (manifest.planHash !== expectedHash) return ok; + if (manifest.planHash !== check.expectedHash || !hasValidMac(manifest, check.key)) return ok; for (const entry of manifest.completed) { - if (!existsSync(entry.path)) continue; - const size = statSync(entry.path).size; - if (size <= 0 || size !== entry.bytes) continue; - const frames = await probeFrames(entry.path); - if (frames !== entry.endFrame - entry.startFrame) continue; - ok.add(entry.index); + const slice = plannedSlice(entry, check.plan); + if (!slice) continue; + const path = segmentOutputPath(check.segmentDir, slice.index); + if (!isPlainFileOfSize(path, entry.bytes)) continue; + if ((await hashSegmentFile(path)) !== entry.sha256) continue; + const frames = await check.probeFrames(path); + if (frames !== slice.endFrame - slice.startFrame) continue; + ok.add(slice.index); } return ok; } diff --git a/packages/producer/src/services/render/segmentPlan.ts b/packages/producer/src/services/render/segmentPlan.ts index 15f6b10a67..9e057af040 100644 --- a/packages/producer/src/services/render/segmentPlan.ts +++ b/packages/producer/src/services/render/segmentPlan.ts @@ -1,8 +1,9 @@ /** - * Segment planning for long-form capture (spec §5 Phase 2). A segment is a - * half-open frame range captured into its own closed-GOP encoder process so - * a crash costs one segment, not the render, and scratch is encoded video. + * Segment planning for long-form capture (spec §5 Phase 2). Each segment is a half-open frame + * range with its own closed-GOP encoder, so a crash costs one segment and scratch is encoded video. */ +import { join } from "node:path"; + export interface SegmentSlice { index: number; startFrame: number; @@ -35,6 +36,11 @@ export function planSegments(totalFrames: number, segmentFrames: number): Segmen return slices; } +/** Zero-padded so lexical order equals frame order in the concat list and on disk. */ +export function segmentOutputPath(segmentDir: string, index: number): string { + return join(segmentDir, `segment_${String(index).padStart(5, "0")}.mp4`); +} + export function resolveSegmentFrames(env: NodeJS.ProcessEnv): number { const raw = env.HF_SEGMENT_FRAMES; if (raw === undefined || raw.trim() === "") return DEFAULT_SEGMENT_FRAMES; diff --git a/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts b/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts index 0778171504..bd81a1adeb 100644 --- a/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts +++ b/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts @@ -8,7 +8,8 @@ const fixtureRoot = mkdtempSync(join(tmpdir(), "hf-segmented-test-")); const framesDir = join(fixtureRoot, "frames"); afterAll(() => rmSync(fixtureRoot, { recursive: true, force: true })); import { createCapturePlan } from "../capturePlan.js"; -import { runCaptureSegmentedStage, segmentOutputPath } from "./captureSegmentedStage.js"; +import { segmentOutputPath } from "../segmentPlan.js"; +import { runCaptureSegmentedStage } from "./captureSegmentedStage.js"; const writeFrame = mock((_buffer: Buffer) => true); const closeEncoder = mock(async () => ({ success: true, durationMs: 123, fileSize: 42 })); diff --git a/packages/producer/src/services/render/stages/captureSegmentedStage.ts b/packages/producer/src/services/render/stages/captureSegmentedStage.ts index 153f96a03a..bcdb05484b 100644 --- a/packages/producer/src/services/render/stages/captureSegmentedStage.ts +++ b/packages/producer/src/services/render/stages/captureSegmentedStage.ts @@ -42,9 +42,10 @@ import { ensureFrameWritten } from "./captureHdrFrameShared.js"; import { reportFrameProgress } from "../shared.js"; import { encoderFailureError } from "../encoderInterruption.js"; import type { SdrSegmentedCapturePlan } from "../capturePlan.js"; -import { planSegments, type SegmentSlice } from "../segmentPlan.js"; +import { planSegments, segmentOutputPath, type SegmentSlice } from "../segmentPlan.js"; import type { SegmentVideoFrames, SegmentWorkerFrames } from "../segmentVideoFrames.js"; import { createSegmentQueue } from "../segmentQueue.js"; +import { hashSegmentFile } from "../segmentManifest.js"; import { isTargetLossError } from "../segmentRecycle.js"; import { OPENING_CAPTURE_SESSION, @@ -104,6 +105,7 @@ export interface CaptureSegmentedStageInput { endFrame: number; path: string; bytes: number; + sha256: string; }) => void; /** Parallel capture sessions pulling from one segment queue. Default 1. */ workerCount?: number; @@ -146,11 +148,6 @@ export type CaptureSegmentedStageResult = /** The first segment's encoder could not spawn — caller replans to sdr_streaming. */ | { success: false }; -/** Zero-padded so lexical order equals frame order in the concat list and on disk. */ -export function segmentOutputPath(segmentDir: string, index: number): string { - return join(segmentDir, `segment_${String(index).padStart(5, "0")}.mp4`); -} - /** * One closed-GOP encoder per segment. `gopSize` is the segment's own frame * count (or the keyframe interval if shorter) so concat-copy has an IDR at every @@ -302,6 +299,7 @@ async function captureSegmentToEncoder( // Recorded so resume can prove the file on disk is the one that // finished; absent only in unit tests, whose fake encoder writes none. bytes: existsSync(segmentPath) ? statSync(segmentPath).size : 0, + sha256: existsSync(segmentPath) ? await hashSegmentFile(segmentPath) : "", }); return encodeResult; } finally { diff --git a/packages/producer/src/services/renderOrchestrator.segmentedDeRetry.test.ts b/packages/producer/src/services/renderOrchestrator.segmentedDeRetry.test.ts index a71d37f50d..dd01426bfe 100644 --- a/packages/producer/src/services/renderOrchestrator.segmentedDeRetry.test.ts +++ b/packages/producer/src/services/renderOrchestrator.segmentedDeRetry.test.ts @@ -23,6 +23,8 @@ const { state, RETRY_REACHED } = vi.hoisted(() => ({ }>, failures: [] as unknown[], resumable: false, + resumeKey: null as Buffer | null, + warnings: [] as string[], sessions: [] as object[], closed: [] as object[], }, @@ -117,6 +119,7 @@ vi.mock("./render/segmentManifest.js", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, + segmentManifestKey: () => state.resumeKey, readSegmentManifest: (dir: string) => state.resumable ? { version: 1, planHash: "", totalFrames: 1200, segmentFrames: 600, completed: [] } @@ -164,6 +167,8 @@ beforeEach(() => { state.sessions = []; state.closed = []; state.resumable = false; + state.resumeKey = null; + state.warnings = []; vi.stubEnv("HF_SEGMENTED_CAPTURE", "true"); vi.stubEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE", ""); vi.stubEnv("HF_DE_PARALLEL_STREAM", ""); @@ -187,7 +192,14 @@ function render(resumeSegments = false): Promise { hdrMode: "force-sdr", resumeSegments, producerConfig: resolveConfig({ useDrawElement: true, browserGpuMode: "hardware" }), - logger: { info() {}, warn() {}, error() {}, debug() {} }, + logger: { + info() {}, + warn(message: string) { + state.warnings.push(message); + }, + error() {}, + debug() {}, + }, }); return executeRenderJob(job, root, join(root, "output.mp4")); } @@ -227,6 +239,22 @@ it("re-captures every segment, including resumed ones, after a drawElement failu expect(state.attempts[1]?.segmentDirEntries).toEqual([]); }); +it("warns once under --resume when no manifest key is available", async () => { + const resumeWarnings = () => state.warnings.filter((w) => w.includes("captured again")); + for (const [resume, key, expected] of [ + [true, null, 1], + [true, Buffer.alloc(32, 1), 0], + [false, null, 0], + ] as const) { + state.attempts = []; + state.warnings = []; + state.resumeKey = key; + state.failures = [new DrawElementCaptureError(640, "suspect small frame")]; + await expect(render(resume)).rejects.toThrow(RETRY_REACHED); + expect(resumeWarnings()).toHaveLength(expected); + } +}); + it("does not retry a second drawElement failure", async () => { state.failures = [ new DrawElementCaptureError(640, "suspect small frame"), diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 4a3e911ceb..0d5802f580 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -195,7 +195,7 @@ import { startRenderEtaGuard, } from "./render/renderEtaGuard.js"; import { checkPageNeverReady } from "./render/pageNeverReady.js"; -import { resolveSegmentFrames } from "./render/segmentPlan.js"; +import { planSegments, resolveSegmentFrames } from "./render/segmentPlan.js"; import { extractAllDeferredFrames } from "./render/deferredVideoFrames.js"; import { measureLocalShownBoxes, oversizedVideos } from "./render/videoShownBoxes.js"; import { createSegmentVideoFrames } from "./render/segmentVideoFrames.js"; @@ -206,6 +206,7 @@ import { probeSegmentFrameCount, readSegmentManifest, segmentDirFor, + segmentManifestKey, validateCompletedSegments, writeSegmentManifest, type SegmentManifest, @@ -3133,6 +3134,16 @@ export async function executeRenderJob( } } +function loadResumeKeyOrWarn(resume: boolean, log: ProducerLogger): Buffer | null { + const key = segmentManifestKey(); + if (resume && !key) { + log.warn( + "[Render] --resume: no preview token could be read or created, so every segment will be captured again", + ); + } + return key; +} + async function executeRenderPipeline(input: { job: RenderJob; projectDir: string; @@ -5101,13 +5112,18 @@ async function executeRenderPipeline(input: { captureQuality: captureOptions.quality, }); const segmentDir = segmentDirFor(join(projectDir, "renders"), segmentPlanHash); + const manifestKey = loadResumeKeyOrWarn(job.config.resumeSegments === true, log); let completedSegments: ReadonlySet = new Set(); if (job.config.resumeSegments === true) { const existing = readSegmentManifest(segmentDir); if (existing) { - completedSegments = await validateCompletedSegments(existing, segmentPlanHash, (path) => - probeSegmentFrameCount(path, extractMediaMetadata), - ); + completedSegments = await validateCompletedSegments(existing, { + expectedHash: segmentPlanHash, + segmentDir, + plan: planSegments(totalFrames, segmentFrames), + key: manifestKey, + probeFrames: (path) => probeSegmentFrameCount(path, extractMediaMetadata), + }); log.info(`[Render] resuming: ${completedSegments.size} segments complete`, { segmentDir, }); @@ -5184,7 +5200,7 @@ async function executeRenderPipeline(input: { ...segmentManifest.completed.filter((e) => e.index !== entry.index), { ...entry, completedAt: new Date().toISOString() }, ]; - writeSegmentManifest(segmentDir, segmentManifest); + writeSegmentManifest(segmentDir, segmentManifest, manifestKey); }, updateCaptureObservability, }), diff --git a/packages/studio-server/src/helpers/proxyCache.test.ts b/packages/studio-server/src/helpers/proxyCache.test.ts index e42d3cdf02..2845843570 100644 --- a/packages/studio-server/src/helpers/proxyCache.test.ts +++ b/packages/studio-server/src/helpers/proxyCache.test.ts @@ -172,11 +172,14 @@ describe("cleanupProxyCache", () => { const hour = 60 * 60 * 1000; const key = basename(proxy(cache, "long"), ".mp4"); const runningKey = basename(proxy(cache, "running"), ".mp4"); - const stalePiece = join(cache, `.part-${key}-0.mp4`); - const stalePieceTemp = join(cache, `.tmp-${randomUUID()}-.part-${key}-1.mp4`); - const freshPiece = join(cache, `.part-${key}-2.mp4`); - const runningPiece = join(cache, `.part-${runningKey}-0.mp4`); + const tag = "0123456789abcdef".repeat(2); + const stalePiece = join(cache, `.part-${key}-${tag}-0.mp4`); + const stalePieceTemp = join(cache, `.tmp-${randomUUID()}-.part-${key}-${tag}-1.mp4`); + const untaggedPiece = join(cache, `.part-${key}-3.mp4`); + const freshPiece = join(cache, `.part-${key}-${tag}-2.mp4`); + const runningPiece = join(cache, `.part-${runningKey}-${tag}-0.mp4`); writeEntry(stalePiece, 3, now - 2 * hour); + writeEntry(untaggedPiece, 3, now - 2 * hour); writeEntry(stalePieceTemp, 3, now - 2 * hour); writeEntry(freshPiece, 3, now - 11 * 60 * 1000); writeEntry(runningPiece, 3, now - 2 * hour); @@ -188,11 +191,36 @@ describe("cleanupProxyCache", () => { protectedPaths: new Set([join(cache, `${runningKey}.mp4`)]), }); - expect(result.removed.sort()).toEqual([stalePiece, stalePieceTemp].sort()); + expect(result.removed.sort()).toEqual([stalePiece, stalePieceTemp, untaggedPiece].sort()); expect(existsSync(freshPiece)).toBe(true); expect(existsSync(runningPiece)).toBe(true); }); + it("sweeps old-style pieces even while that copy runs, and pieces dated in the future", () => { + const cache = cacheDir(); + const now = 1_800_000_000_000; + const hour = 60 * 60 * 1000; + const runningKey = basename(proxy(cache, "running"), ".mp4"); + const stoppedKey = basename(proxy(cache, "stopped"), ".mp4"); + const tag = "0123456789abcdef".repeat(2); + const untagged = join(cache, `.part-${runningKey}-0.mp4`); + const ownPiece = join(cache, `.part-${runningKey}-${tag}-1.mp4`); + const futureDated = join(cache, `.part-${stoppedKey}-${tag}-0.mp4`); + writeEntry(untagged, 3, now - 2 * hour); + writeEntry(ownPiece, 3, now - 2 * hour); + writeEntry(futureDated, 3, now + 365 * 24 * hour); + + const result = cleanupProxyCache(cache, { + now, + staleTempMs: 10 * 60 * 1000, + minSweepIntervalMs: 0, + protectedPaths: new Set([join(cache, `${runningKey}.mp4`)]), + }); + + expect(result.removed.sort()).toEqual([futureDated, untagged].sort()); + expect(existsSync(ownPiece)).toBe(true); + }); + it("never removes a file the transcoder did not name, even from an idle cache over budget", () => { const cache = cacheDir(); const now = 1_800_000_000_000; diff --git a/packages/studio-server/src/helpers/proxyCache.ts b/packages/studio-server/src/helpers/proxyCache.ts index 6adaede311..c4cb686c69 100644 --- a/packages/studio-server/src/helpers/proxyCache.ts +++ b/packages/studio-server/src/helpers/proxyCache.ts @@ -11,8 +11,9 @@ const PROXY_EXTENSIONS: ReadonlySet = new Set([ PROXY_STILL_EXTENSION, ]); const TRANSCODER_PROXY_NAME = /^[0-9a-f]{64}\.\w+$/; -const TRANSCODER_TEMP_NAME = /^\.tmp-[0-9a-f-]{36}-(?:\.part-[0-9a-f]{64}-\d+|[0-9a-f]{64})\.\w+$/; -const TRANSCODER_PIECE_NAME = /^\.part-([0-9a-f]{64})-\d+(\.\w+)$/; +const TRANSCODER_TEMP_NAME = + /^\.tmp-[0-9a-f-]{36}-(?:\.part-[0-9a-f]{64}-(?:[0-9a-f]{32}-)?\d+|[0-9a-f]{64})\.\w+$/; +const TRANSCODER_PIECE_NAME = /^\.part-([0-9a-f]{64})-([0-9a-f]{32}-)?\d+(\.\w+)$/; const STALE_PIECE_MS = 60 * 60 * 1000; export interface ProxyCacheCleanupOptions { @@ -81,8 +82,11 @@ function leftoverIsDead( ): boolean { const piece = TRANSCODER_PIECE_NAME.exec(name); if (!piece) return now - modifiedAt >= staleTempMs; - const copyIsRunning = protectedPaths.has(join(cacheDir, `${piece[1]}${piece[2]}`)); - return !copyIsRunning && now - modifiedAt >= STALE_PIECE_MS; + const taggedForThisVersion = piece[2] !== undefined; + const copyIsRunning = + taggedForThisVersion && protectedPaths.has(join(cacheDir, `${piece[1]}${piece[3]}`)); + const ageEvenIfFutureDated = Math.abs(now - modifiedAt); + return !copyIsRunning && ageEvenIfFutureDated >= STALE_PIECE_MS; } function readCacheInventory( diff --git a/packages/studio-server/src/helpers/proxyLock.test.ts b/packages/studio-server/src/helpers/proxyLock.test.ts index 4915c958bd..436bf15eda 100644 --- a/packages/studio-server/src/helpers/proxyLock.test.ts +++ b/packages/studio-server/src/helpers/proxyLock.test.ts @@ -27,12 +27,20 @@ const within = (work: Promise, ms: number) => ]); describe("acquireCopyLock", () => { + const TAG = "0123456789abcdef".repeat(2); + it("keeps waiting while the owner writes a piece of a long source", async () => { const cachePath = staleLock("c.mp4"); - writeFileSync(join(dir, ".tmp-1234-.part-c-0.mp4"), "x"); + writeFileSync(join(dir, `.tmp-1234-.part-c-${TAG}-0.mp4`), "x"); const abort = new AbortController(); - const waiting = acquireCopyLock(cachePath, () => false, abort.signal, 60_000); + const waiting = acquireCopyLock( + cachePath, + () => false, + abort.signal, + 60_000, + `.part-c-${TAG}-`, + ); setTimeout(() => abort.abort(new Error("stopped")), 400); await expect(within(waiting, 3_000)).rejects.toThrow("stopped"); @@ -52,6 +60,25 @@ describe("acquireCopyLock", () => { ).rejects.toThrow(/a\.mp4.*no progress/); }); + it("gives up on a stalled owner even when a project shipped pieces dated in the future", async () => { + const cachePath = staleLock("d.mp4"); + const nextYear = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); + for (const name of [".part-d-0.mp4", `.part-d-${"f".repeat(32)}-0.mp4`]) { + writeFileSync(join(dir, name), "x"); + utimesSync(join(dir, name), nextYear, nextYear); + } + + const waiting = acquireCopyLock( + cachePath, + () => false, + new AbortController().signal, + 300, + `.part-d-${TAG}-`, + ); + + await expect(within(waiting, 3_000)).rejects.toThrow(CopyLockStalledError); + }); + it("keeps waiting while the owner's temp file is still changing", async () => { const cachePath = staleLock("b.mp4"); writeFileSync(join(dir, ".tmp-1234-b.mp4"), "x"); diff --git a/packages/studio-server/src/helpers/proxyLock.ts b/packages/studio-server/src/helpers/proxyLock.ts index cb86013c01..02238b16b6 100644 --- a/packages/studio-server/src/helpers/proxyLock.ts +++ b/packages/studio-server/src/helpers/proxyLock.ts @@ -7,7 +7,7 @@ import { statSync, writeSync, } from "node:fs"; -import { basename, dirname, extname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; const POLL_MS = 500; const UNREADABLE_LOCK_MS = 10_000; @@ -25,13 +25,20 @@ export class CopyLockStalledError extends Error { } } -function lastActivityMs(cachePath: string, lockPath: string): number { +function lastActivityMs( + cachePath: string, + lockPath: string, + unguessablePieceName: string | undefined, +): number { const name = basename(cachePath); - const stem = basename(cachePath, extname(cachePath)); + const isPiece = (file: string) => + unguessablePieceName !== undefined && + (file.startsWith(unguessablePieceName) || + (file.startsWith(".tmp-") && file.includes(`-${unguessablePieceName}`))); const mine = (file: string) => file === basename(lockPath) || - (file.startsWith(".tmp-") && (file.endsWith(`-${name}`) || file.includes(`-.part-${stem}-`))) || - file.startsWith(`.part-${stem}-`); + (file.startsWith(".tmp-") && file.endsWith(`-${name}`)) || + isPiece(file); let newest = 0; for (const file of readdirSync(dirname(cachePath))) { if (!mine(file)) continue; @@ -78,6 +85,7 @@ export async function acquireCopyLock( copyMade: (path: string) => boolean, signal: AbortSignal, stallMs: number, + unguessablePieceName?: string, ): Promise<(() => void) | null> { const lockPath = lockPathFor(cachePath); for (;;) { @@ -94,7 +102,7 @@ export async function acquireCopyLock( if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; } if (!ownerIsAlive(lockPath)) rmSync(lockPath, { force: true }); - else if (Date.now() - lastActivityMs(cachePath, lockPath) > stallMs) + else if (Date.now() - lastActivityMs(cachePath, lockPath, unguessablePieceName) > stallMs) throw new CopyLockStalledError(basename(cachePath), stallMs); else await pause(POLL_MS, signal); } diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.ts index 8885ced69d..cd13eacd41 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts @@ -164,8 +164,10 @@ async function loadModule( codecName?: string, durationSeconds?: number, streams?: string[], + token: string | null = "a".repeat(64), ): Promise { vi.resetModules(); + vi.doMock("@hyperframes/core/preview-proof", () => ({ previewToken: () => token })); vi.doMock("node:child_process", () => { const mocked = { spawn }; return { ...mocked, default: mocked }; @@ -245,6 +247,157 @@ describe("resolveProxy on a long source", () => { expect(copiesIn(cacheDirectory)).toEqual([cachePath.split("/").at(-1)]); }); + const answerAll = async (calls: SpawnCall[]): Promise => { + for (let answered = 0; ; answered = calls.length) { + await flush(50); + if (calls.length === answered) break; + calls.slice(answered).forEach((call) => succeed(call)); + } + }; + + it("joins pieces through the shared concat input, which allows only local mp4, mov or webm files", async () => { + const { spawn, calls } = createSpawnSpy(); + const { resolveProxy } = await loadModule( + spawn, + FFMPEG_PATH, + false, + null, + undefined, + LONG_SECONDS, + ); + const projectDir = tmpProject(); + const sourcePath = join(projectDir, "long.mov"); + writeFileSync(sourcePath, "source-bytes"); + const result = resolveProxy(projectDir, sourcePath); + await answerAll(calls); + await result; + const joinArgs = calls.at(-1)!.args; + expect(joinArgs.slice(1, 14)).toEqual([ + "-xerror", + "-protocol_whitelist", + "file", + "-f", + "concat", + "-safe", + "0", + "-format_whitelist", + "concat,mov,matroska", + "-i", + expect.stringMatching(/pieces\.txt$/), + "-c", + "copy", + ]); + }); + + it("encodes every piece again when the project shipped pieces at the names it can predict", async () => { + const { spawn, calls } = createSpawnSpy(); + const { resolveProxy, getProxyCachePath } = await loadModule( + spawn, + FFMPEG_PATH, + false, + null, + undefined, + LONG_SECONDS, + ); + const projectDir = tmpProject(); + const sourcePath = join(projectDir, "long.mov"); + writeFileSync(sourcePath, "source-bytes"); + const stem = basename(getProxyCachePath(projectDir, sourcePath), ".mp4"); + const cacheDirectory = join(projectDir, ".transcode-cache"); + mkdirSync(cacheDirectory, { recursive: true }); + for (const index of [0, 1, 2]) { + writeFileSync(join(cacheDirectory, `.part-${stem}-${index}.mp4`), "shipped-playlist"); + } + + const result = resolveProxy(projectDir, sourcePath); + await answerAll(calls); + await result; + expect(startsOf(calls)).toEqual(["0", String(PIECE_SECONDS), String(2 * PIECE_SECONDS)]); + }); + + it("continues its own stopped copy after a restart, but not pieces another user made", async () => { + const projectDir = tmpProject(); + const sourcePath = join(projectDir, "long.mov"); + writeFileSync(sourcePath, "source-bytes"); + const stopAfterFirstPiece = async (token: string) => { + const { spawn, calls } = createSpawnSpy(); + const mod = await loadModule( + spawn, + FFMPEG_PATH, + false, + null, + undefined, + LONG_SECONDS, + undefined, + token, + ); + const run = mod.resolveProxy(projectDir, sourcePath).catch((error: unknown) => error); + await flush(); + succeed(calls[0]!); + await flush(50); + calls[1]!.proc.emit("close", null, "SIGKILL"); + expect(await run).toBeInstanceOf(Error); + }; + const resumeStarts = async (token: string) => { + const { spawn, calls } = createSpawnSpy(); + const mod = await loadModule( + spawn, + FFMPEG_PATH, + false, + null, + undefined, + LONG_SECONDS, + undefined, + token, + ); + const run = mod.resolveProxy(projectDir, sourcePath); + await answerAll(calls); + await run; + return startsOf(calls); + }; + + await stopAfterFirstPiece("a".repeat(64)); + expect(await resumeStarts("b".repeat(64))).toEqual([ + "0", + String(PIECE_SECONDS), + String(2 * PIECE_SECONDS), + ]); + rmSync(join(projectDir, ".transcode-cache"), { recursive: true, force: true }); + await stopAfterFirstPiece("a".repeat(64)); + expect(await resumeStarts("a".repeat(64))).toEqual([ + String(PIECE_SECONDS), + String(2 * PIECE_SECONDS), + ]); + }); + + it("warns once when no preview token can be read, and still makes the copy", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { spawn, calls } = createSpawnSpy(); + const { resolveProxy } = await loadModule( + spawn, + FFMPEG_PATH, + false, + null, + undefined, + LONG_SECONDS, + undefined, + null, + ); + const projectDir = tmpProject(); + for (const name of ["one.mov", "two.mov"]) { + const sourcePath = join(projectDir, name); + writeFileSync(sourcePath, `source-${name}`); + const result = resolveProxy(projectDir, sourcePath); + await answerAll(calls); + await result; + } + const tokenWarnings = warn.mock.calls.filter(([message]) => + String(message).includes("no preview token"), + ); + expect(tokenWarnings).toHaveLength(1); + warn.mockRestore(); + }); + it("copies only the first stretch when asked, as a copy of its own", async () => { const { spawn, calls } = createSpawnSpy(); const { resolveProxy, getProxyCachePath, proxyWindowSeconds } = await loadModule( diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index d16bcb438c..36e948e281 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { createHmac, randomBytes, randomUUID } from "node:crypto"; import { existsSync, mkdtempSync, @@ -12,7 +12,12 @@ import { import os, { availableParallelism, tmpdir, totalmem } from "node:os"; import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { hdrToSdrToneMapFilter, type PreviewProxyBox } from "@hyperframes/core"; -import { findFfBinary, followFfmpegProgress } from "@hyperframes/parsers/ff-binaries"; +import { previewToken } from "@hyperframes/core/preview-proof"; +import { + concatListInputArgs, + findFfBinary, + followFfmpegProgress, +} from "@hyperframes/parsers/ff-binaries"; import { alphaDecoderFor } from "@hyperframes/parsers/media-alpha"; import { ffmpegHasHdrFilters } from "./hdrFilters.js"; import { mediaInputArgs, NOT_MEDIA } from "./mediaExtensions.js"; @@ -71,6 +76,30 @@ export const TRANSCODE_STALL_MS = boundedEnvInteger( /** A longer source is copied in pieces this long; a stopped copy keeps its finished pieces and loses at most one. */ export const PROXY_CHUNK_SECONDS = 15; const CHUNK_PREFIX = ".part-"; +// Without a preview token, pieces are reused only within this process. +const UNSIGNED_PIECE_SECRET = randomBytes(32).toString("hex"); +let warnedNoPieceToken = false; + +function pieceSecret(): string { + const token = previewToken(); + if (token) return token; + if (!warnedNoPieceToken) { + warnedNoPieceToken = true; + console.warn( + "[media-proxy] no preview token could be read, so a stopped copy restarts from its first piece after Studio restarts", + ); + } + return UNSIGNED_PIECE_SECRET; +} + +function unguessablePiecePrefix(cacheDir: string, stem: string): string { + const tag = createHmac("sha256", pieceSecret()) + .update(`proxy-piece\n${stem}`) + .digest("hex") + .slice(0, 32); + return join(cacheDir, `${CHUNK_PREFIX}${stem}-${tag}-`); +} + const CONCAT_TIMEOUT_MS = 2 * 60 * 1000; const FAILURE_CACHE_TTL_MS = 60 * 1000; export const PROXY_PENDING_RETRY_AFTER_SECONDS = 2; @@ -818,12 +847,7 @@ async function runFfmpeg( function concatArgs(listPath: string, target: string, variant: ProxyVariant): string[] { return [ "-y", - "-f", - "concat", - "-safe", - "0", - "-i", - listPath, + ...concatListInputArgs(listPath), "-c", "copy", ...(variant === "h264" ? ["-movflags", "+faststart"] : []), @@ -945,11 +969,16 @@ async function transcodeToCache( return sameContent; } const copyMade = (path: string) => isProjectCacheFile(projectDir, path); + const chunks = { + projectDir, + prefix: unguessablePiecePrefix(cacheDir, basename(cachePath, extname(cachePath))), + }; const release = await acquireCopyLock( cachePath, copyMade, job.abandoned.signal, TRANSCODE_STALL_MS, + basename(chunks.prefix), ).catch((error: unknown) => { throw error instanceof CopyLockStalledError ? new ProxyTranscodeError(error.message, null, "") @@ -961,10 +990,6 @@ async function transcodeToCache( } const tempPath = join(cacheDir, `.tmp-${randomUUID()}-${basename(cachePath)}`); try { - const chunks = { - projectDir, - prefix: join(cacheDir, `${CHUNK_PREFIX}${basename(cachePath, extname(cachePath))}-`), - }; await runFfmpeg( absoluteSourcePath, tempPath,