From cfea8cfff5a5f833afac14866b63c290fa59fc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 18 Nov 2026 06:36:23 +0000 Subject: [PATCH] fix(producer): a render that fails on a missing file says what was gone (#7955) * fix(producer): a render that fails on a missing file says what was gone * fix(producer): a failed look at what was missing never hides the render's own error The missing-path probe now returns nothing when it throws, reads through at most four links and only links that point into the temp root or the cache folder, names a removed folder by its own name, treats a custom cache folder as the cache, and skips programs that could not start. * refactor(producer): split the missing-path link walk into its own step --- packages/cli/src/commands/render.ts | 1 + packages/cli/src/telemetry/events.test.ts | 18 +++ packages/cli/src/telemetry/events.ts | 7 + packages/cli/src/telemetry/missingPathKind.ts | 20 +-- .../cli/src/telemetry/renderObservability.ts | 1 + packages/core/package-subpaths.json | 6 + packages/core/package.json | 10 ++ packages/core/src/missingPath.ts | 18 +++ packages/producer/src/index.ts | 1 + .../producer/src/services/render/cleanup.ts | 3 + .../src/services/render/missingPath.test.ts | 128 ++++++++++++++++ .../src/services/render/missingPath.ts | 105 +++++++++++++ .../src/services/render/observability.ts | 1 + .../renderOrchestrator.missingPath.test.ts | 140 ++++++++++++++++++ .../src/services/renderOrchestrator.ts | 4 + 15 files changed, 445 insertions(+), 18 deletions(-) create mode 100644 packages/core/src/missingPath.ts create mode 100644 packages/producer/src/services/render/missingPath.test.ts create mode 100644 packages/producer/src/services/render/missingPath.ts create mode 100644 packages/producer/src/services/renderOrchestrator.missingPath.test.ts diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index 0b21847659..8919d795ac 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -2010,6 +2010,7 @@ function handleRenderError( errorName: error instanceof Error ? error.name : "unknown", failedStageCode: normalizeStageCode(failedStage || "pipeline"), missingPathKind: missingPathKind(error), + missingPath: job?.errorDetails?.missingPath, ...renderJobObservabilityTelemetryPayload(job), ...getMemorySnapshot(), ...cpu.stop(), diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index 0af176f4ca..0fa9358213 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -690,6 +690,24 @@ describe("render telemetry events", () => { expect(props.hf_env_overrides).toEqual(["HF_DE_VERIFY", "HYPERFRAMES_FONT_CACHE_DIR"]); }); + it("sends only the closed-list missing-path facts on render_error", () => { + trackRenderError({ + fps: 30, + quality: "draft", + docker: false, + missingPath: { scope: "cache_entry", synced: true, root: "system_temp" }, + }); + const props = trackEvent.mock.calls[0]?.[1] as Record; + expect(props).toMatchObject({ + missing_path_scope: "cache_entry", + missing_path_synced: true, + missing_path_root: "system_temp", + }); + expect(Object.values(props).filter((v) => typeof v === "string" && /[\\/]/.test(v))).toEqual( + [], + ); + }); + it("reports an empty array, not an absent field, when no override was resolved", () => { trackRenderError({ fps: 30, quality: "draft", docker: false }); const props = trackEvent.mock.calls[0]?.[1] as Record; diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index eb708ad16f..09f0603552 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -14,6 +14,7 @@ import { renderMotionBlurEventProperties, } from "./renderMotionBlur.js"; import { CliRuntimeError } from "../utils/commandResult.js"; +import type { MissingPathFacts } from "@hyperframes/producer"; // Power state is volatile (a laptop docks/undocks mid-session), so it is // sampled per render event rather than cached with SystemMeta. Attached to @@ -127,6 +128,7 @@ export interface RenderObservabilityTelemetryPayload { observabilityExtractVfrPreflightCount?: number; observabilityExtractCacheHits?: number; observabilityExtractCacheMisses?: number; + observabilityExtractCachePublishFailures?: number; observabilityInitDurationMs?: number; observabilityInitTweenCount?: number; observabilityInitElementCount?: number; @@ -207,6 +209,7 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP observability_extract_vfr_preflight_count: props.observabilityExtractVfrPreflightCount, observability_extract_cache_hits: props.observabilityExtractCacheHits, observability_extract_cache_misses: props.observabilityExtractCacheMisses, + observability_extract_cache_publish_failures: props.observabilityExtractCachePublishFailures, observability_init_duration_ms: props.observabilityInitDurationMs, observability_init_tween_count: props.observabilityInitTweenCount, observability_init_element_count: props.observabilityInitElementCount, @@ -702,6 +705,7 @@ export function trackRenderError( /** failedStage normalized to a stable snake_case code. */ failedStageCode?: string; missingPathKind?: string; + missingPath?: MissingPathFacts; errorMessage?: string; elapsedMs?: number; peakMemoryMb?: number; @@ -731,6 +735,9 @@ export function trackRenderError( error_name: props.errorName, failed_stage_code: props.failedStageCode, missing_path_kind: props.missingPathKind, + missing_path_scope: props.missingPath?.scope, + missing_path_synced: props.missingPath?.synced, + missing_path_root: props.missingPath?.root, error_message: props.errorMessage ? redactTelemetryMessage(props.errorMessage) : undefined, elapsed_ms: props.elapsedMs, ...renderOutputShapeEventProperties(props), diff --git a/packages/cli/src/telemetry/missingPathKind.ts b/packages/cli/src/telemetry/missingPathKind.ts index 281d0cc7b3..be7d12fc9a 100644 --- a/packages/cli/src/telemetry/missingPathKind.ts +++ b/packages/cli/src/telemetry/missingPathKind.ts @@ -1,3 +1,5 @@ +import { missingPathOf } from "@hyperframes/core/missing-path"; + const KINDS: ReadonlyArray<[RegExp, string]> = [ [/(^|[\\/])hfcache-/, "extract_cache"], [/(^|[\\/])__hyperframes_video_frames([\\/]|$)/, "video_frames"], @@ -11,24 +13,6 @@ const KINDS: ReadonlyArray<[RegExp, string]> = [ [/(^|[\\/])(hf-render-|work-[0-9a-f]{8}-)/, "work_dir"], ]; -const quotedPath = (text: string) => /'([^']+)'/.exec(text)?.[1] ?? ""; - -function errorChain(error: unknown): Error[] { - const chain: Error[] = []; - for (let seen = error; seen instanceof Error && chain.length < 5; seen = seen.cause) - chain.push(seen); - return chain; -} - -function missingPathOf(error: unknown): string | undefined { - const enoent = errorChain(error).find( - (seen) => (seen as NodeJS.ErrnoException).code === "ENOENT", - ); - if (enoent) return (enoent as NodeJS.ErrnoException).path ?? quotedPath(enoent.message); - const message = error instanceof Error ? error.message : String(error); - return /\bENOENT\b/.test(message) ? quotedPath(message) : undefined; -} - /** Which kind of file a render found missing (a fixed name, never the path), or undefined when nothing was missing. */ export function missingPathKind(error: unknown): string | undefined { const path = missingPathOf(error); diff --git a/packages/cli/src/telemetry/renderObservability.ts b/packages/cli/src/telemetry/renderObservability.ts index 55fb8ac615..f35dc7b590 100644 --- a/packages/cli/src/telemetry/renderObservability.ts +++ b/packages/cli/src/telemetry/renderObservability.ts @@ -84,6 +84,7 @@ export function renderObservabilityTelemetryPayload( observabilityExtractVfrPreflightCount: extraction?.vfrPreflightCount, observabilityExtractCacheHits: extraction?.cacheHits, observabilityExtractCacheMisses: extraction?.cacheMisses, + observabilityExtractCachePublishFailures: extraction?.cachePublishFailures, observabilityInitDurationMs: init?.initDurationMs, observabilityInitTweenCount: init?.tweenCount, observabilityInitElementCount: init?.elementCount, diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json index 9addcf3143..6b3077e9ca 100644 --- a/packages/core/package-subpaths.json +++ b/packages/core/package-subpaths.json @@ -110,6 +110,12 @@ "types": "./dist/envProxy.d.ts", "environments": ["bun", "node"] }, + "./missing-path": { + "source": "./src/missingPath.ts", + "runtime": "./dist/missingPath.js", + "types": "./dist/missingPath.d.ts", + "environments": ["bun", "node"] + }, "./rename-retry": { "source": "./src/renameWithRetry.ts", "runtime": "./dist/renameWithRetry.js", diff --git a/packages/core/package.json b/packages/core/package.json index f83cd570b2..6f3cc98223 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -122,6 +122,12 @@ "import": "./src/envProxy.ts", "types": "./src/envProxy.ts" }, + "./missing-path": { + "bun": "./src/missingPath.ts", + "node": "./dist/missingPath.js", + "import": "./src/missingPath.ts", + "types": "./src/missingPath.ts" + }, "./rename-retry": { "bun": "./src/renameWithRetry.ts", "node": "./dist/renameWithRetry.js", @@ -744,6 +750,10 @@ "import": "./dist/envProxy.js", "types": "./dist/envProxy.d.ts" }, + "./missing-path": { + "import": "./dist/missingPath.js", + "types": "./dist/missingPath.d.ts" + }, "./rename-retry": { "import": "./dist/renameWithRetry.js", "types": "./dist/renameWithRetry.d.ts" diff --git a/packages/core/src/missingPath.ts b/packages/core/src/missingPath.ts new file mode 100644 index 0000000000..1dc907a441 --- /dev/null +++ b/packages/core/src/missingPath.ts @@ -0,0 +1,18 @@ +const quotedPath = (text: string) => /'([^']+)'/.exec(text)?.[1] ?? ""; + +function errorChain(error: unknown): Error[] { + const chain: Error[] = []; + for (let seen = error; seen instanceof Error && chain.length < 5; seen = seen.cause) + chain.push(seen); + return chain; +} + +/** The path an ENOENT in `error` (or its causes) names, or undefined when nothing was missing. */ +export function missingPathOf(error: unknown): string | undefined { + const enoent = errorChain(error).find( + (seen) => (seen as NodeJS.ErrnoException).code === "ENOENT", + ); + if (enoent) return (enoent as NodeJS.ErrnoException).path ?? quotedPath(enoent.message); + const message = error instanceof Error ? error.message : String(error); + return /\bENOENT\b/.test(message) ? quotedPath(message) : undefined; +} diff --git a/packages/producer/src/index.ts b/packages/producer/src/index.ts index 1ee2942eaf..01adb60a34 100644 --- a/packages/producer/src/index.ts +++ b/packages/producer/src/index.ts @@ -45,6 +45,7 @@ export { createOwnedRenderDir, listAbandonedRenderDirs, } from "./services/render/renderDirOwner.js"; +export type { MissingPathFacts } from "./services/render/missingPath.js"; export { RENDER_REQUEST_VERSION, createRenderRequest, diff --git a/packages/producer/src/services/render/cleanup.ts b/packages/producer/src/services/render/cleanup.ts index d4bc228a96..7a2ef378e6 100644 --- a/packages/producer/src/services/render/cleanup.ts +++ b/packages/producer/src/services/render/cleanup.ts @@ -5,6 +5,7 @@ import { type SubTimelineWaitOutcome } from "@hyperframes/engine"; import type { HdrDiagnostics, RenderJob } from "../renderOrchestrator.js"; import { normalizeErrorMessage } from "../../utils/errorMessage.js"; import type { RenderObservabilitySummary } from "./observability.js"; +import type { MissingPathFacts } from "./missingPath.js"; /** * Build the `RenderJob.errorDetails` shape downstream consumers (SSE, @@ -18,6 +19,7 @@ export function buildRenderErrorDetails(input: { hdrDiagnostics: HdrDiagnostics; observability?: RenderObservabilitySummary; subTimelineWait?: SubTimelineWaitOutcome; + missingPath?: MissingPathFacts; }): NonNullable { const errorMessage = normalizeErrorMessage(input.error); const errorStack = input.error instanceof Error ? input.error.stack : undefined; @@ -36,5 +38,6 @@ export function buildRenderErrorDetails(input: { : undefined, observability: input.observability, subTimelineWait: input.subTimelineWait, + missingPath: input.missingPath, }; } diff --git a/packages/producer/src/services/render/missingPath.test.ts b/packages/producer/src/services/render/missingPath.test.ts new file mode 100644 index 0000000000..5c228eeb36 --- /dev/null +++ b/packages/producer/src/services/render/missingPath.test.ts @@ -0,0 +1,128 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describeMissingPath } from "./missingPath.js"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +/** A temp folder laid out like a render's: the extract cache with one entry, and a work dir whose compiled frames link + * into that entry. */ +function layout() { + const temp = mkdtempSync(join(tmpdir(), "hf-missing-")); + roots.push(temp); + const cacheRoot = join(temp, "hyperframes-extract-cache-u"); + const entry = join(cacheRoot, "hfcache-v10-abc"); + const work = join(temp, "hf-render-xyz"); + const link = join(work, "compiled", "__hyperframes_video_frames", "v1"); + mkdirSync(entry, { recursive: true }); + writeFileSync(join(entry, "frame_00001.jpg"), "x"); + mkdirSync(join(work, "compiled", "__hyperframes_video_frames"), { recursive: true }); + symlinkSync(entry, link, process.platform === "win32" ? "junction" : "dir"); + return { temp, cacheRoot, entry, work, linkedFrame: join(link, "frame_00002.jpg") }; +} + +const enoent = (path: string) => + Object.assign(new Error(`ENOENT: no such file or directory, open '${path}'`), { + code: "ENOENT", + path, + }); + +describe("describeMissingPath", () => { + it("says only the file is gone when its folder is there", () => { + const { temp, linkedFrame } = layout(); + expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })).toEqual({ + scope: "file", + synced: false, + root: "system_temp", + }); + }); + + it("reads through a link to the cache entry that went", () => { + const { temp, entry, linkedFrame } = layout(); + rmSync(entry, { recursive: true, force: true }); + expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("cache_entry"); + }); + + it("names an unfinished cache entry that went", () => { + const { temp, cacheRoot } = layout(); + const partial = join(cacheRoot, "hfcache-v10-def.partial-123"); + expect( + describeMissingPath(enoent(join(partial, "frame_00001.jpg")), { tempRoot: temp })?.scope, + ).toBe("cache_partial"); + }); + + it("names the whole cache, the work folder and the temp folder when those went", () => { + const { temp, cacheRoot, work, entry, linkedFrame } = layout(); + rmSync(cacheRoot, { recursive: true, force: true }); + expect( + describeMissingPath(enoent(join(entry, "frame_00001.jpg")), { tempRoot: temp })?.scope, + ).toBe("cache_root"); + rmSync(work, { recursive: true, force: true }); + expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("work_dir"); + rmSync(temp, { recursive: true, force: true }); + expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("temp_root"); + }); + + it("says whether a sync app owns the path and which root it is under", () => { + const synced = "C:\\Users\\a\\OneDrive - Contoso\\renders\\.frames\\frame_00001.jpg"; + expect(describeMissingPath(enoent(synced), { tempRoot: "C:\\Temp" })).toMatchObject({ + synced: true, + root: "other", + }); + const cacheDir = mkdtempSync(join(tmpdir(), "hf-cache-dir-")); + roots.push(cacheDir); + expect( + describeMissingPath(enoent(join(cacheDir, "hfcache-v10-a", "frame_00001.jpg")), { + tempRoot: join(tmpdir(), "not-here"), + cacheDir, + })?.root, + ).toBe("cache_dir"); + }); + + it("names a removed folder by its own name and the custom cache folder as the cache", () => { + const { temp, entry } = layout(); + rmSync(entry, { recursive: true, force: true }); + expect(describeMissingPath(enoent(entry), { tempRoot: temp })?.scope).toBe("cache_entry"); + const cacheDir = join(temp, "my-cache"); + expect( + describeMissingPath(enoent(join(cacheDir, "x.jpg")), { tempRoot: temp, cacheDir })?.scope, + ).toBe("cache_root"); + expect( + describeMissingPath(enoent(join(temp, "gone", "x.jpg")), { tempRoot: temp })?.scope, + ).toBe("other"); + }); + + it("does not read through a link that points outside the temp root and cache folder", () => { + const { temp, work } = layout(); + const outside = mkdtempSync(join(tmpdir(), "hf-outside-")); + roots.push(outside); + const link = join(work, "out"); + symlinkSync( + join(outside, "hfcache-v10-x", "dir"), + link, + process.platform === "win32" ? "junction" : "dir", + ); + expect(describeMissingPath(enoent(join(link, "x.jpg")), { tempRoot: temp })?.scope).toBe( + "link_target", + ); + }); + + it("stops on a link that points into itself", () => { + const { temp, work } = layout(); + const loop = join(work, "loop"); + symlinkSync(join(loop, "x"), loop, process.platform === "win32" ? "junction" : "dir"); + expect( + describeMissingPath(enoent(join(loop, "x", "frame.jpg")), { tempRoot: temp })?.scope, + ).toBe("link_target"); + }); + + it("says nothing when nothing was missing or a program could not start", () => { + expect(describeMissingPath(new Error("Target closed"))).toBeUndefined(); + const spawn = Object.assign(enoent("ffmpeg"), { syscall: "spawn ffmpeg" }); + expect(describeMissingPath(spawn)).toBeUndefined(); + }); +}); diff --git a/packages/producer/src/services/render/missingPath.ts b/packages/producer/src/services/render/missingPath.ts new file mode 100644 index 0000000000..4dcb1305a7 --- /dev/null +++ b/packages/producer/src/services/render/missingPath.ts @@ -0,0 +1,105 @@ +import { existsSync, lstatSync, readlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, relative, resolve, isAbsolute } from "node:path"; +import { missingPathOf } from "@hyperframes/core/missing-path"; + +/** What was gone around a missing path when the render failed, from closed lists only: never a path or a name. */ +export interface MissingPathFacts { + /** The highest thing gone on the path; a link on the way is read through to its target. */ + scope: + | "file" + | "link_target" + | "cache_entry" + | "cache_partial" + | "cache_root" + | "work_dir" + | "temp_root" + | "other"; + /** Under a folder a sync app owns (OneDrive, Dropbox, iCloud, Google Drive). */ + synced: boolean; + root: "system_temp" | "cache_dir" | "other"; +} + +const SYNCED = /onedrive|dropbox|icloud|mobile documents|clouddocs|google ?drive|cloudstorage/i; + +interface Where { + tempRoot: string; + cacheDir?: string; +} + +const inside = (path: string, folder: string) => { + const rel = relative(resolve(folder), resolve(path)); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +}; + +function isLink(path: string): boolean { + try { + return lstatSync(path).isSymbolicLink(); + } catch { + // expected: a missing path is no link + return false; + } +} + +function scopeOfGone(path: string, where: Where, throughLink: boolean): MissingPathFacts["scope"] { + if (resolve(path) === resolve(where.tempRoot)) return "temp_root"; + if (where.cacheDir && resolve(path) === resolve(where.cacheDir)) return "cache_root"; + const name = basename(path); + if (/\.partial/.test(name)) return "cache_partial"; + if (/^hfcache-/.test(name)) return "cache_entry"; + if (/^hyperframes-extract-cache/.test(name)) return "cache_root"; + if (/^(hf-render-|work-[0-9a-f]{8}-)/.test(name)) return "work_dir"; + return throughLink ? "link_target" : "other"; +} + +const MAX_LINK_HOPS = 4; + +function rootOf(path: string, where: Where): MissingPathFacts["root"] { + if (inside(path, where.tempRoot)) return "system_temp"; + if (where.cacheDir && inside(path, where.cacheDir)) return "cache_dir"; + return "other"; +} + +/** A dangling link on the way, read through to its target only when that target is in the temp root or the cache folder. */ +function goneThroughLink(link: string, where: Where, hops: number): MissingPathFacts["scope"] { + const target = resolve(dirname(link), readlinkSync(link).replace(/^\\\\\?\\/, "")); + return rootOf(target, where) !== "other" && hops < MAX_LINK_HOPS + ? highestGone(target, where, hops + 1) + : "link_target"; +} + +/** The highest thing gone on `path`: the first folder up whose parent is there. */ +function highestGone(path: string, where: Where, hops = 0): MissingPathFacts["scope"] { + for (let at = path; dirname(at) !== at; at = dirname(at)) { + const parent = dirname(at); + if (isLink(parent) && !existsSync(parent)) return goneThroughLink(parent, where, hops); + if (!existsSync(parent)) continue; + const scope = scopeOfGone(at, where, hops > 0); + return at === path && scope === "other" ? "file" : scope; + } + return "other"; +} + +const isSpawnFailure = (error: unknown) => + !!(error as NodeJS.ErrnoException | undefined)?.syscall?.startsWith("spawn"); + +/** What was gone around the path a failed render's ENOENT names; undefined when it names none or the look fails. + * Call it before the render's own cleanup runs. */ +export function describeMissingPath( + error: unknown, + { cacheDir, tempRoot = tmpdir() }: { cacheDir?: string; tempRoot?: string } = {}, +): MissingPathFacts | undefined { + try { + const path = missingPathOf(error); + if (!path || isSpawnFailure(error)) return undefined; + const where = { tempRoot, cacheDir }; + return { + scope: highestGone(path, where), + synced: SYNCED.test(path), + root: rootOf(path, where), + }; + } catch { + // expected: the folders can change under the look; the render's own error must still be what is reported + return undefined; + } +} diff --git a/packages/producer/src/services/render/observability.ts b/packages/producer/src/services/render/observability.ts index 363dee4a7d..b7ea7d4f3e 100644 --- a/packages/producer/src/services/render/observability.ts +++ b/packages/producer/src/services/render/observability.ts @@ -224,6 +224,7 @@ export interface RenderExtractionObservability { vfrPreflightCount?: number; cacheHits?: number; cacheMisses?: number; + cachePublishFailures?: number; /** Per-source transient download/metadata/FFmpeg retries performed during extraction. */ transientRetries?: number; /** diff --git a/packages/producer/src/services/renderOrchestrator.missingPath.test.ts b/packages/producer/src/services/renderOrchestrator.missingPath.test.ts new file mode 100644 index 0000000000..cd3911c759 --- /dev/null +++ b/packages/producer/src/services/renderOrchestrator.missingPath.test.ts @@ -0,0 +1,140 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveConfig } from "@hyperframes/engine"; +import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js"; + +// Entry: executeRenderJob. Capture setup fails on a frame whose cache entry another hand removed; the failure must +// say what was gone, read before the render removes its own folders. +const trace = vi.hoisted(() => ({ missing: "", probeThrows: false })); + +vi.mock("@hyperframes/core/missing-path", async (importOriginal) => { + const real = await importOriginal(); + return { + missingPathOf: (error: unknown) => { + if (trace.probeThrows) throw new RangeError("Maximum call stack size exceeded"); + return real.missingPathOf(error); + }, + }; +}); + +vi.mock("./render/shared.js", async (importOriginal) => ({ + ...(await importOriginal()), + createMemorySampler: () => ({ + peakRssBytes: () => 0, + peakHeapUsedBytes: () => 0, + stop: () => {}, + }), +})); +vi.mock("./fileServer.js", async (importOriginal) => ({ + ...(await importOriginal()), + createFileServer: async () => ({ + url: "http://127.0.0.1:1", + port: 1, + close: () => {}, + addPreHeadScript: () => {}, + }), +})); +vi.mock("./render/stages/compileStage.js", () => ({ + runCompileStage: async () => ({ + compiled: { + html: "
", + hasShaderTransitions: false, + renderModeHints: { reasons: [] }, + }, + composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] }, + deviceScaleFactor: 1, + outputWidth: 16, + outputHeight: 16, + compileOnlyMs: 0, + forceScreenshot: true, + }), +})); +vi.mock("./render/stages/probeStage.js", () => ({ + runProbeStage: async ({ compiled }: { compiled: object }) => ({ + compiled, + fileServer: null, + probeSession: null, + lastBrowserConsole: [], + duration: 40 / 30, + totalFrames: 40, + browserProbeMs: 0, + beginFrameStalled: false, + }), +})); +vi.mock("./render/stages/extractVideosStage.js", () => ({ + runExtractVideosStage: async () => ({ + extractionResult: null, + frameLookup: null, + videoReadinessSkipIds: [], + videoMetadataHints: [], + nativeHdrVideoIds: new Set(), + videoTransfers: new Map(), + nativeHdrImageIds: new Set(), + imageTransfers: new Map(), + hdrImageSrcPaths: new Map(), + imageColorSpaces: [], + videoExtractMs: 0, + failureToEnforce: null, + }), +})); +vi.mock("./render/stages/audioStage.js", () => ({ + runAudioStage: async ({ workDir }: { workDir: string }) => ({ + audioOutputPath: join(workDir, "audio.wav"), + hasAudio: false, + audioProcessMs: 0, + }), +})); +vi.mock("./render/captureCost.js", async (importOriginal) => ({ + ...(await importOriginal()), + resolveRenderWorkerCount: () => { + throw Object.assign(new Error(`ENOENT: no such file or directory, open '${trace.missing}'`), { + code: "ENOENT", + path: trace.missing, + }); + }, +})); + +const dirs: string[] = []; +afterEach(() => { + trace.probeThrows = false; + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function failingRender() { + const root = mkdtempSync(join(tmpdir(), "hf-missing-render-")); + dirs.push(root); + writeFileSync(join(root, "index.html"), "
"); + const cacheRoot = join(root, "hyperframes-extract-cache-u"); + mkdirSync(cacheRoot); + trace.missing = join(cacheRoot, "hfcache-v10-gone", "frame_00001.jpg"); + const job = createRenderJob({ + fps: 30, + quality: "draft", + format: "mp4", + hdrMode: "force-sdr", + producerConfig: resolveConfig({ enablePageSideCompositing: false }), + logger: { info() {}, warn() {}, error() {}, debug() {} }, + }); + return { job, run: () => executeRenderJob(job, root, join(root, "out.mp4")) }; +} + +it("a render that fails on a missing frame says the cache entry holding it was gone", async () => { + const { job, run } = failingRender(); + await expect(run()).rejects.toThrow(/ENOENT/); + expect(job.errorDetails?.missingPath).toEqual({ + scope: "cache_entry", + synced: false, + root: "system_temp", + }); +}); + +it("a missing-path look that throws leaves the render's own failure as it was", async () => { + trace.probeThrows = true; + const { job, run } = failingRender(); + await expect(run()).rejects.toThrow(/ENOENT/); + expect(job.status).toBe("failed"); + expect(job.error).toMatch(/ENOENT/); + expect(job.errorDetails?.missingPath).toBeUndefined(); +}); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index b1b19a3743..10c069fe20 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -169,6 +169,7 @@ import { } from "./render/shared.js"; import { frameCacheOptions } from "./render/frameCacheBudget.js"; import { buildRenderErrorDetails } from "./render/cleanup.js"; +import { describeMissingPath, type MissingPathFacts } from "./render/missingPath.js"; import { publishRenderFailure } from "./render/renderEventPublisher.js"; import { EncoderInterruptedError } from "./render/encoderInterruption.js"; import { RenderExecutionContext } from "./render/renderExecutionContext.js"; @@ -347,6 +348,7 @@ function summarizeExtractionObservability( vfrPreflightCount: phaseBreakdown?.vfrPreflightCount, cacheHits: phaseBreakdown?.cacheHits, cacheMisses: phaseBreakdown?.cacheMisses, + cachePublishFailures: phaseBreakdown?.cachePublishFailures, transientRetries: phaseBreakdown?.transientRetries, ...coverageGauges, authoredTimedClipCount, @@ -840,6 +842,7 @@ export interface RenderJob extends Partial { observability?: RenderObservabilitySummary; /** Worst sub-composition timeline wait outcome across sessions captured before the failure. */ subTimelineWait?: SubTimelineWaitOutcome; + missingPath?: MissingPathFacts; }; } @@ -6033,6 +6036,7 @@ async function executeRenderPipeline(input: { hdrDiagnostics, observability: observabilitySummary, subTimelineWait: worstSubTimelineWaitOutcome(dedupPerfs), + missingPath: describeMissingPath(error, { cacheDir: cfg.extractCacheDir }), }); publishRenderFailure( job,