From a82ee10c68cef925d9bf949e528db08bec4f1f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 18 Nov 2026 13:41:39 +0000 Subject: [PATCH] feat(producer): a failed render says which file was gone again, looking only in its own folders (#8168) * feat(producer): a render that fails on a missing file says what was gone again Restores the diagnostic the previous commit on main took out, as it was; the commits after it confine what it looks at. * test(producer): render failure diagnostics never probe a path from error text or outside the render * fix(producer): probe a missing path only from a real error, inside the render's folders describeMissingPath now reads only the path Node set on an ENOENT error, never one quoted in a message the page can write, and looks on disk only under the render's temp root, extract cache or work folder. Any other path is named from its text, so a network share is never contacted and the facts cannot tell an existing path from a missing one. * test(producer): a missing-path probe never passes an outside link or climbs above its folder * fix(producer): walk down from the render's own folder and check each link before following it * build(producer): give the packed size budget its 5% headroom with the missing-file diagnostic back --- packages/cli/src/commands/render.ts | 1 + packages/cli/src/telemetry/events.test.ts | 18 ++ packages/cli/src/telemetry/events.ts | 7 + packages/cli/src/telemetry/missingPathKind.ts | 20 +- .../cli/src/telemetry/renderObservability.ts | 1 + packages/core/package-subpaths.json | 6 + packages/core/package.json | 10 + packages/core/src/missingPath.ts | 25 ++ packages/producer/src/index.ts | 1 + .../producer/src/services/render/cleanup.ts | 3 + .../src/services/render/missingPath.test.ts | 217 ++++++++++++++++++ .../src/services/render/missingPath.ts | 143 ++++++++++++ .../src/services/render/observability.ts | 1 + .../renderOrchestrator.missingPath.test.ts | 140 +++++++++++ .../src/services/renderOrchestrator.ts | 4 + scripts/check-no-main-deletions.mjs | 9 - scripts/package-size-budgets.json | 2 +- 17 files changed, 580 insertions(+), 28 deletions(-) create mode 100644 packages/core/src/missingPath.ts create mode 100644 packages/producer/src/services/render/missingPath.test.ts create mode 100644 packages/producer/src/services/render/missingPath.ts create mode 100644 packages/producer/src/services/renderOrchestrator.missingPath.test.ts diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index e79b7d8050..c1c7aaf2e5 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -2010,6 +2010,7 @@ function handleRenderError( errorName: error instanceof Error ? error.name : "unknown", failedStageCode: normalizeStageCode(failedStage || "pipeline"), missingPathKind: missingPathKind(error), + missingPath: job?.errorDetails?.missingPath, ...renderJobObservabilityTelemetryPayload(job), ...getMemorySnapshot(), ...cpu.stop(), diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index 0af176f4ca..0fa9358213 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -690,6 +690,24 @@ describe("render telemetry events", () => { expect(props.hf_env_overrides).toEqual(["HF_DE_VERIFY", "HYPERFRAMES_FONT_CACHE_DIR"]); }); + it("sends only the closed-list missing-path facts on render_error", () => { + trackRenderError({ + fps: 30, + quality: "draft", + docker: false, + missingPath: { scope: "cache_entry", synced: true, root: "system_temp" }, + }); + const props = trackEvent.mock.calls[0]?.[1] as Record; + expect(props).toMatchObject({ + missing_path_scope: "cache_entry", + missing_path_synced: true, + missing_path_root: "system_temp", + }); + expect(Object.values(props).filter((v) => typeof v === "string" && /[\\/]/.test(v))).toEqual( + [], + ); + }); + it("reports an empty array, not an absent field, when no override was resolved", () => { trackRenderError({ fps: 30, quality: "draft", docker: false }); const props = trackEvent.mock.calls[0]?.[1] as Record; diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index eb708ad16f..09f0603552 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -14,6 +14,7 @@ import { renderMotionBlurEventProperties, } from "./renderMotionBlur.js"; import { CliRuntimeError } from "../utils/commandResult.js"; +import type { MissingPathFacts } from "@hyperframes/producer"; // Power state is volatile (a laptop docks/undocks mid-session), so it is // sampled per render event rather than cached with SystemMeta. Attached to @@ -127,6 +128,7 @@ export interface RenderObservabilityTelemetryPayload { observabilityExtractVfrPreflightCount?: number; observabilityExtractCacheHits?: number; observabilityExtractCacheMisses?: number; + observabilityExtractCachePublishFailures?: number; observabilityInitDurationMs?: number; observabilityInitTweenCount?: number; observabilityInitElementCount?: number; @@ -207,6 +209,7 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP observability_extract_vfr_preflight_count: props.observabilityExtractVfrPreflightCount, observability_extract_cache_hits: props.observabilityExtractCacheHits, observability_extract_cache_misses: props.observabilityExtractCacheMisses, + observability_extract_cache_publish_failures: props.observabilityExtractCachePublishFailures, observability_init_duration_ms: props.observabilityInitDurationMs, observability_init_tween_count: props.observabilityInitTweenCount, observability_init_element_count: props.observabilityInitElementCount, @@ -702,6 +705,7 @@ export function trackRenderError( /** failedStage normalized to a stable snake_case code. */ failedStageCode?: string; missingPathKind?: string; + missingPath?: MissingPathFacts; errorMessage?: string; elapsedMs?: number; peakMemoryMb?: number; @@ -731,6 +735,9 @@ export function trackRenderError( error_name: props.errorName, failed_stage_code: props.failedStageCode, missing_path_kind: props.missingPathKind, + missing_path_scope: props.missingPath?.scope, + missing_path_synced: props.missingPath?.synced, + missing_path_root: props.missingPath?.root, error_message: props.errorMessage ? redactTelemetryMessage(props.errorMessage) : undefined, elapsed_ms: props.elapsedMs, ...renderOutputShapeEventProperties(props), diff --git a/packages/cli/src/telemetry/missingPathKind.ts b/packages/cli/src/telemetry/missingPathKind.ts index 281d0cc7b3..be7d12fc9a 100644 --- a/packages/cli/src/telemetry/missingPathKind.ts +++ b/packages/cli/src/telemetry/missingPathKind.ts @@ -1,3 +1,5 @@ +import { missingPathOf } from "@hyperframes/core/missing-path"; + const KINDS: ReadonlyArray<[RegExp, string]> = [ [/(^|[\\/])hfcache-/, "extract_cache"], [/(^|[\\/])__hyperframes_video_frames([\\/]|$)/, "video_frames"], @@ -11,24 +13,6 @@ const KINDS: ReadonlyArray<[RegExp, string]> = [ [/(^|[\\/])(hf-render-|work-[0-9a-f]{8}-)/, "work_dir"], ]; -const quotedPath = (text: string) => /'([^']+)'/.exec(text)?.[1] ?? ""; - -function errorChain(error: unknown): Error[] { - const chain: Error[] = []; - for (let seen = error; seen instanceof Error && chain.length < 5; seen = seen.cause) - chain.push(seen); - return chain; -} - -function missingPathOf(error: unknown): string | undefined { - const enoent = errorChain(error).find( - (seen) => (seen as NodeJS.ErrnoException).code === "ENOENT", - ); - if (enoent) return (enoent as NodeJS.ErrnoException).path ?? quotedPath(enoent.message); - const message = error instanceof Error ? error.message : String(error); - return /\bENOENT\b/.test(message) ? quotedPath(message) : undefined; -} - /** Which kind of file a render found missing (a fixed name, never the path), or undefined when nothing was missing. */ export function missingPathKind(error: unknown): string | undefined { const path = missingPathOf(error); diff --git a/packages/cli/src/telemetry/renderObservability.ts b/packages/cli/src/telemetry/renderObservability.ts index 55fb8ac615..f35dc7b590 100644 --- a/packages/cli/src/telemetry/renderObservability.ts +++ b/packages/cli/src/telemetry/renderObservability.ts @@ -84,6 +84,7 @@ export function renderObservabilityTelemetryPayload( observabilityExtractVfrPreflightCount: extraction?.vfrPreflightCount, observabilityExtractCacheHits: extraction?.cacheHits, observabilityExtractCacheMisses: extraction?.cacheMisses, + observabilityExtractCachePublishFailures: extraction?.cachePublishFailures, observabilityInitDurationMs: init?.initDurationMs, observabilityInitTweenCount: init?.tweenCount, observabilityInitElementCount: init?.elementCount, diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json index 9addcf3143..6b3077e9ca 100644 --- a/packages/core/package-subpaths.json +++ b/packages/core/package-subpaths.json @@ -110,6 +110,12 @@ "types": "./dist/envProxy.d.ts", "environments": ["bun", "node"] }, + "./missing-path": { + "source": "./src/missingPath.ts", + "runtime": "./dist/missingPath.js", + "types": "./dist/missingPath.d.ts", + "environments": ["bun", "node"] + }, "./rename-retry": { "source": "./src/renameWithRetry.ts", "runtime": "./dist/renameWithRetry.js", diff --git a/packages/core/package.json b/packages/core/package.json index b64c180f18..692d71587f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -122,6 +122,12 @@ "import": "./src/envProxy.ts", "types": "./src/envProxy.ts" }, + "./missing-path": { + "bun": "./src/missingPath.ts", + "node": "./dist/missingPath.js", + "import": "./src/missingPath.ts", + "types": "./src/missingPath.ts" + }, "./rename-retry": { "bun": "./src/renameWithRetry.ts", "node": "./dist/renameWithRetry.js", @@ -744,6 +750,10 @@ "import": "./dist/envProxy.js", "types": "./dist/envProxy.d.ts" }, + "./missing-path": { + "import": "./dist/missingPath.js", + "types": "./dist/missingPath.d.ts" + }, "./rename-retry": { "import": "./dist/renameWithRetry.js", "types": "./dist/renameWithRetry.d.ts" diff --git a/packages/core/src/missingPath.ts b/packages/core/src/missingPath.ts new file mode 100644 index 0000000000..342e48bd84 --- /dev/null +++ b/packages/core/src/missingPath.ts @@ -0,0 +1,25 @@ +const quotedPath = (text: string) => /'([^']+)'/.exec(text)?.[1] ?? ""; + +function errorChain(error: unknown): Error[] { + const chain: Error[] = []; + for (let seen = error; seen instanceof Error && chain.length < 5; seen = seen.cause) + chain.push(seen); + return chain; +} + +/** The path an ENOENT in `error` (or its causes) names, or undefined when nothing was missing. */ +export function missingPathOf(error: unknown): string | undefined { + const enoent = errorChain(error).find( + (seen) => (seen as NodeJS.ErrnoException).code === "ENOENT", + ); + if (enoent) return (enoent as NodeJS.ErrnoException).path ?? quotedPath(enoent.message); + const message = error instanceof Error ? error.message : String(error); + return /\bENOENT\b/.test(message) ? quotedPath(message) : undefined; +} + +export function errnoMissingPathOf(error: unknown): string | undefined { + const enoent = errorChain(error).find( + (seen) => (seen as NodeJS.ErrnoException).code === "ENOENT", + ) as NodeJS.ErrnoException | undefined; + return enoent?.path; +} diff --git a/packages/producer/src/index.ts b/packages/producer/src/index.ts index 1ee2942eaf..01adb60a34 100644 --- a/packages/producer/src/index.ts +++ b/packages/producer/src/index.ts @@ -45,6 +45,7 @@ export { createOwnedRenderDir, listAbandonedRenderDirs, } from "./services/render/renderDirOwner.js"; +export type { MissingPathFacts } from "./services/render/missingPath.js"; export { RENDER_REQUEST_VERSION, createRenderRequest, diff --git a/packages/producer/src/services/render/cleanup.ts b/packages/producer/src/services/render/cleanup.ts index d4bc228a96..7a2ef378e6 100644 --- a/packages/producer/src/services/render/cleanup.ts +++ b/packages/producer/src/services/render/cleanup.ts @@ -5,6 +5,7 @@ import { type SubTimelineWaitOutcome } from "@hyperframes/engine"; import type { HdrDiagnostics, RenderJob } from "../renderOrchestrator.js"; import { normalizeErrorMessage } from "../../utils/errorMessage.js"; import type { RenderObservabilitySummary } from "./observability.js"; +import type { MissingPathFacts } from "./missingPath.js"; /** * Build the `RenderJob.errorDetails` shape downstream consumers (SSE, @@ -18,6 +19,7 @@ export function buildRenderErrorDetails(input: { hdrDiagnostics: HdrDiagnostics; observability?: RenderObservabilitySummary; subTimelineWait?: SubTimelineWaitOutcome; + missingPath?: MissingPathFacts; }): NonNullable { const errorMessage = normalizeErrorMessage(input.error); const errorStack = input.error instanceof Error ? input.error.stack : undefined; @@ -36,5 +38,6 @@ export function buildRenderErrorDetails(input: { : undefined, observability: input.observability, subTimelineWait: input.subTimelineWait, + missingPath: input.missingPath, }; } diff --git a/packages/producer/src/services/render/missingPath.test.ts b/packages/producer/src/services/render/missingPath.test.ts new file mode 100644 index 0000000000..bccddcbfd1 --- /dev/null +++ b/packages/producer/src/services/render/missingPath.test.ts @@ -0,0 +1,217 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import { describeMissingPath } from "./missingPath.js"; + +const touched = vi.hoisted(() => [] as string[]); +vi.mock("node:fs", async (importOriginal) => { + const real = await importOriginal(); + const recorded = + (call: (path: string, ...rest: A) => R) => + (path: string, ...rest: A) => { + touched.push(String(path)); + return call(path, ...rest); + }; + return { + ...real, + existsSync: recorded(real.existsSync), + lstatSync: recorded(real.lstatSync as (path: string) => ReturnType), + readlinkSync: recorded(real.readlinkSync as (path: string) => string), + }; +}); + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +/** A temp folder laid out like a render's: the extract cache with one entry, and a work dir whose compiled frames link + * into that entry. */ +function layout() { + const temp = mkdtempSync(join(tmpdir(), "hf-missing-")); + roots.push(temp); + const cacheRoot = join(temp, "hyperframes-extract-cache-u"); + const entry = join(cacheRoot, "hfcache-v10-abc"); + const work = join(temp, "hf-render-xyz"); + const link = join(work, "compiled", "__hyperframes_video_frames", "v1"); + mkdirSync(entry, { recursive: true }); + writeFileSync(join(entry, "frame_00001.jpg"), "x"); + mkdirSync(join(work, "compiled", "__hyperframes_video_frames"), { recursive: true }); + symlinkSync(entry, link, process.platform === "win32" ? "junction" : "dir"); + return { temp, cacheRoot, entry, work, linkedFrame: join(link, "frame_00002.jpg") }; +} + +const enoent = (path: string) => + Object.assign(new Error(`ENOENT: no such file or directory, open '${path}'`), { + code: "ENOENT", + path, + }); + +describe("describeMissingPath", () => { + it("says only the file is gone when its folder is there", () => { + const { temp, linkedFrame } = layout(); + expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })).toEqual({ + scope: "file", + synced: false, + root: "system_temp", + }); + }); + + it("reads through a link to the cache entry that went", () => { + const { temp, entry, linkedFrame } = layout(); + rmSync(entry, { recursive: true, force: true }); + expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("cache_entry"); + }); + + it("names an unfinished cache entry that went", () => { + const { temp, cacheRoot } = layout(); + const partial = join(cacheRoot, "hfcache-v10-def.partial-123"); + expect( + describeMissingPath(enoent(join(partial, "frame_00001.jpg")), { tempRoot: temp })?.scope, + ).toBe("cache_partial"); + }); + + it("names the whole cache, the work folder and the temp folder when those went", () => { + const { temp, cacheRoot, work, entry, linkedFrame } = layout(); + rmSync(cacheRoot, { recursive: true, force: true }); + expect( + describeMissingPath(enoent(join(entry, "frame_00001.jpg")), { tempRoot: temp })?.scope, + ).toBe("cache_root"); + rmSync(work, { recursive: true, force: true }); + expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("work_dir"); + rmSync(temp, { recursive: true, force: true }); + expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("temp_root"); + }); + + it("says whether a sync app owns the path and which root it is under", () => { + const synced = "C:\\Users\\a\\OneDrive - Contoso\\renders\\.frames\\frame_00001.jpg"; + expect(describeMissingPath(enoent(synced), { tempRoot: "C:\\Temp" })).toMatchObject({ + synced: true, + root: "other", + }); + const cacheDir = mkdtempSync(join(tmpdir(), "hf-cache-dir-")); + roots.push(cacheDir); + expect( + describeMissingPath(enoent(join(cacheDir, "hfcache-v10-a", "frame_00001.jpg")), { + tempRoot: join(tmpdir(), "not-here"), + cacheDir, + })?.root, + ).toBe("cache_dir"); + }); + + it("names a removed folder by its own name and the custom cache folder as the cache", () => { + const { temp, entry } = layout(); + rmSync(entry, { recursive: true, force: true }); + expect(describeMissingPath(enoent(entry), { tempRoot: temp })?.scope).toBe("cache_entry"); + const cacheDir = join(temp, "my-cache"); + expect( + describeMissingPath(enoent(join(cacheDir, "x.jpg")), { tempRoot: temp, cacheDir })?.scope, + ).toBe("cache_root"); + expect( + describeMissingPath(enoent(join(temp, "gone", "x.jpg")), { tempRoot: temp })?.scope, + ).toBe("other"); + }); + + it("does not read through a link that points outside the temp root and cache folder", () => { + const { temp, work } = layout(); + const outside = mkdtempSync(join(tmpdir(), "hf-outside-")); + roots.push(outside); + const link = join(work, "out"); + symlinkSync( + join(outside, "hfcache-v10-x", "dir"), + link, + process.platform === "win32" ? "junction" : "dir", + ); + expect(describeMissingPath(enoent(join(link, "x.jpg")), { tempRoot: temp })?.scope).toBe( + "link_target", + ); + }); + + it("stops on a link that points into itself", () => { + const { temp, work } = layout(); + const loop = join(work, "loop"); + symlinkSync(join(loop, "x"), loop, process.platform === "win32" ? "junction" : "dir"); + expect( + describeMissingPath(enoent(join(loop, "x", "frame.jpg")), { tempRoot: temp })?.scope, + ).toBe("link_target"); + }); + + it("says nothing when nothing was missing or a program could not start", () => { + expect(describeMissingPath(new Error("Target closed"))).toBeUndefined(); + const spawn = Object.assign(enoent("ffmpeg"), { syscall: "spawn ffmpeg" }); + expect(describeMissingPath(spawn)).toBeUndefined(); + }); + + it("never looks at a path that only the error's text names, as page-written text can", () => { + const { temp, linkedFrame } = layout(); + const written = new Error(`Composition failed to start: ENOENT, open '${linkedFrame}'`); + expect(describeMissingPath(written, { tempRoot: temp })).toBeUndefined(); + }); + + it("names a path outside the render's folders from its text, so it cannot tell whether one exists", () => { + const { temp } = layout(); + const outside = mkdtempSync(join(tmpdir(), "hf-outside-")); + roots.push(outside); + const there = describeMissingPath(enoent(join(outside, "x.jpg")), { tempRoot: temp }); + const gone = describeMissingPath(enoent(join(outside, "gone", "x.jpg")), { tempRoot: temp }); + expect(there).toEqual({ scope: "other", synced: false, root: "other" }); + expect(gone).toEqual(there); + expect( + describeMissingPath(enoent("//attacker.example/share/x.jpg"), { tempRoot: temp }), + ).toEqual(there); + }); + + /** The error Node raises reading `path`. */ + const readError = (path: string) => { + try { + readFileSync(path); + } catch (error) { + return error; + } + throw new Error(`${path} exists`); + }; + const under = (path: string, folder: string) => !relative(folder, path).startsWith(".."); + + it("never looks past a link out of the render's folders, so what is beyond cannot change the facts", () => { + const { temp, work } = layout(); + const outside = mkdtempSync(join(tmpdir(), "hf-outside-")); + roots.push(outside); + mkdirSync(join(outside, "present")); + symlinkSync(outside, join(work, "out"), process.platform === "win32" ? "junction" : "dir"); + touched.length = 0; + const present = describeMissingPath(readError(join(work, "out", "present", "x.jpg")), { + tempRoot: temp, + }); + const absent = describeMissingPath(readError(join(work, "out", "absent", "x.jpg")), { + tempRoot: temp, + }); + expect(present?.scope).toBe("link_target"); + expect(absent).toEqual(present); + expect(touched.filter((path) => path.startsWith(outside) || !under(path, temp))).toEqual([]); + }); + + it("names a missing work folder outside the temp root without looking above it", () => { + const parent = mkdtempSync(join(tmpdir(), "hf-work-parent-")); + roots.push(parent); + const workDir = join(parent, "work"); + const where = { tempRoot: join(tmpdir(), "not-here"), workDir }; + const error = readError(join(workDir, "frame.jpg")); + touched.length = 0; + expect(describeMissingPath(error, where)?.scope).toBe("work_dir"); + rmSync(parent, { recursive: true, force: true }); + expect(describeMissingPath(error, where)?.scope).toBe("work_dir"); + expect(touched.filter((path) => !under(path, workDir))).toEqual([]); + }); + + it("looks inside the render's work folder when it is outside the temp root", () => { + const work = mkdtempSync(join(tmpdir(), "hf-work-elsewhere-")); + roots.push(work); + expect( + describeMissingPath(enoent(join(work, "x.jpg")), { + tempRoot: join(tmpdir(), "not-here"), + workDir: work, + })?.scope, + ).toBe("file"); + }); +}); diff --git a/packages/producer/src/services/render/missingPath.ts b/packages/producer/src/services/render/missingPath.ts new file mode 100644 index 0000000000..127fd62a7f --- /dev/null +++ b/packages/producer/src/services/render/missingPath.ts @@ -0,0 +1,143 @@ +import { existsSync, lstatSync, readlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, relative, resolve, isAbsolute, sep } from "node:path"; +import { errnoMissingPathOf } from "@hyperframes/core/missing-path"; + +/** What was gone around a missing path when the render failed, from closed lists only: never a path or a name. */ +export interface MissingPathFacts { + /** The highest thing gone on the path; a link on the way is read through to its target. */ + scope: + | "file" + | "link_target" + | "cache_entry" + | "cache_partial" + | "cache_root" + | "work_dir" + | "temp_root" + | "other"; + /** Under a folder a sync app owns (OneDrive, Dropbox, iCloud, Google Drive). */ + synced: boolean; + root: "system_temp" | "cache_dir" | "other"; +} + +const SYNCED = /onedrive|dropbox|icloud|mobile documents|clouddocs|google ?drive|cloudstorage/i; + +interface Where { + tempRoot: string; + cacheDir?: string; + workDir?: string; +} + +const inside = (path: string, folder: string) => { + const rel = relative(resolve(folder), resolve(path)); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +}; + +function lstatOrNull(path: string) { + try { + return lstatSync(path); + } catch { + // expected: what is gone has nothing to stat + return null; + } +} + +const ROOT_SCOPES = [ + ["tempRoot", "temp_root"], + ["cacheDir", "cache_root"], + ["workDir", "work_dir"], +] as const; + +const NAME_SCOPES: ReadonlyArray<[RegExp, MissingPathFacts["scope"]]> = [ + [/\.partial/, "cache_partial"], + [/^hfcache-/, "cache_entry"], + [/^hyperframes-extract-cache/, "cache_root"], + [/^(hf-render-|work-[0-9a-f]{8}-)/, "work_dir"], +]; + +function scopeOfGone(path: string, where: Where, throughLink: boolean): MissingPathFacts["scope"] { + const root = ROOT_SCOPES.find(([key]) => { + const dir = where[key]; + return !!dir && resolve(path) === resolve(dir); + }); + if (root) return root[1]; + const name = basename(path); + return ( + NAME_SCOPES.find(([pattern]) => pattern.test(name))?.[1] ?? + (throughLink ? "link_target" : "other") + ); +} + +const MAX_LINK_HOPS = 4; + +function rootOf(path: string, where: Where): MissingPathFacts["root"] { + if (inside(path, where.tempRoot)) return "system_temp"; + if (where.cacheDir && inside(path, where.cacheDir)) return "cache_dir"; + return "other"; +} + +const ownerOf = (path: string, where: Where) => + [where.tempRoot, where.cacheDir, where.workDir] + .filter((root): root is string => !!root && inside(path, root)) + .sort((a, b) => resolve(a).length - resolve(b).length)[0]; + +const linkTarget = (link: string, folder: string) => + resolve(folder, readlinkSync(link).replace(/^\\\\\?\\/, "")); + +function goneAt( + path: string, + last: boolean, + where: Where, + hops: number, +): MissingPathFacts["scope"] { + const scope = scopeOfGone(path, where, hops > 0); + return last && scope === "other" ? "file" : scope; +} + +/** Walks down from the render's own folder with lstat, checking each link's target before going through it. */ +function highestGone(path: string, where: Where, hops = 0): MissingPathFacts["scope"] | undefined { + const root = ownerOf(path, where); + if (!root || hops > MAX_LINK_HOPS) return "link_target"; + if (!existsSync(root)) return scopeOfGone(root, where, hops > 0); + const parts = relative(resolve(root), resolve(path)).split(sep).filter(Boolean); + let at = resolve(root); + for (const [index, part] of parts.entries()) { + const next = join(at, part); + const stat = lstatOrNull(next); + if (!stat) return goneAt(next, index === parts.length - 1, where, hops); + at = stat.isSymbolicLink() ? linkTarget(next, at) : next; + const gone = at === next ? undefined : highestGone(at, where, hops + 1); + if (gone) return gone; + } + return undefined; +} + +const isSpawnFailure = (error: unknown) => + !!(error as NodeJS.ErrnoException | undefined)?.syscall?.startsWith("spawn"); + +/** What was gone around the path a failed render's ENOENT names; undefined when it names none or the look fails. + * Only the render's own folders are looked at, never a share; call it before the render's own cleanup runs. */ +export function describeMissingPath( + error: unknown, + { + cacheDir, + workDir, + tempRoot = tmpdir(), + }: { cacheDir?: string; workDir?: string; tempRoot?: string } = {}, +): MissingPathFacts | undefined { + try { + const path = errnoMissingPathOf(error); + if (!path || isSpawnFailure(error)) return undefined; + const where = { tempRoot, cacheDir, workDir }; + return { + scope: ownerOf(path, where) + ? (highestGone(path, where) ?? "other") + : scopeOfGone(path, where, false), + synced: SYNCED.test(path), + root: rootOf(path, where), + }; + } catch { + // expected: the folders can change under the look; the render's own error must still be what is reported + return undefined; + } +} diff --git a/packages/producer/src/services/render/observability.ts b/packages/producer/src/services/render/observability.ts index 363dee4a7d..b7ea7d4f3e 100644 --- a/packages/producer/src/services/render/observability.ts +++ b/packages/producer/src/services/render/observability.ts @@ -224,6 +224,7 @@ export interface RenderExtractionObservability { vfrPreflightCount?: number; cacheHits?: number; cacheMisses?: number; + cachePublishFailures?: number; /** Per-source transient download/metadata/FFmpeg retries performed during extraction. */ transientRetries?: number; /** diff --git a/packages/producer/src/services/renderOrchestrator.missingPath.test.ts b/packages/producer/src/services/renderOrchestrator.missingPath.test.ts new file mode 100644 index 0000000000..dfcbd56738 --- /dev/null +++ b/packages/producer/src/services/renderOrchestrator.missingPath.test.ts @@ -0,0 +1,140 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveConfig } from "@hyperframes/engine"; +import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js"; + +// Entry: executeRenderJob. Capture setup fails on a frame whose cache entry another hand removed; the failure must +// say what was gone, read before the render removes its own folders. +const trace = vi.hoisted(() => ({ missing: "", probeThrows: false })); + +vi.mock("@hyperframes/core/missing-path", async (importOriginal) => { + const real = await importOriginal(); + return { + errnoMissingPathOf: (error: unknown) => { + if (trace.probeThrows) throw new RangeError("Maximum call stack size exceeded"); + return real.errnoMissingPathOf(error); + }, + }; +}); + +vi.mock("./render/shared.js", async (importOriginal) => ({ + ...(await importOriginal()), + createMemorySampler: () => ({ + peakRssBytes: () => 0, + peakHeapUsedBytes: () => 0, + stop: () => {}, + }), +})); +vi.mock("./fileServer.js", async (importOriginal) => ({ + ...(await importOriginal()), + createFileServer: async () => ({ + url: "http://127.0.0.1:1", + port: 1, + close: () => {}, + addPreHeadScript: () => {}, + }), +})); +vi.mock("./render/stages/compileStage.js", () => ({ + runCompileStage: async () => ({ + compiled: { + html: "
", + hasShaderTransitions: false, + renderModeHints: { reasons: [] }, + }, + composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] }, + deviceScaleFactor: 1, + outputWidth: 16, + outputHeight: 16, + compileOnlyMs: 0, + forceScreenshot: true, + }), +})); +vi.mock("./render/stages/probeStage.js", () => ({ + runProbeStage: async ({ compiled }: { compiled: object }) => ({ + compiled, + fileServer: null, + probeSession: null, + lastBrowserConsole: [], + duration: 40 / 30, + totalFrames: 40, + browserProbeMs: 0, + beginFrameStalled: false, + }), +})); +vi.mock("./render/stages/extractVideosStage.js", () => ({ + runExtractVideosStage: async () => ({ + extractionResult: null, + frameLookup: null, + videoReadinessSkipIds: [], + videoMetadataHints: [], + nativeHdrVideoIds: new Set(), + videoTransfers: new Map(), + nativeHdrImageIds: new Set(), + imageTransfers: new Map(), + hdrImageSrcPaths: new Map(), + imageColorSpaces: [], + videoExtractMs: 0, + failureToEnforce: null, + }), +})); +vi.mock("./render/stages/audioStage.js", () => ({ + runAudioStage: async ({ workDir }: { workDir: string }) => ({ + audioOutputPath: join(workDir, "audio.wav"), + hasAudio: false, + audioProcessMs: 0, + }), +})); +vi.mock("./render/captureCost.js", async (importOriginal) => ({ + ...(await importOriginal()), + resolveRenderWorkerCount: () => { + throw Object.assign(new Error(`ENOENT: no such file or directory, open '${trace.missing}'`), { + code: "ENOENT", + path: trace.missing, + }); + }, +})); + +const dirs: string[] = []; +afterEach(() => { + trace.probeThrows = false; + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function failingRender() { + const root = mkdtempSync(join(tmpdir(), "hf-missing-render-")); + dirs.push(root); + writeFileSync(join(root, "index.html"), "
"); + const cacheRoot = join(root, "hyperframes-extract-cache-u"); + mkdirSync(cacheRoot); + trace.missing = join(cacheRoot, "hfcache-v10-gone", "frame_00001.jpg"); + const job = createRenderJob({ + fps: 30, + quality: "draft", + format: "mp4", + hdrMode: "force-sdr", + producerConfig: resolveConfig({ enablePageSideCompositing: false }), + logger: { info() {}, warn() {}, error() {}, debug() {} }, + }); + return { job, run: () => executeRenderJob(job, root, join(root, "out.mp4")) }; +} + +it("a render that fails on a missing frame says the cache entry holding it was gone", async () => { + const { job, run } = failingRender(); + await expect(run()).rejects.toThrow(/ENOENT/); + expect(job.errorDetails?.missingPath).toEqual({ + scope: "cache_entry", + synced: false, + root: "system_temp", + }); +}); + +it("a missing-path look that throws leaves the render's own failure as it was", async () => { + trace.probeThrows = true; + const { job, run } = failingRender(); + await expect(run()).rejects.toThrow(/ENOENT/); + expect(job.status).toBe("failed"); + expect(job.error).toMatch(/ENOENT/); + expect(job.errorDetails?.missingPath).toBeUndefined(); +}); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index b1b19a3743..4a3e911ceb 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -169,6 +169,7 @@ import { } from "./render/shared.js"; import { frameCacheOptions } from "./render/frameCacheBudget.js"; import { buildRenderErrorDetails } from "./render/cleanup.js"; +import { describeMissingPath, type MissingPathFacts } from "./render/missingPath.js"; import { publishRenderFailure } from "./render/renderEventPublisher.js"; import { EncoderInterruptedError } from "./render/encoderInterruption.js"; import { RenderExecutionContext } from "./render/renderExecutionContext.js"; @@ -347,6 +348,7 @@ function summarizeExtractionObservability( vfrPreflightCount: phaseBreakdown?.vfrPreflightCount, cacheHits: phaseBreakdown?.cacheHits, cacheMisses: phaseBreakdown?.cacheMisses, + cachePublishFailures: phaseBreakdown?.cachePublishFailures, transientRetries: phaseBreakdown?.transientRetries, ...coverageGauges, authoredTimedClipCount, @@ -840,6 +842,7 @@ export interface RenderJob extends Partial { observability?: RenderObservabilitySummary; /** Worst sub-composition timeline wait outcome across sessions captured before the failure. */ subTimelineWait?: SubTimelineWaitOutcome; + missingPath?: MissingPathFacts; }; } @@ -6033,6 +6036,7 @@ async function executeRenderPipeline(input: { hdrDiagnostics, observability: observabilitySummary, subTimelineWait: worstSubTimelineWaitOutcome(dedupPerfs), + missingPath: describeMissingPath(error, { cacheDir: cfg.extractCacheDir, workDir }), }); publishRenderFailure( job, diff --git a/scripts/check-no-main-deletions.mjs b/scripts/check-no-main-deletions.mjs index 177254be7e..3039b109d7 100644 --- a/scripts/check-no-main-deletions.mjs +++ b/scripts/check-no-main-deletions.mjs @@ -2147,15 +2147,6 @@ export const ALLOWED_DELETIONS = new Map([ "packages/engine/src/services/frameCapture-pageNetwork.test.ts", "pinned the removed per-page filter; browserManager.test.ts pins when a render browser goes behind the proxy", ], - ...[ - "packages/core/src/missingPath.ts", - "packages/producer/src/services/render/missingPath.ts", - "packages/producer/src/services/render/missingPath.test.ts", - "packages/producer/src/services/renderOrchestrator.missingPath.test.ts", - ].map((path) => [ - path, - "the revert of #7955 removes the missing-path detail; it returns looking only inside folders the render owns", - ]), ]); export function parseBase(argv, fallback = "origin/main") { diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index e262caa0eb..bd9e71eab8 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -36,7 +36,7 @@ "unpacked": 554000 }, "@hyperframes/producer": { - "packed": 936000, + "packed": 993551, "unpacked": 3491355 }, "@hyperframes/sdk": {