mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
feat(producer): a failed render says which file was gone again, looking only in its own folders (#8168)
* feat(producer): a render that fails on a missing file says what was gone again Restores the diagnostic the previous commit on main took out, as it was; the commits after it confine what it looks at. * test(producer): render failure diagnostics never probe a path from error text or outside the render * fix(producer): probe a missing path only from a real error, inside the render's folders describeMissingPath now reads only the path Node set on an ENOENT error, never one quoted in a message the page can write, and looks on disk only under the render's temp root, extract cache or work folder. Any other path is named from its text, so a network share is never contacted and the facts cannot tell an existing path from a missing one. * test(producer): a missing-path probe never passes an outside link or climbs above its folder * fix(producer): walk down from the render's own folder and check each link before following it * build(producer): give the packed size budget its 5% headroom with the missing-file diagnostic back
This commit is contained in:
@@ -2010,6 +2010,7 @@ function handleRenderError(
|
||||
errorName: error instanceof Error ? error.name : "unknown",
|
||||
failedStageCode: normalizeStageCode(failedStage || "pipeline"),
|
||||
missingPathKind: missingPathKind(error),
|
||||
missingPath: job?.errorDetails?.missingPath,
|
||||
...renderJobObservabilityTelemetryPayload(job),
|
||||
...getMemorySnapshot(),
|
||||
...cpu.stop(),
|
||||
|
||||
@@ -690,6 +690,24 @@ describe("render telemetry events", () => {
|
||||
expect(props.hf_env_overrides).toEqual(["HF_DE_VERIFY", "HYPERFRAMES_FONT_CACHE_DIR"]);
|
||||
});
|
||||
|
||||
it("sends only the closed-list missing-path facts on render_error", () => {
|
||||
trackRenderError({
|
||||
fps: 30,
|
||||
quality: "draft",
|
||||
docker: false,
|
||||
missingPath: { scope: "cache_entry", synced: true, root: "system_temp" },
|
||||
});
|
||||
const props = trackEvent.mock.calls[0]?.[1] as Record<string, unknown>;
|
||||
expect(props).toMatchObject({
|
||||
missing_path_scope: "cache_entry",
|
||||
missing_path_synced: true,
|
||||
missing_path_root: "system_temp",
|
||||
});
|
||||
expect(Object.values(props).filter((v) => typeof v === "string" && /[\\/]/.test(v))).toEqual(
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an empty array, not an absent field, when no override was resolved", () => {
|
||||
trackRenderError({ fps: 30, quality: "draft", docker: false });
|
||||
const props = trackEvent.mock.calls[0]?.[1] as Record<string, unknown>;
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
renderMotionBlurEventProperties,
|
||||
} from "./renderMotionBlur.js";
|
||||
import { CliRuntimeError } from "../utils/commandResult.js";
|
||||
import type { MissingPathFacts } from "@hyperframes/producer";
|
||||
|
||||
// Power state is volatile (a laptop docks/undocks mid-session), so it is
|
||||
// sampled per render event rather than cached with SystemMeta. Attached to
|
||||
@@ -127,6 +128,7 @@ export interface RenderObservabilityTelemetryPayload {
|
||||
observabilityExtractVfrPreflightCount?: number;
|
||||
observabilityExtractCacheHits?: number;
|
||||
observabilityExtractCacheMisses?: number;
|
||||
observabilityExtractCachePublishFailures?: number;
|
||||
observabilityInitDurationMs?: number;
|
||||
observabilityInitTweenCount?: number;
|
||||
observabilityInitElementCount?: number;
|
||||
@@ -207,6 +209,7 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP
|
||||
observability_extract_vfr_preflight_count: props.observabilityExtractVfrPreflightCount,
|
||||
observability_extract_cache_hits: props.observabilityExtractCacheHits,
|
||||
observability_extract_cache_misses: props.observabilityExtractCacheMisses,
|
||||
observability_extract_cache_publish_failures: props.observabilityExtractCachePublishFailures,
|
||||
observability_init_duration_ms: props.observabilityInitDurationMs,
|
||||
observability_init_tween_count: props.observabilityInitTweenCount,
|
||||
observability_init_element_count: props.observabilityInitElementCount,
|
||||
@@ -702,6 +705,7 @@ export function trackRenderError(
|
||||
/** failedStage normalized to a stable snake_case code. */
|
||||
failedStageCode?: string;
|
||||
missingPathKind?: string;
|
||||
missingPath?: MissingPathFacts;
|
||||
errorMessage?: string;
|
||||
elapsedMs?: number;
|
||||
peakMemoryMb?: number;
|
||||
@@ -731,6 +735,9 @@ export function trackRenderError(
|
||||
error_name: props.errorName,
|
||||
failed_stage_code: props.failedStageCode,
|
||||
missing_path_kind: props.missingPathKind,
|
||||
missing_path_scope: props.missingPath?.scope,
|
||||
missing_path_synced: props.missingPath?.synced,
|
||||
missing_path_root: props.missingPath?.root,
|
||||
error_message: props.errorMessage ? redactTelemetryMessage(props.errorMessage) : undefined,
|
||||
elapsed_ms: props.elapsedMs,
|
||||
...renderOutputShapeEventProperties(props),
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { missingPathOf } from "@hyperframes/core/missing-path";
|
||||
|
||||
const KINDS: ReadonlyArray<[RegExp, string]> = [
|
||||
[/(^|[\\/])hfcache-/, "extract_cache"],
|
||||
[/(^|[\\/])__hyperframes_video_frames([\\/]|$)/, "video_frames"],
|
||||
@@ -11,24 +13,6 @@ const KINDS: ReadonlyArray<[RegExp, string]> = [
|
||||
[/(^|[\\/])(hf-render-|work-[0-9a-f]{8}-)/, "work_dir"],
|
||||
];
|
||||
|
||||
const quotedPath = (text: string) => /'([^']+)'/.exec(text)?.[1] ?? "";
|
||||
|
||||
function errorChain(error: unknown): Error[] {
|
||||
const chain: Error[] = [];
|
||||
for (let seen = error; seen instanceof Error && chain.length < 5; seen = seen.cause)
|
||||
chain.push(seen);
|
||||
return chain;
|
||||
}
|
||||
|
||||
function missingPathOf(error: unknown): string | undefined {
|
||||
const enoent = errorChain(error).find(
|
||||
(seen) => (seen as NodeJS.ErrnoException).code === "ENOENT",
|
||||
);
|
||||
if (enoent) return (enoent as NodeJS.ErrnoException).path ?? quotedPath(enoent.message);
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return /\bENOENT\b/.test(message) ? quotedPath(message) : undefined;
|
||||
}
|
||||
|
||||
/** Which kind of file a render found missing (a fixed name, never the path), or undefined when nothing was missing. */
|
||||
export function missingPathKind(error: unknown): string | undefined {
|
||||
const path = missingPathOf(error);
|
||||
|
||||
@@ -84,6 +84,7 @@ export function renderObservabilityTelemetryPayload(
|
||||
observabilityExtractVfrPreflightCount: extraction?.vfrPreflightCount,
|
||||
observabilityExtractCacheHits: extraction?.cacheHits,
|
||||
observabilityExtractCacheMisses: extraction?.cacheMisses,
|
||||
observabilityExtractCachePublishFailures: extraction?.cachePublishFailures,
|
||||
observabilityInitDurationMs: init?.initDurationMs,
|
||||
observabilityInitTweenCount: init?.tweenCount,
|
||||
observabilityInitElementCount: init?.elementCount,
|
||||
|
||||
@@ -110,6 +110,12 @@
|
||||
"types": "./dist/envProxy.d.ts",
|
||||
"environments": ["bun", "node"]
|
||||
},
|
||||
"./missing-path": {
|
||||
"source": "./src/missingPath.ts",
|
||||
"runtime": "./dist/missingPath.js",
|
||||
"types": "./dist/missingPath.d.ts",
|
||||
"environments": ["bun", "node"]
|
||||
},
|
||||
"./rename-retry": {
|
||||
"source": "./src/renameWithRetry.ts",
|
||||
"runtime": "./dist/renameWithRetry.js",
|
||||
|
||||
@@ -122,6 +122,12 @@
|
||||
"import": "./src/envProxy.ts",
|
||||
"types": "./src/envProxy.ts"
|
||||
},
|
||||
"./missing-path": {
|
||||
"bun": "./src/missingPath.ts",
|
||||
"node": "./dist/missingPath.js",
|
||||
"import": "./src/missingPath.ts",
|
||||
"types": "./src/missingPath.ts"
|
||||
},
|
||||
"./rename-retry": {
|
||||
"bun": "./src/renameWithRetry.ts",
|
||||
"node": "./dist/renameWithRetry.js",
|
||||
@@ -744,6 +750,10 @@
|
||||
"import": "./dist/envProxy.js",
|
||||
"types": "./dist/envProxy.d.ts"
|
||||
},
|
||||
"./missing-path": {
|
||||
"import": "./dist/missingPath.js",
|
||||
"types": "./dist/missingPath.d.ts"
|
||||
},
|
||||
"./rename-retry": {
|
||||
"import": "./dist/renameWithRetry.js",
|
||||
"types": "./dist/renameWithRetry.d.ts"
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
const quotedPath = (text: string) => /'([^']+)'/.exec(text)?.[1] ?? "";
|
||||
|
||||
function errorChain(error: unknown): Error[] {
|
||||
const chain: Error[] = [];
|
||||
for (let seen = error; seen instanceof Error && chain.length < 5; seen = seen.cause)
|
||||
chain.push(seen);
|
||||
return chain;
|
||||
}
|
||||
|
||||
/** The path an ENOENT in `error` (or its causes) names, or undefined when nothing was missing. */
|
||||
export function missingPathOf(error: unknown): string | undefined {
|
||||
const enoent = errorChain(error).find(
|
||||
(seen) => (seen as NodeJS.ErrnoException).code === "ENOENT",
|
||||
);
|
||||
if (enoent) return (enoent as NodeJS.ErrnoException).path ?? quotedPath(enoent.message);
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return /\bENOENT\b/.test(message) ? quotedPath(message) : undefined;
|
||||
}
|
||||
|
||||
export function errnoMissingPathOf(error: unknown): string | undefined {
|
||||
const enoent = errorChain(error).find(
|
||||
(seen) => (seen as NodeJS.ErrnoException).code === "ENOENT",
|
||||
) as NodeJS.ErrnoException | undefined;
|
||||
return enoent?.path;
|
||||
}
|
||||
@@ -45,6 +45,7 @@ export {
|
||||
createOwnedRenderDir,
|
||||
listAbandonedRenderDirs,
|
||||
} from "./services/render/renderDirOwner.js";
|
||||
export type { MissingPathFacts } from "./services/render/missingPath.js";
|
||||
export {
|
||||
RENDER_REQUEST_VERSION,
|
||||
createRenderRequest,
|
||||
|
||||
@@ -5,6 +5,7 @@ import { type SubTimelineWaitOutcome } from "@hyperframes/engine";
|
||||
import type { HdrDiagnostics, RenderJob } from "../renderOrchestrator.js";
|
||||
import { normalizeErrorMessage } from "../../utils/errorMessage.js";
|
||||
import type { RenderObservabilitySummary } from "./observability.js";
|
||||
import type { MissingPathFacts } from "./missingPath.js";
|
||||
|
||||
/**
|
||||
* Build the `RenderJob.errorDetails` shape downstream consumers (SSE,
|
||||
@@ -18,6 +19,7 @@ export function buildRenderErrorDetails(input: {
|
||||
hdrDiagnostics: HdrDiagnostics;
|
||||
observability?: RenderObservabilitySummary;
|
||||
subTimelineWait?: SubTimelineWaitOutcome;
|
||||
missingPath?: MissingPathFacts;
|
||||
}): NonNullable<RenderJob["errorDetails"]> {
|
||||
const errorMessage = normalizeErrorMessage(input.error);
|
||||
const errorStack = input.error instanceof Error ? input.error.stack : undefined;
|
||||
@@ -36,5 +38,6 @@ export function buildRenderErrorDetails(input: {
|
||||
: undefined,
|
||||
observability: input.observability,
|
||||
subTimelineWait: input.subTimelineWait,
|
||||
missingPath: input.missingPath,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, relative } from "node:path";
|
||||
import { describeMissingPath } from "./missingPath.js";
|
||||
|
||||
const touched = vi.hoisted(() => [] as string[]);
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const real = await importOriginal<typeof import("node:fs")>();
|
||||
const recorded =
|
||||
<A extends unknown[], R>(call: (path: string, ...rest: A) => R) =>
|
||||
(path: string, ...rest: A) => {
|
||||
touched.push(String(path));
|
||||
return call(path, ...rest);
|
||||
};
|
||||
return {
|
||||
...real,
|
||||
existsSync: recorded(real.existsSync),
|
||||
lstatSync: recorded(real.lstatSync as (path: string) => ReturnType<typeof real.lstatSync>),
|
||||
readlinkSync: recorded(real.readlinkSync as (path: string) => string),
|
||||
};
|
||||
});
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** A temp folder laid out like a render's: the extract cache with one entry, and a work dir whose compiled frames link
|
||||
* into that entry. */
|
||||
function layout() {
|
||||
const temp = mkdtempSync(join(tmpdir(), "hf-missing-"));
|
||||
roots.push(temp);
|
||||
const cacheRoot = join(temp, "hyperframes-extract-cache-u");
|
||||
const entry = join(cacheRoot, "hfcache-v10-abc");
|
||||
const work = join(temp, "hf-render-xyz");
|
||||
const link = join(work, "compiled", "__hyperframes_video_frames", "v1");
|
||||
mkdirSync(entry, { recursive: true });
|
||||
writeFileSync(join(entry, "frame_00001.jpg"), "x");
|
||||
mkdirSync(join(work, "compiled", "__hyperframes_video_frames"), { recursive: true });
|
||||
symlinkSync(entry, link, process.platform === "win32" ? "junction" : "dir");
|
||||
return { temp, cacheRoot, entry, work, linkedFrame: join(link, "frame_00002.jpg") };
|
||||
}
|
||||
|
||||
const enoent = (path: string) =>
|
||||
Object.assign(new Error(`ENOENT: no such file or directory, open '${path}'`), {
|
||||
code: "ENOENT",
|
||||
path,
|
||||
});
|
||||
|
||||
describe("describeMissingPath", () => {
|
||||
it("says only the file is gone when its folder is there", () => {
|
||||
const { temp, linkedFrame } = layout();
|
||||
expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })).toEqual({
|
||||
scope: "file",
|
||||
synced: false,
|
||||
root: "system_temp",
|
||||
});
|
||||
});
|
||||
|
||||
it("reads through a link to the cache entry that went", () => {
|
||||
const { temp, entry, linkedFrame } = layout();
|
||||
rmSync(entry, { recursive: true, force: true });
|
||||
expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("cache_entry");
|
||||
});
|
||||
|
||||
it("names an unfinished cache entry that went", () => {
|
||||
const { temp, cacheRoot } = layout();
|
||||
const partial = join(cacheRoot, "hfcache-v10-def.partial-123");
|
||||
expect(
|
||||
describeMissingPath(enoent(join(partial, "frame_00001.jpg")), { tempRoot: temp })?.scope,
|
||||
).toBe("cache_partial");
|
||||
});
|
||||
|
||||
it("names the whole cache, the work folder and the temp folder when those went", () => {
|
||||
const { temp, cacheRoot, work, entry, linkedFrame } = layout();
|
||||
rmSync(cacheRoot, { recursive: true, force: true });
|
||||
expect(
|
||||
describeMissingPath(enoent(join(entry, "frame_00001.jpg")), { tempRoot: temp })?.scope,
|
||||
).toBe("cache_root");
|
||||
rmSync(work, { recursive: true, force: true });
|
||||
expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("work_dir");
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
expect(describeMissingPath(enoent(linkedFrame), { tempRoot: temp })?.scope).toBe("temp_root");
|
||||
});
|
||||
|
||||
it("says whether a sync app owns the path and which root it is under", () => {
|
||||
const synced = "C:\\Users\\a\\OneDrive - Contoso\\renders\\.frames\\frame_00001.jpg";
|
||||
expect(describeMissingPath(enoent(synced), { tempRoot: "C:\\Temp" })).toMatchObject({
|
||||
synced: true,
|
||||
root: "other",
|
||||
});
|
||||
const cacheDir = mkdtempSync(join(tmpdir(), "hf-cache-dir-"));
|
||||
roots.push(cacheDir);
|
||||
expect(
|
||||
describeMissingPath(enoent(join(cacheDir, "hfcache-v10-a", "frame_00001.jpg")), {
|
||||
tempRoot: join(tmpdir(), "not-here"),
|
||||
cacheDir,
|
||||
})?.root,
|
||||
).toBe("cache_dir");
|
||||
});
|
||||
|
||||
it("names a removed folder by its own name and the custom cache folder as the cache", () => {
|
||||
const { temp, entry } = layout();
|
||||
rmSync(entry, { recursive: true, force: true });
|
||||
expect(describeMissingPath(enoent(entry), { tempRoot: temp })?.scope).toBe("cache_entry");
|
||||
const cacheDir = join(temp, "my-cache");
|
||||
expect(
|
||||
describeMissingPath(enoent(join(cacheDir, "x.jpg")), { tempRoot: temp, cacheDir })?.scope,
|
||||
).toBe("cache_root");
|
||||
expect(
|
||||
describeMissingPath(enoent(join(temp, "gone", "x.jpg")), { tempRoot: temp })?.scope,
|
||||
).toBe("other");
|
||||
});
|
||||
|
||||
it("does not read through a link that points outside the temp root and cache folder", () => {
|
||||
const { temp, work } = layout();
|
||||
const outside = mkdtempSync(join(tmpdir(), "hf-outside-"));
|
||||
roots.push(outside);
|
||||
const link = join(work, "out");
|
||||
symlinkSync(
|
||||
join(outside, "hfcache-v10-x", "dir"),
|
||||
link,
|
||||
process.platform === "win32" ? "junction" : "dir",
|
||||
);
|
||||
expect(describeMissingPath(enoent(join(link, "x.jpg")), { tempRoot: temp })?.scope).toBe(
|
||||
"link_target",
|
||||
);
|
||||
});
|
||||
|
||||
it("stops on a link that points into itself", () => {
|
||||
const { temp, work } = layout();
|
||||
const loop = join(work, "loop");
|
||||
symlinkSync(join(loop, "x"), loop, process.platform === "win32" ? "junction" : "dir");
|
||||
expect(
|
||||
describeMissingPath(enoent(join(loop, "x", "frame.jpg")), { tempRoot: temp })?.scope,
|
||||
).toBe("link_target");
|
||||
});
|
||||
|
||||
it("says nothing when nothing was missing or a program could not start", () => {
|
||||
expect(describeMissingPath(new Error("Target closed"))).toBeUndefined();
|
||||
const spawn = Object.assign(enoent("ffmpeg"), { syscall: "spawn ffmpeg" });
|
||||
expect(describeMissingPath(spawn)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("never looks at a path that only the error's text names, as page-written text can", () => {
|
||||
const { temp, linkedFrame } = layout();
|
||||
const written = new Error(`Composition failed to start: ENOENT, open '${linkedFrame}'`);
|
||||
expect(describeMissingPath(written, { tempRoot: temp })).toBeUndefined();
|
||||
});
|
||||
|
||||
it("names a path outside the render's folders from its text, so it cannot tell whether one exists", () => {
|
||||
const { temp } = layout();
|
||||
const outside = mkdtempSync(join(tmpdir(), "hf-outside-"));
|
||||
roots.push(outside);
|
||||
const there = describeMissingPath(enoent(join(outside, "x.jpg")), { tempRoot: temp });
|
||||
const gone = describeMissingPath(enoent(join(outside, "gone", "x.jpg")), { tempRoot: temp });
|
||||
expect(there).toEqual({ scope: "other", synced: false, root: "other" });
|
||||
expect(gone).toEqual(there);
|
||||
expect(
|
||||
describeMissingPath(enoent("//attacker.example/share/x.jpg"), { tempRoot: temp }),
|
||||
).toEqual(there);
|
||||
});
|
||||
|
||||
/** The error Node raises reading `path`. */
|
||||
const readError = (path: string) => {
|
||||
try {
|
||||
readFileSync(path);
|
||||
} catch (error) {
|
||||
return error;
|
||||
}
|
||||
throw new Error(`${path} exists`);
|
||||
};
|
||||
const under = (path: string, folder: string) => !relative(folder, path).startsWith("..");
|
||||
|
||||
it("never looks past a link out of the render's folders, so what is beyond cannot change the facts", () => {
|
||||
const { temp, work } = layout();
|
||||
const outside = mkdtempSync(join(tmpdir(), "hf-outside-"));
|
||||
roots.push(outside);
|
||||
mkdirSync(join(outside, "present"));
|
||||
symlinkSync(outside, join(work, "out"), process.platform === "win32" ? "junction" : "dir");
|
||||
touched.length = 0;
|
||||
const present = describeMissingPath(readError(join(work, "out", "present", "x.jpg")), {
|
||||
tempRoot: temp,
|
||||
});
|
||||
const absent = describeMissingPath(readError(join(work, "out", "absent", "x.jpg")), {
|
||||
tempRoot: temp,
|
||||
});
|
||||
expect(present?.scope).toBe("link_target");
|
||||
expect(absent).toEqual(present);
|
||||
expect(touched.filter((path) => path.startsWith(outside) || !under(path, temp))).toEqual([]);
|
||||
});
|
||||
|
||||
it("names a missing work folder outside the temp root without looking above it", () => {
|
||||
const parent = mkdtempSync(join(tmpdir(), "hf-work-parent-"));
|
||||
roots.push(parent);
|
||||
const workDir = join(parent, "work");
|
||||
const where = { tempRoot: join(tmpdir(), "not-here"), workDir };
|
||||
const error = readError(join(workDir, "frame.jpg"));
|
||||
touched.length = 0;
|
||||
expect(describeMissingPath(error, where)?.scope).toBe("work_dir");
|
||||
rmSync(parent, { recursive: true, force: true });
|
||||
expect(describeMissingPath(error, where)?.scope).toBe("work_dir");
|
||||
expect(touched.filter((path) => !under(path, workDir))).toEqual([]);
|
||||
});
|
||||
|
||||
it("looks inside the render's work folder when it is outside the temp root", () => {
|
||||
const work = mkdtempSync(join(tmpdir(), "hf-work-elsewhere-"));
|
||||
roots.push(work);
|
||||
expect(
|
||||
describeMissingPath(enoent(join(work, "x.jpg")), {
|
||||
tempRoot: join(tmpdir(), "not-here"),
|
||||
workDir: work,
|
||||
})?.scope,
|
||||
).toBe("file");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
import { existsSync, lstatSync, readlinkSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, join, relative, resolve, isAbsolute, sep } from "node:path";
|
||||
import { errnoMissingPathOf } from "@hyperframes/core/missing-path";
|
||||
|
||||
/** What was gone around a missing path when the render failed, from closed lists only: never a path or a name. */
|
||||
export interface MissingPathFacts {
|
||||
/** The highest thing gone on the path; a link on the way is read through to its target. */
|
||||
scope:
|
||||
| "file"
|
||||
| "link_target"
|
||||
| "cache_entry"
|
||||
| "cache_partial"
|
||||
| "cache_root"
|
||||
| "work_dir"
|
||||
| "temp_root"
|
||||
| "other";
|
||||
/** Under a folder a sync app owns (OneDrive, Dropbox, iCloud, Google Drive). */
|
||||
synced: boolean;
|
||||
root: "system_temp" | "cache_dir" | "other";
|
||||
}
|
||||
|
||||
const SYNCED = /onedrive|dropbox|icloud|mobile documents|clouddocs|google ?drive|cloudstorage/i;
|
||||
|
||||
interface Where {
|
||||
tempRoot: string;
|
||||
cacheDir?: string;
|
||||
workDir?: string;
|
||||
}
|
||||
|
||||
const inside = (path: string, folder: string) => {
|
||||
const rel = relative(resolve(folder), resolve(path));
|
||||
return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
|
||||
};
|
||||
|
||||
function lstatOrNull(path: string) {
|
||||
try {
|
||||
return lstatSync(path);
|
||||
} catch {
|
||||
// expected: what is gone has nothing to stat
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const ROOT_SCOPES = [
|
||||
["tempRoot", "temp_root"],
|
||||
["cacheDir", "cache_root"],
|
||||
["workDir", "work_dir"],
|
||||
] as const;
|
||||
|
||||
const NAME_SCOPES: ReadonlyArray<[RegExp, MissingPathFacts["scope"]]> = [
|
||||
[/\.partial/, "cache_partial"],
|
||||
[/^hfcache-/, "cache_entry"],
|
||||
[/^hyperframes-extract-cache/, "cache_root"],
|
||||
[/^(hf-render-|work-[0-9a-f]{8}-)/, "work_dir"],
|
||||
];
|
||||
|
||||
function scopeOfGone(path: string, where: Where, throughLink: boolean): MissingPathFacts["scope"] {
|
||||
const root = ROOT_SCOPES.find(([key]) => {
|
||||
const dir = where[key];
|
||||
return !!dir && resolve(path) === resolve(dir);
|
||||
});
|
||||
if (root) return root[1];
|
||||
const name = basename(path);
|
||||
return (
|
||||
NAME_SCOPES.find(([pattern]) => pattern.test(name))?.[1] ??
|
||||
(throughLink ? "link_target" : "other")
|
||||
);
|
||||
}
|
||||
|
||||
const MAX_LINK_HOPS = 4;
|
||||
|
||||
function rootOf(path: string, where: Where): MissingPathFacts["root"] {
|
||||
if (inside(path, where.tempRoot)) return "system_temp";
|
||||
if (where.cacheDir && inside(path, where.cacheDir)) return "cache_dir";
|
||||
return "other";
|
||||
}
|
||||
|
||||
const ownerOf = (path: string, where: Where) =>
|
||||
[where.tempRoot, where.cacheDir, where.workDir]
|
||||
.filter((root): root is string => !!root && inside(path, root))
|
||||
.sort((a, b) => resolve(a).length - resolve(b).length)[0];
|
||||
|
||||
const linkTarget = (link: string, folder: string) =>
|
||||
resolve(folder, readlinkSync(link).replace(/^\\\\\?\\/, ""));
|
||||
|
||||
function goneAt(
|
||||
path: string,
|
||||
last: boolean,
|
||||
where: Where,
|
||||
hops: number,
|
||||
): MissingPathFacts["scope"] {
|
||||
const scope = scopeOfGone(path, where, hops > 0);
|
||||
return last && scope === "other" ? "file" : scope;
|
||||
}
|
||||
|
||||
/** Walks down from the render's own folder with lstat, checking each link's target before going through it. */
|
||||
function highestGone(path: string, where: Where, hops = 0): MissingPathFacts["scope"] | undefined {
|
||||
const root = ownerOf(path, where);
|
||||
if (!root || hops > MAX_LINK_HOPS) return "link_target";
|
||||
if (!existsSync(root)) return scopeOfGone(root, where, hops > 0);
|
||||
const parts = relative(resolve(root), resolve(path)).split(sep).filter(Boolean);
|
||||
let at = resolve(root);
|
||||
for (const [index, part] of parts.entries()) {
|
||||
const next = join(at, part);
|
||||
const stat = lstatOrNull(next);
|
||||
if (!stat) return goneAt(next, index === parts.length - 1, where, hops);
|
||||
at = stat.isSymbolicLink() ? linkTarget(next, at) : next;
|
||||
const gone = at === next ? undefined : highestGone(at, where, hops + 1);
|
||||
if (gone) return gone;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const isSpawnFailure = (error: unknown) =>
|
||||
!!(error as NodeJS.ErrnoException | undefined)?.syscall?.startsWith("spawn");
|
||||
|
||||
/** What was gone around the path a failed render's ENOENT names; undefined when it names none or the look fails.
|
||||
* Only the render's own folders are looked at, never a share; call it before the render's own cleanup runs. */
|
||||
export function describeMissingPath(
|
||||
error: unknown,
|
||||
{
|
||||
cacheDir,
|
||||
workDir,
|
||||
tempRoot = tmpdir(),
|
||||
}: { cacheDir?: string; workDir?: string; tempRoot?: string } = {},
|
||||
): MissingPathFacts | undefined {
|
||||
try {
|
||||
const path = errnoMissingPathOf(error);
|
||||
if (!path || isSpawnFailure(error)) return undefined;
|
||||
const where = { tempRoot, cacheDir, workDir };
|
||||
return {
|
||||
scope: ownerOf(path, where)
|
||||
? (highestGone(path, where) ?? "other")
|
||||
: scopeOfGone(path, where, false),
|
||||
synced: SYNCED.test(path),
|
||||
root: rootOf(path, where),
|
||||
};
|
||||
} catch {
|
||||
// expected: the folders can change under the look; the render's own error must still be what is reported
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -224,6 +224,7 @@ export interface RenderExtractionObservability {
|
||||
vfrPreflightCount?: number;
|
||||
cacheHits?: number;
|
||||
cacheMisses?: number;
|
||||
cachePublishFailures?: number;
|
||||
/** Per-source transient download/metadata/FFmpeg retries performed during extraction. */
|
||||
transientRetries?: number;
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { resolveConfig } from "@hyperframes/engine";
|
||||
import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js";
|
||||
|
||||
// Entry: executeRenderJob. Capture setup fails on a frame whose cache entry another hand removed; the failure must
|
||||
// say what was gone, read before the render removes its own folders.
|
||||
const trace = vi.hoisted(() => ({ missing: "", probeThrows: false }));
|
||||
|
||||
vi.mock("@hyperframes/core/missing-path", async (importOriginal) => {
|
||||
const real = await importOriginal<typeof import("@hyperframes/core/missing-path")>();
|
||||
return {
|
||||
errnoMissingPathOf: (error: unknown) => {
|
||||
if (trace.probeThrows) throw new RangeError("Maximum call stack size exceeded");
|
||||
return real.errnoMissingPathOf(error);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./render/shared.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./render/shared.js")>()),
|
||||
createMemorySampler: () => ({
|
||||
peakRssBytes: () => 0,
|
||||
peakHeapUsedBytes: () => 0,
|
||||
stop: () => {},
|
||||
}),
|
||||
}));
|
||||
vi.mock("./fileServer.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./fileServer.js")>()),
|
||||
createFileServer: async () => ({
|
||||
url: "http://127.0.0.1:1",
|
||||
port: 1,
|
||||
close: () => {},
|
||||
addPreHeadScript: () => {},
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/compileStage.js", () => ({
|
||||
runCompileStage: async () => ({
|
||||
compiled: {
|
||||
html: "<div></div>",
|
||||
hasShaderTransitions: false,
|
||||
renderModeHints: { reasons: [] },
|
||||
},
|
||||
composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] },
|
||||
deviceScaleFactor: 1,
|
||||
outputWidth: 16,
|
||||
outputHeight: 16,
|
||||
compileOnlyMs: 0,
|
||||
forceScreenshot: true,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/probeStage.js", () => ({
|
||||
runProbeStage: async ({ compiled }: { compiled: object }) => ({
|
||||
compiled,
|
||||
fileServer: null,
|
||||
probeSession: null,
|
||||
lastBrowserConsole: [],
|
||||
duration: 40 / 30,
|
||||
totalFrames: 40,
|
||||
browserProbeMs: 0,
|
||||
beginFrameStalled: false,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/extractVideosStage.js", () => ({
|
||||
runExtractVideosStage: async () => ({
|
||||
extractionResult: null,
|
||||
frameLookup: null,
|
||||
videoReadinessSkipIds: [],
|
||||
videoMetadataHints: [],
|
||||
nativeHdrVideoIds: new Set(),
|
||||
videoTransfers: new Map(),
|
||||
nativeHdrImageIds: new Set(),
|
||||
imageTransfers: new Map(),
|
||||
hdrImageSrcPaths: new Map(),
|
||||
imageColorSpaces: [],
|
||||
videoExtractMs: 0,
|
||||
failureToEnforce: null,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/audioStage.js", () => ({
|
||||
runAudioStage: async ({ workDir }: { workDir: string }) => ({
|
||||
audioOutputPath: join(workDir, "audio.wav"),
|
||||
hasAudio: false,
|
||||
audioProcessMs: 0,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/captureCost.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./render/captureCost.js")>()),
|
||||
resolveRenderWorkerCount: () => {
|
||||
throw Object.assign(new Error(`ENOENT: no such file or directory, open '${trace.missing}'`), {
|
||||
code: "ENOENT",
|
||||
path: trace.missing,
|
||||
});
|
||||
},
|
||||
}));
|
||||
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => {
|
||||
trace.probeThrows = false;
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function failingRender() {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-missing-render-"));
|
||||
dirs.push(root);
|
||||
writeFileSync(join(root, "index.html"), "<div></div>");
|
||||
const cacheRoot = join(root, "hyperframes-extract-cache-u");
|
||||
mkdirSync(cacheRoot);
|
||||
trace.missing = join(cacheRoot, "hfcache-v10-gone", "frame_00001.jpg");
|
||||
const job = createRenderJob({
|
||||
fps: 30,
|
||||
quality: "draft",
|
||||
format: "mp4",
|
||||
hdrMode: "force-sdr",
|
||||
producerConfig: resolveConfig({ enablePageSideCompositing: false }),
|
||||
logger: { info() {}, warn() {}, error() {}, debug() {} },
|
||||
});
|
||||
return { job, run: () => executeRenderJob(job, root, join(root, "out.mp4")) };
|
||||
}
|
||||
|
||||
it("a render that fails on a missing frame says the cache entry holding it was gone", async () => {
|
||||
const { job, run } = failingRender();
|
||||
await expect(run()).rejects.toThrow(/ENOENT/);
|
||||
expect(job.errorDetails?.missingPath).toEqual({
|
||||
scope: "cache_entry",
|
||||
synced: false,
|
||||
root: "system_temp",
|
||||
});
|
||||
});
|
||||
|
||||
it("a missing-path look that throws leaves the render's own failure as it was", async () => {
|
||||
trace.probeThrows = true;
|
||||
const { job, run } = failingRender();
|
||||
await expect(run()).rejects.toThrow(/ENOENT/);
|
||||
expect(job.status).toBe("failed");
|
||||
expect(job.error).toMatch(/ENOENT/);
|
||||
expect(job.errorDetails?.missingPath).toBeUndefined();
|
||||
});
|
||||
@@ -169,6 +169,7 @@ import {
|
||||
} from "./render/shared.js";
|
||||
import { frameCacheOptions } from "./render/frameCacheBudget.js";
|
||||
import { buildRenderErrorDetails } from "./render/cleanup.js";
|
||||
import { describeMissingPath, type MissingPathFacts } from "./render/missingPath.js";
|
||||
import { publishRenderFailure } from "./render/renderEventPublisher.js";
|
||||
import { EncoderInterruptedError } from "./render/encoderInterruption.js";
|
||||
import { RenderExecutionContext } from "./render/renderExecutionContext.js";
|
||||
@@ -347,6 +348,7 @@ function summarizeExtractionObservability(
|
||||
vfrPreflightCount: phaseBreakdown?.vfrPreflightCount,
|
||||
cacheHits: phaseBreakdown?.cacheHits,
|
||||
cacheMisses: phaseBreakdown?.cacheMisses,
|
||||
cachePublishFailures: phaseBreakdown?.cachePublishFailures,
|
||||
transientRetries: phaseBreakdown?.transientRetries,
|
||||
...coverageGauges,
|
||||
authoredTimedClipCount,
|
||||
@@ -840,6 +842,7 @@ export interface RenderJob extends Partial<ReferenceBackgroundResult> {
|
||||
observability?: RenderObservabilitySummary;
|
||||
/** Worst sub-composition timeline wait outcome across sessions captured before the failure. */
|
||||
subTimelineWait?: SubTimelineWaitOutcome;
|
||||
missingPath?: MissingPathFacts;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -6033,6 +6036,7 @@ async function executeRenderPipeline(input: {
|
||||
hdrDiagnostics,
|
||||
observability: observabilitySummary,
|
||||
subTimelineWait: worstSubTimelineWaitOutcome(dedupPerfs),
|
||||
missingPath: describeMissingPath(error, { cacheDir: cfg.extractCacheDir, workDir }),
|
||||
});
|
||||
publishRenderFailure(
|
||||
job,
|
||||
|
||||
@@ -2147,15 +2147,6 @@ export const ALLOWED_DELETIONS = new Map([
|
||||
"packages/engine/src/services/frameCapture-pageNetwork.test.ts",
|
||||
"pinned the removed per-page filter; browserManager.test.ts pins when a render browser goes behind the proxy",
|
||||
],
|
||||
...[
|
||||
"packages/core/src/missingPath.ts",
|
||||
"packages/producer/src/services/render/missingPath.ts",
|
||||
"packages/producer/src/services/render/missingPath.test.ts",
|
||||
"packages/producer/src/services/renderOrchestrator.missingPath.test.ts",
|
||||
].map((path) => [
|
||||
path,
|
||||
"the revert of #7955 removes the missing-path detail; it returns looking only inside folders the render owns",
|
||||
]),
|
||||
]);
|
||||
|
||||
export function parseBase(argv, fallback = "origin/main") {
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
"unpacked": 554000
|
||||
},
|
||||
"@hyperframes/producer": {
|
||||
"packed": 936000,
|
||||
"packed": 993551,
|
||||
"unpacked": 3491355
|
||||
},
|
||||
"@hyperframes/sdk": {
|
||||
|
||||
Reference in New Issue
Block a user