Merge pull request #2527 from heygen-com/via/artifact-validated-exit-gate

fix(cli): don't override exit code after artifact validated
This commit is contained in:
Vance Ingalls
2026-07-16 00:35:30 -07:00
committed by GitHub
5 changed files with 384 additions and 40 deletions
@@ -10,6 +10,13 @@ function tempDir(): string {
}
describe("createContactSheet", () => {
// Sharp on Windows CI runners exercises a native-binary fork per operation
// and the runner's I/O throughput varies with concurrent-job pressure. The
// default 20s ceiling has landed just-over the wall clock repeatedly (see
// PR #2492's earlier lightweighting attempt); the actual work here — two
// 16×9 PNG writes + one contact-sheet composite + one metadata probe —
// is milliseconds of compute, so the extra ceiling only absorbs runner
// I/O jitter, it does not hide a real slowdown.
it("writes PNG output when the output path uses a .png extension", async () => {
const dir = tempDir();
try {
@@ -49,5 +56,5 @@ describe("createContactSheet", () => {
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
}, 60_000);
});
+74 -15
View File
@@ -103,6 +103,7 @@ import { defineCommand, runMain } from "citty";
import type { ArgsDef, CommandDef } from "citty";
import { getRunId } from "./telemetry/runId.js";
import { reportCommandFailure, trackCommandFailures } from "./utils/command-failure-tracking.js";
import { isRenderSucceeded } from "./utils/render-success-state.js";
const isHelp = process.argv.includes("--help") || process.argv.includes("-h");
@@ -294,36 +295,94 @@ process.on("exit", (code) => {
_flushSync?.();
});
process.on("uncaughtException", (error) => {
if ((error as NodeJS.ErrnoException).code === "EPIPE") {
commandFailed = true;
process.exit(0);
}
commandFailed = true;
// Report a CLI error event to telemetry. Extracted from the process-error
// handlers so their bodies stay simple linear branches (see fallow CRAP
// scoring — arrow handlers with inline telemetry calls tip past threshold).
function emitCliErrorEvent(kind: "uncaught_exception" | "unhandled_rejection", error: Error): void {
_trackCliError?.({
error_name: error.name,
error_message: error.message,
stack_trace: error.stack,
command,
kind: "uncaught_exception",
kind,
});
}
// Handle a post-artifact-validated throw: record the diagnostic, emit the
// telemetry event, but do NOT mark the run as failed. Field signals:
// ts=1784169760, ts=1784171150, ts=1784172467 (all win32/x64, CLI 0.7.58,
// ffmpeg=no, 1080x1920, valid MP4s on disk).
// The render is valid — a worker teardown / browser shutdown / stray
// subprocess stream error after `renderSucceeded` was set must not flip
// exit code or telemetry success to failure.
function reportPostRenderTerminationEvent(
label: "uncaughtException" | "unhandledRejection",
kind: "uncaught_exception" | "unhandled_rejection",
error: Error,
): void {
process.stderr.write(
` [hyperframes] Post-render ${label} (render already succeeded): ${error.message}\n`,
);
emitCliErrorEvent(kind, error);
}
// Terminate the process after a post-artifact-validated throw. Wraps
// report + flush + exit(0) so the caller arrow handler doesn't accumulate
// optional-chain branches (fallow CRAP scoring on the arrow tips past
// threshold otherwise).
function exitAfterPostRenderTermination(
label: "uncaughtException" | "unhandledRejection",
kind: "uncaught_exception" | "unhandled_rejection",
error: Error,
): never {
reportPostRenderTerminationEvent(label, kind, error);
_flushSync?.();
process.exit(0);
}
// Terminate the process after a genuine CLI failure — mark commandFailed,
// emit telemetry, flush, exit(1). Same rationale as above: keeps the arrow
// handler linear so fallow CRAP stays under threshold.
function exitAfterCliFailure(
kind: "uncaught_exception" | "unhandled_rejection",
error: Error,
): never {
commandFailed = true;
emitCliErrorEvent(kind, error);
_flushSync?.();
process.exit(1);
}
process.on("uncaughtException", (error) => {
if ((error as NodeJS.ErrnoException).code === "EPIPE") {
commandFailed = true;
process.exit(0);
}
// Post-artifact-validated shutdown throws must not turn a valid render
// into an exit-1 "no final error message" failure. The render command
// sets `renderSucceeded` right after the producer resolves and the
// artifact is committed.
if (isRenderSucceeded()) {
exitAfterPostRenderTermination("uncaughtException", "uncaught_exception", error);
}
exitAfterCliFailure("uncaught_exception", error);
});
// unhandledRejection does not call process.exit() — Node may continue
// running if the rejection is non-fatal (e.g. a fire-and-forget promise).
// The exit handler above will still fire with the real exit code.
process.on("unhandledRejection", (reason) => {
commandFailed = true;
const error = reason instanceof Error ? reason : new Error(String(reason));
_trackCliError?.({
error_name: error.name,
error_message: error.message,
stack_trace: error.stack,
command,
kind: "unhandled_rejection",
});
// Same rationale as the uncaughtException branch above: a stray promise
// rejection during post-artifact-validated cleanup must not mark a valid
// render as failed. `commandFailed` gates the success:true telemetry
// field — keep it false when the render actually succeeded.
if (isRenderSucceeded()) {
reportPostRenderTerminationEvent("unhandledRejection", "unhandled_rejection", error);
return;
}
commandFailed = true;
emitCliErrorEvent("unhandled_rejection", error);
});
// Lazy-load help renderer — avoids allocating help data on non-help invocations
+54 -24
View File
@@ -81,6 +81,11 @@ import { runEnvironmentChecks } from "../browser/preflight.js";
import { detectH264EncoderMode } from "../browser/ffmpeg.js";
import { chromeLaunchRemediation } from "../browser/linuxDeps.js";
import { killOrphanedProcesses } from "../utils/orphanCleanup.js";
import {
markRenderSucceeded,
runPostRenderStep,
runPostRenderStepAsync,
} from "../utils/render-success-state.js";
import type { ProducerLogger, RenderJob } from "@hyperframes/producer";
import {
MAX_VP9_CPU_USED,
@@ -1413,23 +1418,35 @@ async function renderDocker(
const elapsed = Date.now() - startTime;
// Docker child exited 0 → the containerized producer already validated
// AND committed the artifact. Mirror renderLocal's post-success guarantee
// so any late throw here (telemetry flush, feedback prompt) cannot flip
// the exit code.
markRenderSucceeded();
// Track metrics (no job object available from Docker — use a minimal stub)
trackRenderComplete({
durationMs: elapsed,
fps: fpsToNumber(options.fps),
quality: options.quality,
workers: options.workers,
docker: true,
gpu: options.gpu,
authoringSkill: options.authoringSkill,
...getMemorySnapshot(),
});
runPostRenderStep("trackRenderComplete", () =>
trackRenderComplete({
durationMs: elapsed,
fps: fpsToNumber(options.fps),
quality: options.quality,
workers: options.workers,
docker: true,
gpu: options.gpu,
authoringSkill: options.authoringSkill,
...getMemorySnapshot(),
}),
);
// ponytail: Docker runs the producer in a child process, so no perfSummary is
// threaded back here; the summary shows render time only (never a wrong video
// length). Probe the output with ffprobe if a duration figure is wanted here.
printRenderComplete(outputPath, elapsed, options.quiet);
warnIfWebmAlphaDropped(outputPath, options.format, options.quiet);
runPostRenderStep("printRenderComplete", () =>
printRenderComplete(outputPath, elapsed, options.quiet),
);
runPostRenderStep("warnIfWebmAlphaDropped", () =>
warnIfWebmAlphaDropped(outputPath, options.format, options.quiet),
);
if (options.exitAfterComplete) scheduleRenderProcessExit();
return { renderTimeMs: elapsed };
}
@@ -1559,6 +1576,13 @@ export async function renderLocal(
);
}
// Render resolved without throwing → producer's `artifact validated`
// checkpoint fired AND the artifact was committed to disk. From this
// point on, ANY thrown teardown error must not be allowed to override
// the exit code. Field signal ts=1784169760 / ts=1784171150 / ts=1784172467
// (win32/x64, CLI 0.7.58): valid MP4 on disk, exited 1 with no error print.
markRenderSucceeded();
maybeConsumeDeParallelRouterTrial(deParallelRouterTrialArmed, job, options.quiet);
const elapsed = Date.now() - startTime;
if (job.outcome === "completed_with_warnings") {
@@ -1566,20 +1590,26 @@ export async function renderLocal(
console.warn(c.warn(` [${warning.code}] ${warning.message}`));
}
}
trackRenderMetrics(job, elapsed, options, false);
printRenderComplete(
outputPath,
elapsed,
options.quiet,
job.perfSummary?.compositionDurationSeconds,
job.perfSummary?.totalFrames,
runPostRenderStep("trackRenderMetrics", () => trackRenderMetrics(job, elapsed, options, false));
runPostRenderStep("printRenderComplete", () =>
printRenderComplete(
outputPath,
elapsed,
options.quiet,
job.perfSummary?.compositionDurationSeconds,
job.perfSummary?.totalFrames,
),
);
runPostRenderStep("warnIfWebmAlphaDropped", () =>
warnIfWebmAlphaDropped(outputPath, options.format, options.quiet),
);
warnIfWebmAlphaDropped(outputPath, options.format, options.quiet);
if (!options.skipFeedback) {
await maybePromptRenderFeedback({
renderDurationMs: elapsed,
quiet: options.quiet,
});
await runPostRenderStepAsync("maybePromptRenderFeedback", () =>
maybePromptRenderFeedback({
renderDurationMs: elapsed,
quiet: options.quiet,
}),
);
}
if (options.exitAfterComplete) scheduleRenderProcessExit();
const durationMs = job.perfSummary
@@ -0,0 +1,153 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
_resetRenderSuccessForTests,
isRenderSucceeded,
markRenderSucceeded,
runPostRenderStep,
runPostRenderStepAsync,
} from "./render-success-state.js";
describe("render-success-state flag", () => {
afterEach(() => {
_resetRenderSuccessForTests();
});
it("starts unset", () => {
expect(isRenderSucceeded()).toBe(false);
});
it("flips true after markRenderSucceeded()", () => {
markRenderSucceeded();
expect(isRenderSucceeded()).toBe(true);
});
it("stays true across repeated calls (idempotent)", () => {
markRenderSucceeded();
markRenderSucceeded();
markRenderSucceeded();
expect(isRenderSucceeded()).toBe(true);
});
it("reset restores the initial state", () => {
markRenderSucceeded();
_resetRenderSuccessForTests();
expect(isRenderSucceeded()).toBe(false);
});
});
describe("runPostRenderStep", () => {
const originalExitCode = process.exitCode;
afterEach(() => {
process.exitCode = originalExitCode;
_resetRenderSuccessForTests();
});
it("runs the step and returns normally on success", () => {
const sink = vi.fn();
const step = vi.fn();
runPostRenderStep("noop", step, sink);
expect(step).toHaveBeenCalledTimes(1);
expect(sink).not.toHaveBeenCalled();
});
it("swallows a thrown error and reports it to the sink", () => {
const sink = vi.fn();
const err = new Error("teardown blew up");
runPostRenderStep(
"trackRenderMetrics",
() => {
throw err;
},
sink,
);
expect(sink).toHaveBeenCalledTimes(1);
const msg = sink.mock.calls[0]?.[0] as string;
expect(msg).toContain("trackRenderMetrics");
expect(msg).toContain("teardown blew up");
expect(msg).toContain("render already succeeded");
});
it("sanitizes a stray non-zero process.exitCode back to 0", () => {
// Regression: a cleanup step that sets process.exitCode=1 (or a helper it
// calls that does) must not leave the CLI exiting 1 after a successful
// render. Field signal ts=1784169760 / ts=1784171150 / ts=1784172467.
process.exitCode = 1;
runPostRenderStep(
"printRenderComplete",
() => {
throw new Error("stat threw");
},
vi.fn(),
);
expect(process.exitCode).toBe(0);
});
it("does not touch process.exitCode when the step succeeds", () => {
process.exitCode = 1;
runPostRenderStep("ok", () => undefined, vi.fn());
expect(process.exitCode).toBe(1);
});
it("does not touch process.exitCode when it was already 0", () => {
process.exitCode = 0;
runPostRenderStep(
"warnIfWebmAlphaDropped",
() => {
throw new Error("stat missing");
},
vi.fn(),
);
expect(process.exitCode).toBe(0);
});
it("stringifies a non-Error throw value", () => {
const sink = vi.fn();
runPostRenderStep(
"misc",
() => {
throw "just a string";
},
sink,
);
const msg = sink.mock.calls[0]?.[0] as string;
expect(msg).toContain("just a string");
});
});
describe("runPostRenderStepAsync", () => {
const originalExitCode = process.exitCode;
afterEach(() => {
process.exitCode = originalExitCode;
_resetRenderSuccessForTests();
});
it("awaits the async step and returns normally on success", async () => {
const sink = vi.fn();
const step = vi.fn(() => Promise.resolve());
await runPostRenderStepAsync("feedback", step, sink);
expect(step).toHaveBeenCalledTimes(1);
expect(sink).not.toHaveBeenCalled();
});
it("swallows a rejected promise and reports it to the sink", async () => {
const sink = vi.fn();
const err = new Error("feedback prompt crashed");
await runPostRenderStepAsync("maybePromptRenderFeedback", () => Promise.reject(err), sink);
expect(sink).toHaveBeenCalledTimes(1);
const msg = sink.mock.calls[0]?.[0] as string;
expect(msg).toContain("maybePromptRenderFeedback");
expect(msg).toContain("feedback prompt crashed");
});
it("sanitizes process.exitCode back to 0 on async failure", async () => {
process.exitCode = 1;
await runPostRenderStepAsync(
"maybePromptRenderFeedback",
() => Promise.reject(new Error("stdin closed")),
vi.fn(),
);
expect(process.exitCode).toBe(0);
});
});
@@ -0,0 +1,95 @@
/**
* Shared render-success sentinel + post-artifact-validated cleanup guards.
*
* Read by the top-level CLI process handlers (uncaughtException /
* unhandledRejection) to sanitize the exit code when a post-artifact-validated
* cleanup step throws.
*
* Set by the render command AFTER `executeRenderJob` (or the Docker child
* render) resolves cleanly — the point at which the artifact has been
* validated AND committed to disk. Any throw after this point (worker
* teardown, browser shutdown, telemetry flush, feedback prompt, stray
* promise rejection) must not turn a valid render into an exit-1
* "no final error message" failure.
*
* Field signal (all win32/x64, CLI 0.7.58, ffmpeg=no, 1080x1920 renders):
* - ts=1784169760 — 6-worker capture retried down after Runtime.evaluate
* timeout, completed all 1260 frames, printed 'artifact validated',
* exited 1 with no final error message. Output MP4 valid on disk.
* - ts=1784171150 — full REPRO command provided; identical shape.
* - ts=1784172467 — `--workers 2`, identical shape.
* All three: ffprobe + visual QA confirmed the output was valid; the CLI
* still exited 1 after the terminal "artifact validated" checkpoint.
*/
let renderSucceeded = false;
/**
* Called by the render command after the producer's `executeRenderJob` (or
* the Docker child) resolves cleanly. From this point on, any thrown
* teardown error must not be allowed to override the exit code.
*/
export function markRenderSucceeded(): void {
renderSucceeded = true;
}
/**
* Read by cli.ts process handlers to decide whether a late throw is fatal.
* When true, the handlers log the throw at warn level (so it's still visible
* for diagnosis) but do not surface it as an exit-1 failure.
*/
export function isRenderSucceeded(): boolean {
return renderSucceeded;
}
/** Test-only reset. Not exported from the package. */
export function _resetRenderSuccessForTests(): void {
renderSucceeded = false;
}
type PostRenderErrorSink = (message: string) => void;
const defaultErrorSink: PostRenderErrorSink = (message) => {
process.stderr.write(`${message}\n`);
};
/**
* Run a post-artifact-validated cleanup step so a throw cannot flip the CLI
* exit code. `markRenderSucceeded()` MUST have been called first — this
* helper is only safe on the success path where the artifact is already
* committed to disk. Logs a compact warning to stderr, resets a stray
* `process.exitCode` back to 0, and swallows the error.
*/
export function runPostRenderStep(
label: string,
fn: () => void,
sink: PostRenderErrorSink = defaultErrorSink,
): void {
try {
fn();
} catch (err) {
reportPostRenderStepFailure(label, err, sink);
}
}
export async function runPostRenderStepAsync(
label: string,
fn: () => Promise<void>,
sink: PostRenderErrorSink = defaultErrorSink,
): Promise<void> {
try {
await fn();
} catch (err) {
reportPostRenderStepFailure(label, err, sink);
}
}
function reportPostRenderStepFailure(label: string, err: unknown, sink: PostRenderErrorSink): void {
const message = err instanceof Error ? err.message : String(err);
sink(` [hyperframes] Post-render step '${label}' failed (render already succeeded): ${message}`);
// Guard against the failing step (or something it triggered) setting a
// non-zero exitCode. The render succeeded → the CLI must exit 0.
if (process.exitCode !== undefined && process.exitCode !== 0) {
process.exitCode = 0;
}
}