From 0f624f59fe9a1fe1a4181519734784b8cbec8e16 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 May 2026 23:10:54 +0000 Subject: [PATCH] fix(aws-lambda): surface sparticuz wedge as typed non-retryable error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repeated Sandbox.Timedout chunks can leave @sparticuz/chromium returning a falsy/empty path on subsequent invocations — warm instances on the same execution environment never re-extract chromium. The downstream puppeteer-core assertion about needing an executablePath or channel buries the actionable cause; a cost- analysis sweep took ~30 min to root-cause from that trace. Guard the resolver: if mod.executablePath() returns a non-string, empty string, or a path that does not exist on disk, throw a typed ChromeBinaryUnavailableError whose message points at the recycle remedy (env-var bump or redeploy). Add the error name to the three NON_RETRYABLE lists so SFN short-circuits instead of burning four 15-min retries on a function that won't recover. Same typed-error contract for the chrome-headless-shell fallback so both sources fail consistently. Tests pin the wedge path (empty string + non-existent file) and the carried metadata (source + resolvedPath). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../HyperframesRenderStack.snapshot.test.ts | 1 + .../src/cdk/HyperframesRenderStack.ts | 6 ++ packages/aws-lambda/src/chromium.test.ts | 73 ++++++++++++++++--- packages/aws-lambda/src/chromium.ts | 56 ++++++++++++-- packages/aws-lambda/src/index.ts | 1 + 5 files changed, 123 insertions(+), 14 deletions(-) diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts index 1baadb27d..788bfb2c8 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts @@ -65,6 +65,7 @@ const EXPECTED_NON_RETRYABLE_ERRORS = new Set([ "FONT_FETCH_FAILED", "PLAN_TOO_LARGE", "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED", + "ChromeBinaryUnavailableError", ]); function doSynth(): { diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts index 128c24e1c..71a7b7e69 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts @@ -189,6 +189,9 @@ export class HyperframesRenderStack extends Construct { * the snapshot test. */ private buildStateMachineDefinition(): sfn.IChainable { + // `ChromeBinaryUnavailableError` is non-retryable: a wedged warm + // instance keeps returning the same falsy executablePath until the + // env recycles, so retries just burn the 4× 15-min budget. const NON_RETRYABLE_PLAN = [ "FFMPEG_VERSION_MISMATCH", "PLAN_HASH_MISMATCH", @@ -196,16 +199,19 @@ export class HyperframesRenderStack extends Construct { "FONT_FETCH_FAILED", "PLAN_TOO_LARGE", "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED", + "ChromeBinaryUnavailableError", ]; const NON_RETRYABLE_CHUNK = [ "FFMPEG_VERSION_MISMATCH", "PLAN_HASH_MISMATCH", "BROWSER_GPU_NOT_SOFTWARE", + "ChromeBinaryUnavailableError", ]; const NON_RETRYABLE_ASSEMBLE = [ "FFMPEG_VERSION_MISMATCH", "PLAN_HASH_MISMATCH", "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED", + "ChromeBinaryUnavailableError", ]; const plan = new tasks.LambdaInvoke(this, "Plan", { diff --git a/packages/aws-lambda/src/chromium.test.ts b/packages/aws-lambda/src/chromium.test.ts index 75953e281..2ae7d2a35 100644 --- a/packages/aws-lambda/src/chromium.test.ts +++ b/packages/aws-lambda/src/chromium.test.ts @@ -13,6 +13,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { _setSparticuzChromiumForTests, + ChromeBinaryUnavailableError, resolveChromeArgs, resolveChromeExecutablePath, resolveChromeSource, @@ -60,13 +61,65 @@ describe("resolveChromeSource", () => { describe("resolveChromeExecutablePath", () => { it("returns the path from a stubbed sparticuz module", async () => { + process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "sparticuz"; + const dir = mkdtempSync(join(tmpdir(), "hf-chrome-test-")); + const binPath = join(dir, "chromium"); + writeFileSync(binPath, "fake binary"); + try { + _setSparticuzChromiumForTests({ + args: ["--fake-arg"], + executablePath: async () => binPath, + }); + expect(await resolveChromeExecutablePath()).toBe(binPath); + expect(await resolveChromeArgs()).toEqual(["--fake-arg"]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // Real-world wedge: a chunk hits `Sandbox.Timedout` mid-extraction, + // leaving sparticuz's internal state inconsistent. On subsequent + // invocations executablePath() returns empty/undefined and puppeteer- + // core's downstream assertion buries the actionable signal. These + // tests pin the typed-error contract so SFN can short-circuit retries. + it("throws ChromeBinaryUnavailableError when sparticuz returns empty string", async () => { process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "sparticuz"; _setSparticuzChromiumForTests({ - args: ["--fake-arg"], - executablePath: async () => "/tmp/sparticuz-chromium", + args: [], + executablePath: async () => "", }); - expect(await resolveChromeExecutablePath()).toBe("/tmp/sparticuz-chromium"); - expect(await resolveChromeArgs()).toEqual(["--fake-arg"]); + await expect(resolveChromeExecutablePath()).rejects.toBeInstanceOf( + ChromeBinaryUnavailableError, + ); + }); + + it("throws ChromeBinaryUnavailableError when sparticuz returns a non-existent path", async () => { + process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "sparticuz"; + _setSparticuzChromiumForTests({ + args: [], + executablePath: async () => "/nonexistent/sparticuz/chromium", + }); + await expect(resolveChromeExecutablePath()).rejects.toBeInstanceOf( + ChromeBinaryUnavailableError, + ); + }); + + it("ChromeBinaryUnavailableError carries the source + resolved path", async () => { + process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "sparticuz"; + _setSparticuzChromiumForTests({ + args: [], + executablePath: async () => "/missing", + }); + try { + await resolveChromeExecutablePath(); + throw new Error("expected throw"); + } catch (err) { + expect(err).toBeInstanceOf(ChromeBinaryUnavailableError); + const typed = err as ChromeBinaryUnavailableError; + expect(typed.source).toBe("sparticuz"); + expect(typed.resolvedPath).toBe("/missing"); + expect(typed.name).toBe("ChromeBinaryUnavailableError"); + } }); it("reads chrome-headless-shell path from HYPERFRAMES_LAMBDA_CHROME_PATH", async () => { @@ -83,17 +136,19 @@ describe("resolveChromeExecutablePath", () => { } }); - it("throws if chrome-headless-shell path is missing", async () => { + it("throws ChromeBinaryUnavailableError if chrome-headless-shell path is missing", async () => { process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "chrome-headless-shell"; delete process.env.HYPERFRAMES_LAMBDA_CHROME_PATH; - await expect(resolveChromeExecutablePath()).rejects.toThrow( - /HYPERFRAMES_LAMBDA_CHROME_PATH to be set/, + await expect(resolveChromeExecutablePath()).rejects.toBeInstanceOf( + ChromeBinaryUnavailableError, ); }); - it("throws if chrome-headless-shell path doesn't exist on disk", async () => { + it("throws ChromeBinaryUnavailableError if chrome-headless-shell path doesn't exist on disk", async () => { process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "chrome-headless-shell"; process.env.HYPERFRAMES_LAMBDA_CHROME_PATH = "/nonexistent/path/chrome-headless-shell"; - await expect(resolveChromeExecutablePath()).rejects.toThrow(/does not exist/); + await expect(resolveChromeExecutablePath()).rejects.toBeInstanceOf( + ChromeBinaryUnavailableError, + ); }); }); diff --git a/packages/aws-lambda/src/chromium.ts b/packages/aws-lambda/src/chromium.ts index af0009237..79c26b29d 100644 --- a/packages/aws-lambda/src/chromium.ts +++ b/packages/aws-lambda/src/chromium.ts @@ -37,6 +37,34 @@ import { existsSync } from "node:fs"; /** Discriminator for the two supported Chrome sources. */ export type ChromeSource = "sparticuz" | "chrome-headless-shell"; +/** + * Thrown when the Chrome binary resolver can't produce a usable path. + * The class name is the SFN `Retry: { ErrorEquals: [...] }` discriminator — + * see {@link HyperframesRenderStack}'s NON_RETRYABLE_* lists. + */ +export class ChromeBinaryUnavailableError extends Error { + // Lambda's runtime serializes the error envelope's `errorType` from + // `err.name`; this class-field override sets it across the structured + // clone. Read indirectly; fallow can't follow. + // fallow-ignore-next-line unused-class-member + override readonly name = "ChromeBinaryUnavailableError"; + readonly source: ChromeSource; + readonly resolvedPath: string | null; + constructor(source: ChromeSource, resolvedPath: string | null, hint: string) { + super(`[chromium] Chrome binary unavailable (source=${source}): ${hint}`); + this.source = source; + this.resolvedPath = resolvedPath; + } +} + +const SPARTICUZ_WEDGE_HINT = + "@sparticuz/chromium.executablePath() returned a falsy value or a path that doesn't exist on disk. " + + "This typically happens after a chunk hits `Sandbox.Timedout` mid-extraction and leaves /tmp in a " + + "wedged state — subsequent invocations land on the same warm instance and never re-extract. " + + "Recycle the function (e.g. `aws lambda update-function-configuration ... --environment ...` with a " + + "bumped marker var, or redeploy via `hyperframes lambda deploy --skip-build`) to force fresh " + + "execution environments. Tracking: investigate the upstream wedge so this auto-recovers."; + /** * Read which Chrome source the bundled ZIP was built against. Defaults to * `"sparticuz"` so a fresh build with no env override picks the primary @@ -60,22 +88,40 @@ export function resolveChromeSource(): ChromeSource { * `HYPERFRAMES_LAMBDA_CHROME_PATH`. Throws if absent or non-existent so a * misconfigured deploy fails loudly at boot rather than at first frame. */ +// fallow-ignore-next-line complexity export async function resolveChromeExecutablePath(): Promise { const source = resolveChromeSource(); if (source === "sparticuz") { const mod = await loadSparticuzChromium(); - return mod.executablePath(); + const path = await mod.executablePath(); + // Guard against the wedge described in ChromeBinaryUnavailableError. + // sparticuz's contract is "return the path to a usable binary" — when + // it returns null/undefined/"" we can't hand that to puppeteer-core + // (which will throw an unrelated-looking assertion). Same when the + // returned path doesn't exist (extraction failed but the function + // call returned). + if (!path || typeof path !== "string") { + throw new ChromeBinaryUnavailableError(source, null, SPARTICUZ_WEDGE_HINT); + } + if (!existsSync(path)) { + throw new ChromeBinaryUnavailableError(source, path, SPARTICUZ_WEDGE_HINT); + } + return path; } const explicit = process.env.HYPERFRAMES_LAMBDA_CHROME_PATH; if (!explicit) { - throw new Error( - "[chromium] HYPERFRAMES_LAMBDA_CHROME_SOURCE=chrome-headless-shell requires " + + throw new ChromeBinaryUnavailableError( + source, + null, + "HYPERFRAMES_LAMBDA_CHROME_SOURCE=chrome-headless-shell requires " + "HYPERFRAMES_LAMBDA_CHROME_PATH to be set to the absolute path of the bundled binary.", ); } if (!existsSync(explicit)) { - throw new Error( - `[chromium] HYPERFRAMES_LAMBDA_CHROME_PATH=${JSON.stringify(explicit)} does not exist`, + throw new ChromeBinaryUnavailableError( + source, + explicit, + `HYPERFRAMES_LAMBDA_CHROME_PATH=${JSON.stringify(explicit)} does not exist on disk.`, ); } return explicit; diff --git a/packages/aws-lambda/src/index.ts b/packages/aws-lambda/src/index.ts index ef41f4d90..eba39a530 100644 --- a/packages/aws-lambda/src/index.ts +++ b/packages/aws-lambda/src/index.ts @@ -39,6 +39,7 @@ export { // the package barrel — it's a test-only DI seam. Test files import it // directly from `./chromium.js`. export { + ChromeBinaryUnavailableError, type ChromeSource, resolveChromeArgs, resolveChromeExecutablePath,