fix(producer): retry transient deterministic font fetches (#2865)

* fix(producer): retry transient deterministic font fetches

* fix(producer): secure Lambda font cache directory

packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts

@@ -182,6 +182,7 @@ describe("HyperframesRenderStack — snapshot", () => {
         present: true,
       });
     }
+    expect(collected.has("FONT_FETCH_UNAVAILABLE")).toBe(false);
   });
 
   it("classifies plan v2 integrity failures as terminal in every v2 Lambda task", () => {

packages/aws-lambda/src/handler.test.ts

@@ -249,6 +249,8 @@ describe("handler dispatch", () => {
       "PLAN_TOO_LARGE",
       "PLAN_PROTOCOL_UNSUPPORTED",
       "PLAN_V2_INTEGRITY_UNRECOVERABLE",
+      "FONT_FETCH_FAILED",
+      "FONT_FETCH_UNAVAILABLE",
       "VIDEO_SOURCE_UNRENDERABLE",
       "VIDEO_EXTRACTION_FAILED",
       "INVALID_VIDEO_METADATA",

packages/aws-lambda/src/handler.ts

@@ -150,6 +150,8 @@ function normalizeTerminalErrorName(error: unknown): void {
     candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" ||
     candidate.code === "PLAN_TOO_LARGE" ||
     candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" ||
+    candidate.code === "FONT_FETCH_FAILED" ||
+    candidate.code === "FONT_FETCH_UNAVAILABLE" ||
     candidate.code === "VIDEO_SOURCE_UNRENDERABLE" ||
     candidate.code === "VIDEO_EXTRACTION_FAILED" ||
     candidate.code === "INVALID_VIDEO_METADATA"

packages/gcp-cloud-run/src/server.test.ts

@@ -578,7 +578,9 @@ describe("createApp HTTP mapping", () => {
   it.each([
     ["VIDEO_SOURCE_UNRENDERABLE", 400],
     ["VIDEO_EXTRACTION_FAILED", 500],
-  ] as const)("routes producer video code %s with HTTP %s", async (code, expectedStatus) => {
+    ["FONT_FETCH_FAILED", 400],
+    ["FONT_FETCH_UNAVAILABLE", 500],
+  ] as const)("routes producer code %s with HTTP %s", async (code, expectedStatus) => {
     const gcs = new FakeGcs();
     await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
     const app = createApp(

packages/gcp-cloud-run/src/server.ts

@@ -183,6 +183,8 @@ function normalizeTerminalErrorName(error: unknown): void {
     candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" ||
     candidate.code === "PLAN_TOO_LARGE" ||
     candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" ||
+    candidate.code === "FONT_FETCH_FAILED" ||
+    candidate.code === "FONT_FETCH_UNAVAILABLE" ||
     candidate.code === "VIDEO_SOURCE_UNRENDERABLE" ||
     candidate.code === "VIDEO_EXTRACTION_FAILED" ||
     candidate.code === "INVALID_VIDEO_METADATA"

packages/producer/src/index.ts

@@ -114,7 +114,13 @@ export { normalizeErrorMessage } from "./utils/errorMessage.js";
 // timing. The render pipeline runs this in its compile stage; the CLI audit
 // paths (snapshot/check) reuse it so their captures match the render.
 export {
+  FONT_FETCH_FAILED,
+  FONT_FETCH_UNAVAILABLE,
+  FontFetchError,
+  FontFetchUnavailableError,
   injectDeterministicFontFaces,
+  type FontFetchErrorCode,
+  type FontFetchRetryPolicy,
   type InjectDeterministicFontFacesOptions,
 } from "./services/deterministicFonts.js";
 export { quantizeTimeToFrame } from "./utils/parityContract.js";

packages/producer/src/services/deterministicFonts-failClosed.test.ts

@@ -17,7 +17,9 @@
 import { describe, expect, it } from "bun:test";
 import {
   FONT_FETCH_FAILED,
+  FONT_FETCH_UNAVAILABLE,
   FontFetchError,
+  FontFetchUnavailableError,
   injectDeterministicFontFaces,
 } from "./deterministicFonts.js";
 
@@ -75,6 +77,15 @@ function makeGoogleFontFetch(cssRequests: string[]): typeof fetch {
   }) as unknown as typeof fetch;
 }
 
+async function rejectedError(promise: Promise<unknown>): Promise<unknown> {
+  try {
+    await promise;
+  } catch (error) {
+    return error;
+  }
+  throw new Error("Expected promise to reject");
+}
+
 describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)", () => {
   it("swallows a network failure and returns the original HTML (no throw)", async () => {
     const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
@@ -141,19 +152,18 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: true", () => {
     });
   }
 
-  it("throws FontFetchError on a network failure", async () => {
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+  it("throws FontFetchUnavailableError after retrying a network failure", async () => {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         fetchImpl: makeFailingFetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+        fontFetchRetryPolicy: { baseDelayMs: 0 },
+      }),
+    );
     expect(caught).toBeInstanceOf(FontFetchError);
-    expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED);
-    expect((caught as FontFetchError).code).toBe("FONT_FETCH_FAILED");
+    expect(caught).toBeInstanceOf(FontFetchUnavailableError);
+    expect((caught as FontFetchError).code).toBe(FONT_FETCH_UNAVAILABLE);
+    expect((caught as FontFetchError).code).toBe("FONT_FETCH_UNAVAILABLE");
     expect((caught as FontFetchError).familyName).toBe("NotARealFontFamilyForTest");
     expect((caught as Error).message).toContain("simulated network failure");
   });
@@ -164,62 +174,53 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: true", () => {
     // unresolvable (no alias, no Google Fonts, no system capture) which IS
     // a fail-closed error — the render would use a fallback font, producing
     // non-deterministic output across machines.
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         allowSystemFontCapture: false,
         fetchImpl: makeHttp400Fetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+      }),
+    );
     expect(caught).toBeInstanceOf(FontFetchError);
     expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED);
     expect((caught as FontFetchError).familyName).toBe("NotARealFontFamilyForTest");
   });
 
   it("throws on a 404 response when font is completely unresolvable", async () => {
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         allowSystemFontCapture: false,
         fetchImpl: makeHttp404Fetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+      }),
+    );
     expect(caught).toBeInstanceOf(FontFetchError);
     expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED);
   });
 
-  it("throws FontFetchError on a 5xx response — non-deterministic, could differ on retry", async () => {
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+  it("throws FontFetchUnavailableError on an exhausted 5xx response", async () => {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         fetchImpl: makeHttp503Fetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+        fontFetchRetryPolicy: { baseDelayMs: 0 },
+      }),
+    );
     expect(caught).toBeInstanceOf(FontFetchError);
-    expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED);
+    expect(caught).toBeInstanceOf(FontFetchUnavailableError);
+    expect((caught as FontFetchError).code).toBe(FONT_FETCH_UNAVAILABLE);
     expect((caught as Error).message).toContain("HTTP 503");
     expect((caught as Error).message).toContain("NotARealFontFamilyForTest");
   });
 
   it("includes the requested URL in 5xx errors", async () => {
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         fetchImpl: makeHttp503Fetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+        fontFetchRetryPolicy: { baseDelayMs: 0 },
+      }),
+    );
     expect((caught as FontFetchError).url).toContain("fonts.googleapis.com");
     expect((caught as FontFetchError).url).toContain("NotARealFontFamilyForTest");
   });

packages/producer/src/services/deterministicFonts-retries.test.ts

@@ -0,0 +1,417 @@
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  FONT_FETCH_FAILED,
+  FONT_FETCH_UNAVAILABLE,
+  FontFetchError,
+  FontFetchUnavailableError,
+  injectDeterministicFontFaces,
+  type FontFetchRetryPolicy,
+} from "./deterministicFonts.js";
+
+let previousCacheDir: string | undefined;
+let testCacheDir: string;
+
+beforeAll(() => {
+  previousCacheDir = process.env.HYPERFRAMES_FONT_CACHE_DIR;
+  testCacheDir = mkdtempSync(join(tmpdir(), "hf-font-retries-"));
+  process.env.HYPERFRAMES_FONT_CACHE_DIR = testCacheDir;
+});
+
+afterAll(() => {
+  if (previousCacheDir === undefined) delete process.env.HYPERFRAMES_FONT_CACHE_DIR;
+  else process.env.HYPERFRAMES_FONT_CACHE_DIR = previousCacheDir;
+  rmSync(testCacheDir, { recursive: true, force: true });
+});
+
+const FAST_RETRY = {
+  maxAttempts: 2,
+  attemptTimeoutMs: 20,
+  maxElapsedMs: 100,
+  baseDelayMs: 0,
+} as const;
+
+let familySequence = 0;
+
+function unresolvedHtml(): { family: string; html: string } {
+  familySequence += 1;
+  const family = `RetryTestFont${process.pid}${familySequence}`;
+  return {
+    family,
+    html: `<!doctype html><html><head><style>
+      body { font-family: "${family}", sans-serif; }
+    </style></head><body>retry</body></html>`,
+  };
+}
+
+function googleCss(woff2Url: string): Response {
+  return new Response(
+    `@font-face {
+      font-style: normal;
+      font-weight: 400;
+      src: url(${woff2Url}) format('woff2');
+    }`,
+    { status: 200 },
+  );
+}
+
+function requestUrl(input: string | URL | Request): string {
+  return input instanceof Request ? input.url : String(input);
+}
+
+function injectWithRetries(
+  html: string,
+  fetchImpl: typeof fetch,
+  fontFetchRetryPolicy: Partial<FontFetchRetryPolicy> = FAST_RETRY,
+): Promise<string> {
+  return injectDeterministicFontFaces(html, {
+    failClosedFontFetch: true,
+    allowSystemFontCapture: false,
+    fetchImpl,
+    fontFetchRetryPolicy,
+  });
+}
+
+describe("deterministic Google Fonts retries", () => {
+  for (const status of [408, 425, 429, 500, 502, 503]) {
+    it(`retries HTTP ${status} exactly once, then reports temporary unavailability`, async () => {
+      const { html } = unresolvedHtml();
+      let calls = 0;
+      const fetchImpl = (async () => {
+        calls += 1;
+        return new Response("", {
+          status,
+          headers: status === 429 ? { "Retry-After": "0" } : undefined,
+        });
+      }) as typeof fetch;
+
+      const result = injectWithRetries(html, fetchImpl);
+
+      await expect(result).rejects.toMatchObject({
+        code: FONT_FETCH_UNAVAILABLE,
+        name: "FontFetchUnavailableError",
+      });
+      expect(calls).toBe(2);
+    });
+  }
+
+  it("recovers when the Google Fonts CSS retry succeeds", async () => {
+    const { html } = unresolvedHtml();
+    const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-css.woff2`;
+    let cssCalls = 0;
+    let woff2Calls = 0;
+    const fetchImpl = (async (input: string | URL | Request) => {
+      if (requestUrl(input).startsWith("https://fonts.googleapis.com/")) {
+        cssCalls += 1;
+        return cssCalls === 1 ? new Response("", { status: 502 }) : googleCss(woff2Url);
+      }
+      woff2Calls += 1;
+      return new Response(new Uint8Array([1, 2, 3]), { status: 200 });
+    }) as typeof fetch;
+
+    const result = await injectWithRetries(html, fetchImpl);
+
+    expect(result).toContain("data-hyperframes-deterministic-fonts");
+    expect(cssCalls).toBe(2);
+    expect(woff2Calls).toBe(1);
+  });
+
+  it("does not wait for a stalled retryable response-body cancellation", async () => {
+    const { html } = unresolvedHtml();
+    const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-cancel.woff2`;
+    let cssCalls = 0;
+    let cancelledBodies = 0;
+    const fetchImpl = (async (input: string | URL | Request) => {
+      if (!requestUrl(input).startsWith("https://fonts.googleapis.com/")) {
+        return new Response(new Uint8Array([10, 11, 12]), { status: 200 });
+      }
+      cssCalls += 1;
+      if (cssCalls === 1) {
+        return new Response(
+          new ReadableStream({
+            cancel() {
+              cancelledBodies += 1;
+              return new Promise<void>(() => {});
+            },
+          }),
+          { status: 502 },
+        );
+      }
+      return googleCss(woff2Url);
+    }) as typeof fetch;
+
+    const result = await injectWithRetries(html, fetchImpl);
+
+    expect(result).toContain("data-hyperframes-deterministic-fonts");
+    expect(cssCalls).toBe(2);
+    expect(cancelledBodies).toBe(1);
+  });
+
+  it("recovers when a woff2 retry succeeds", async () => {
+    const { html } = unresolvedHtml();
+    const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-woff2.woff2`;
+    let cssCalls = 0;
+    let woff2Calls = 0;
+    const fetchImpl = (async (input: string | URL | Request) => {
+      if (requestUrl(input).startsWith("https://fonts.googleapis.com/")) {
+        cssCalls += 1;
+        return googleCss(woff2Url);
+      }
+      woff2Calls += 1;
+      return woff2Calls === 1
+        ? new Response("", { status: 503 })
+        : new Response(new Uint8Array([4, 5, 6]), { status: 200 });
+    }) as typeof fetch;
+
+    const result = await injectWithRetries(html, fetchImpl);
+
+    expect(result).toContain("data-hyperframes-deterministic-fonts");
+    expect(cssCalls).toBe(1);
+    expect(woff2Calls).toBe(2);
+  });
+
+  it("reports temporary unavailability when woff2 retries are exhausted", async () => {
+    const { html } = unresolvedHtml();
+    const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-woff2-exhausted.woff2`;
+    let cssCalls = 0;
+    let woff2Calls = 0;
+    const fetchImpl = (async (input: string | URL | Request) => {
+      if (requestUrl(input).startsWith("https://fonts.googleapis.com/")) {
+        cssCalls += 1;
+        return googleCss(woff2Url);
+      }
+      woff2Calls += 1;
+      return new Response("", { status: 502 });
+    }) as typeof fetch;
+
+    const result = injectWithRetries(html, fetchImpl);
+
+    await expect(result).rejects.toMatchObject({
+      code: FONT_FETCH_UNAVAILABLE,
+      url: woff2Url,
+    });
+    expect(cssCalls).toBe(1);
+    expect(woff2Calls).toBe(2);
+  });
+
+  it("retries transport exceptions and preserves the final cause", async () => {
+    const { html } = unresolvedHtml();
+    let calls = 0;
+    const fetchImpl = (async () => {
+      calls += 1;
+      throw new TypeError(`network failure ${calls}`);
+    }) as typeof fetch;
+
+    let caught: unknown;
+    try {
+      await injectWithRetries(html, fetchImpl);
+    } catch (error) {
+      caught = error;
+    }
+
+    expect(caught).toBeInstanceOf(FontFetchUnavailableError);
+    expect(caught).toBeInstanceOf(FontFetchError);
+    expect((caught as FontFetchUnavailableError).code).toBe(FONT_FETCH_UNAVAILABLE);
+    expect((caught as Error).message).toContain("network failure 2");
+    expect(calls).toBe(2);
+  });
+
+  it("retries a response-body transport failure", async () => {
+    const { html } = unresolvedHtml();
+    const woff2Url = `https://fonts.gstatic.com/s/retry/${process.pid}-body.woff2`;
+    let cssCalls = 0;
+    const fetchImpl = (async (input: string | URL | Request) => {
+      if (!requestUrl(input).startsWith("https://fonts.googleapis.com/")) {
+        return new Response(new Uint8Array([7, 8, 9]), { status: 200 });
+      }
+      cssCalls += 1;
+      if (cssCalls === 1) {
+        return new Response(
+          new ReadableStream({
+            start(controller) {
+              controller.error(new TypeError("CSS body connection reset"));
+            },
+          }),
+          { status: 200 },
+        );
+      }
+      return googleCss(woff2Url);
+    }) as typeof fetch;
+
+    const result = await injectWithRetries(html, fetchImpl);
+
+    expect(result).toContain("data-hyperframes-deterministic-fonts");
+    expect(cssCalls).toBe(2);
+  });
+
+  it("times out each hung attempt and reports temporary unavailability", async () => {
+    const { html } = unresolvedHtml();
+    let calls = 0;
+    const fetchImpl = ((_input: string | URL | Request, init?: RequestInit) => {
+      calls += 1;
+      return new Promise<Response>((_resolve, reject) => {
+        const signal = init?.signal;
+        if (!signal) {
+          reject(new Error("missing timeout signal"));
+          return;
+        }
+        signal.addEventListener("abort", () => reject(signal.reason), { once: true });
+      });
+    }) as typeof fetch;
+
+    const result = injectWithRetries(html, fetchImpl, {
+      ...FAST_RETRY,
+      attemptTimeoutMs: 5,
+      maxElapsedMs: 50,
+    });
+
+    await expect(result).rejects.toMatchObject({ code: FONT_FETCH_UNAVAILABLE });
+    expect(calls).toBe(2);
+  });
+
+  it("does not exceed the shared budget to honor a long Retry-After", async () => {
+    const { html } = unresolvedHtml();
+    let calls = 0;
+    const fetchImpl = (async () => {
+      calls += 1;
+      return new Response("", {
+        status: 429,
+        headers: { "Retry-After": "1" },
+      });
+    }) as typeof fetch;
+
+    const result = injectWithRetries(html, fetchImpl, {
+      ...FAST_RETRY,
+      maxElapsedMs: 100,
+    });
+
+    await expect(result).rejects.toMatchObject({ code: FONT_FETCH_UNAVAILABLE });
+    expect(calls).toBe(1);
+  });
+
+  it("shares one elapsed-time budget across multiple font resources", async () => {
+    const { html } = unresolvedHtml();
+    const firstUrl = `https://fonts.gstatic.com/s/retry/${process.pid}-budget-first.woff2`;
+    const secondUrl = `https://fonts.gstatic.com/s/retry/${process.pid}-budget-second.woff2`;
+    let firstCalls = 0;
+    let secondCalls = 0;
+    const fetchImpl = ((input: string | URL | Request, init?: RequestInit): Promise<Response> => {
+      const url = requestUrl(input);
+      if (url.startsWith("https://fonts.googleapis.com/")) {
+        return Promise.resolve(
+          new Response(
+            `@font-face {
+              font-style: normal;
+              font-weight: 400;
+              src: url(${firstUrl}) format('woff2');
+            }
+            @font-face {
+              font-style: normal;
+              font-weight: 700;
+              src: url(${secondUrl}) format('woff2');
+            }`,
+            { status: 200 },
+          ),
+        );
+      }
+      if (url === firstUrl) {
+        firstCalls += 1;
+        return new Promise<Response>((resolve, reject) => {
+          const timeout = setTimeout(
+            () => resolve(new Response(new Uint8Array([13, 14, 15]), { status: 200 })),
+            30,
+          );
+          init?.signal?.addEventListener(
+            "abort",
+            () => {
+              clearTimeout(timeout);
+              reject(init.signal?.reason);
+            },
+            { once: true },
+          );
+        });
+      }
+      secondCalls += 1;
+      return new Promise<Response>((_resolve, reject) => {
+        init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), {
+          once: true,
+        });
+      });
+    }) as typeof fetch;
+
+    const result = injectWithRetries(html, fetchImpl, {
+      maxAttempts: 2,
+      attemptTimeoutMs: 70,
+      maxElapsedMs: 100,
+      baseDelayMs: 0,
+    });
+
+    await expect(result).rejects.toMatchObject({ code: FONT_FETCH_UNAVAILABLE });
+    expect(firstCalls).toBe(1);
+    expect(secondCalls).toBe(1);
+  });
+
+  it("does not retry an ordinary 4xx and retains deterministic failure classification", async () => {
+    const { html } = unresolvedHtml();
+    let calls = 0;
+    const fetchImpl = (async () => {
+      calls += 1;
+      return new Response("", { status: 400 });
+    }) as typeof fetch;
+
+    const result = injectWithRetries(html, fetchImpl);
+
+    await expect(result).rejects.toMatchObject({
+      code: FONT_FETCH_FAILED,
+      name: "FontFetchError",
+    });
+    expect(calls).toBe(1);
+  });
+
+  it("propagates caller cancellation without wrapping or retrying it", async () => {
+    const { html } = unresolvedHtml();
+    const controller = new AbortController();
+    const cancellation = new Error("render cancelled by caller");
+    let calls = 0;
+    const fetchImpl = ((_input: string | URL | Request, init?: RequestInit) => {
+      calls += 1;
+      return new Promise<Response>((_resolve, reject) => {
+        const signal = init?.signal;
+        signal?.addEventListener("abort", () => reject(signal.reason), { once: true });
+      });
+    }) as typeof fetch;
+
+    const result = injectDeterministicFontFaces(html, {
+      failClosedFontFetch: true,
+      allowSystemFontCapture: false,
+      fetchImpl,
+      abortSignal: controller.signal,
+      fontFetchRetryPolicy: FAST_RETRY,
+    });
+    controller.abort(cancellation);
+
+    await expect(result).rejects.toBe(cancellation);
+    expect(calls).toBe(1);
+  });
+
+  it("keeps fail-open mode at one attempt with no typed failure", async () => {
+    const { html } = unresolvedHtml();
+    let calls = 0;
+    const fetchImpl = (async () => {
+      calls += 1;
+      return new Response("", { status: 503 });
+    }) as typeof fetch;
+
+    const result = await injectDeterministicFontFaces(html, {
+      failClosedFontFetch: false,
+      allowSystemFontCapture: false,
+      fetchImpl,
+      fontFetchRetryPolicy: FAST_RETRY,
+    });
+
+    expect(result).not.toContain("data-hyperframes-deterministic-fonts");
+    expect(calls).toBe(1);
+  });
+});

packages/producer/src/services/deterministicFonts.ts

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { defaultLogger } from "../logger.js";
@@ -581,19 +581,22 @@ function warnUnresolvedFonts(unresolved: string[]): void {
 // Google Fonts on-demand fetch + local cache
 // ---------------------------------------------------------------------------
 
-// On AWS Lambda `$HOME` resolves to a `/home/sbx_*` tree that's
-// read-only; only `/tmp` is writable. Route the cache there when
-// running inside Lambda, and honor `HYPERFRAMES_FONT_CACHE_DIR` as
-// an explicit override for any environment.
+let lambdaFontCacheRoot: string | undefined;
+
+// On AWS Lambda `$HOME` resolves to a `/home/sbx_*` tree that's read-only;
+// only `/tmp` is writable. Create one private, unguessable cache directory per
+// warm process and reuse it across invocations. Honor HYPERFRAMES_FONT_CACHE_DIR
+// as an explicit override for any environment.
 function resolveFontCacheRoot(): string {
-  return (
-    process.env.HYPERFRAMES_FONT_CACHE_DIR ??
-    (process.env.AWS_LAMBDA_FUNCTION_NAME
-      ? join(tmpdir(), "hyperframes", "fonts")
-      : join(homedir(), ".cache", "hyperframes", "fonts"))
-  );
+  if (process.env.HYPERFRAMES_FONT_CACHE_DIR) {
+    return process.env.HYPERFRAMES_FONT_CACHE_DIR;
+  }
+  if (process.env.AWS_LAMBDA_FUNCTION_NAME) {
+    lambdaFontCacheRoot ??= mkdtempSync(join(tmpdir(), "hyperframes-fonts-"));
+    return lambdaFontCacheRoot;
+  }
+  return join(homedir(), ".cache", "hyperframes", "fonts");
 }
-const GOOGLE_FONTS_CACHE_DIR = resolveFontCacheRoot();
 
 // Chrome UA triggers woff2 responses from Google Fonts CSS API
 const WOFF2_USER_AGENT =
@@ -607,7 +610,7 @@ function fontSlug(familyName: string): string {
 }
 
 function fontCacheDir(slug: string): string {
-  const dir = join(GOOGLE_FONTS_CACHE_DIR, slug);
+  const dir = join(resolveFontCacheRoot(), slug);
   if (!existsSync(dir)) {
     mkdirSync(dir, { recursive: true });
   }
@@ -637,37 +640,77 @@ type GoogleFontFace = {
 };
 
 /**
- * Typed code classifying a font-fetch failure as non-retryable for
- * distributed workflow adapters — a missing Google Fonts entry will not heal
- * on retry.
+ * Typed codes let distributed workflow adapters distinguish deterministic
+ * resolution failures from temporary upstream unavailability.
  */
 export const FONT_FETCH_FAILED = "FONT_FETCH_FAILED";
+export const FONT_FETCH_UNAVAILABLE = "FONT_FETCH_UNAVAILABLE";
+export type FontFetchErrorCode = typeof FONT_FETCH_FAILED | typeof FONT_FETCH_UNAVAILABLE;
 
 /**
  * Typed error thrown by {@link injectDeterministicFontFaces} when
- * `failClosedFontFetch === true` and an external font fetch fails. The
- * default (swallow + warn) preserves the in-process behavior.
+ * `failClosedFontFetch === true` and deterministic font resolution fails.
+ * The default (swallow + warn) preserves the in-process behavior.
  */
 export class FontFetchError extends Error {
-  readonly code: typeof FONT_FETCH_FAILED = FONT_FETCH_FAILED;
+  readonly code: FontFetchErrorCode;
   readonly familyName: string;
   readonly url: string;
   readonly cause?: unknown;
 
-  constructor(familyName: string, url: string, message: string, cause?: unknown) {
+  constructor(
+    familyName: string,
+    url: string,
+    message: string,
+    cause?: unknown,
+    code: FontFetchErrorCode = FONT_FETCH_FAILED,
+  ) {
     super(message);
     this.name = "FontFetchError";
+    this.code = code;
     this.familyName = familyName;
     this.url = url;
     this.cause = cause;
   }
 }
 
+/**
+ * Retryable font-fetch failure. Distributed adapters map this code to an
+ * unavailable response so the workflow can retry the plan activity.
+ */
+export class FontFetchUnavailableError extends FontFetchError {
+  constructor(familyName: string, url: string, message: string, cause?: unknown) {
+    super(familyName, url, message, cause, FONT_FETCH_UNAVAILABLE);
+    this.name = "FontFetchUnavailableError";
+  }
+}
+
+export interface FontFetchRetryPolicy {
+  /** Total fetch attempts for one CSS or woff2 URL. Default: 2. */
+  maxAttempts: number;
+  /** Timeout for each individual fetch attempt. Default: 8 seconds. */
+  attemptTimeoutMs: number;
+  /** Shared wall-clock budget for all Google Fonts requests in one compile. Default: 20 seconds. */
+  maxElapsedMs: number;
+  /** Initial full-jitter backoff ceiling. Default: 250 ms. */
+  baseDelayMs: number;
+}
+
+const DEFAULT_FONT_FETCH_RETRY_POLICY: FontFetchRetryPolicy = {
+  maxAttempts: 2,
+  attemptTimeoutMs: 8_000,
+  maxElapsedMs: 20_000,
+  baseDelayMs: 250,
+};
+
 /** Internal threading of the failClosed flag + fetch override through callers. */
 interface InternalFontFetchOptions {
   failClosedFontFetch: boolean;
   fetchImpl: typeof fetch;
   allowSystemFontCapture: boolean;
+  abortSignal?: AbortSignal;
+  retryPolicy: FontFetchRetryPolicy;
+  retryDeadlineMs: number;
 }
 
 /**
@@ -680,16 +723,158 @@ function fontFetchError(
   url: string,
   what: "Google Fonts CSS" | `Google Fonts woff2 (${string}/${string})`,
   cause: { status: number } | { error: unknown },
+  unavailable = false,
 ): FontFetchError {
   const reason =
     "status" in cause
       ? `returned HTTP ${cause.status}`
-      : `failed: ${(cause.error as Error).message}`;
+      : `failed: ${cause.error instanceof Error ? cause.error.message : String(cause.error)}`;
   const message =
     `[deterministicFonts] ${what} fetch for ${JSON.stringify(familyName)} ${reason}. ` +
     `Distributed renders require deterministic fonts; system-font fallback would produce ` +
     `non-byte-identical output.`;
-  return new FontFetchError(familyName, url, message, "error" in cause ? cause.error : undefined);
+  const errorCause = "error" in cause ? cause.error : undefined;
+  return unavailable
+    ? new FontFetchUnavailableError(familyName, url, message, errorCause)
+    : new FontFetchError(familyName, url, message, errorCause);
+}
+
+function isRetryableFontFetchStatus(status: number): boolean {
+  return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+
+function retryAfterMs(response: Response): number | null {
+  const value = response.headers.get("retry-after");
+  if (value === null) return null;
+  const seconds = Number(value);
+  if (Number.isFinite(seconds)) return Math.max(0, seconds * 1_000);
+  const date = Date.parse(value);
+  return Number.isNaN(date) ? null : Math.max(0, date - Date.now());
+}
+
+function callerAbortReason(signal: AbortSignal): unknown {
+  return signal.reason ?? new DOMException("Font fetch cancelled", "AbortError");
+}
+
+function throwIfCallerAborted(signal: AbortSignal | undefined): void {
+  if (signal?.aborted) throw callerAbortReason(signal);
+}
+
+async function waitForFontFetchRetry(
+  delayMs: number,
+  signal: AbortSignal | undefined,
+): Promise<void> {
+  throwIfCallerAborted(signal);
+  if (delayMs <= 0) return;
+  await new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, delayMs);
+    const onAbort = (): void => {
+      clearTimeout(timeout);
+      reject(signal ? callerAbortReason(signal) : new DOMException("Cancelled", "AbortError"));
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+    if (signal?.aborted) onAbort();
+  });
+}
+
+function retryDelayMs(
+  response: Response | undefined,
+  attempt: number,
+  baseDelayMs: number,
+): number {
+  const requestedDelay = response ? retryAfterMs(response) : null;
+  if (requestedDelay !== null) return requestedDelay;
+  const ceiling = baseDelayMs * 2 ** attempt;
+  return Math.floor(Math.random() * (ceiling + 1));
+}
+
+function cancelResponseBody(response: Response): void {
+  try {
+    void response.body?.cancel().catch(() => {
+      // Best-effort connection cleanup must not replace the original status.
+    });
+  } catch {
+    // Best-effort connection cleanup must not replace the original status.
+  }
+}
+
+type FontFetchResult<T> =
+  | { ok: true; response: Response; body: T }
+  | { ok: false; response: Response };
+
+type FontFetchAttemptResult<T> =
+  | { completed: true; result: FontFetchResult<T> }
+  | {
+      completed: false;
+      response?: Response;
+      cause: { status: number } | { error: unknown };
+    };
+
+async function runFontFetchAttempt<T>(
+  url: string,
+  init: RequestInit | undefined,
+  readBody: (response: Response) => Promise<T>,
+  options: InternalFontFetchOptions,
+  remainingMs: number,
+): Promise<FontFetchAttemptResult<T>> {
+  const timeoutSignal = AbortSignal.timeout(
+    Math.max(1, Math.min(options.retryPolicy.attemptTimeoutMs, remainingMs)),
+  );
+  const signal = options.abortSignal
+    ? AbortSignal.any([options.abortSignal, timeoutSignal])
+    : timeoutSignal;
+  let response: Response | undefined;
+  try {
+    response = await options.fetchImpl(url, { ...init, signal });
+    if (!isRetryableFontFetchStatus(response.status)) {
+      if (!response.ok) return { completed: true, result: { ok: false, response } };
+      const body = await readBody(response);
+      return { completed: true, result: { ok: true, response, body } };
+    }
+    cancelResponseBody(response);
+    return { completed: false, response, cause: { status: response.status } };
+  } catch (error) {
+    throwIfCallerAborted(options.abortSignal);
+    return { completed: false, response, cause: { error } };
+  }
+}
+
+async function fetchFontResource<T>(
+  url: string,
+  init: RequestInit | undefined,
+  readBody: (response: Response) => Promise<T>,
+  familyName: string,
+  what: "Google Fonts CSS" | `Google Fonts woff2 (${string}/${string})`,
+  options: InternalFontFetchOptions,
+): Promise<FontFetchResult<T>> {
+  if (!options.failClosedFontFetch) {
+    const response = await options.fetchImpl(url, { ...init, signal: options.abortSignal });
+    if (!response.ok) return { ok: false, response };
+    return { ok: true, response, body: await readBody(response) };
+  }
+
+  let lastCause: { status: number } | { error: unknown } = {
+    error: new DOMException("Font fetch budget exhausted", "TimeoutError"),
+  };
+  for (let attempt = 0; attempt < options.retryPolicy.maxAttempts; attempt += 1) {
+    throwIfCallerAborted(options.abortSignal);
+    const remainingMs = options.retryDeadlineMs - Date.now();
+    if (remainingMs <= 0) break;
+
+    const attemptResult = await runFontFetchAttempt(url, init, readBody, options, remainingMs);
+    if (attemptResult.completed) return attemptResult.result;
+    lastCause = attemptResult.cause;
+
+    if (attempt + 1 >= options.retryPolicy.maxAttempts) break;
+    const delayMs = retryDelayMs(attemptResult.response, attempt, options.retryPolicy.baseDelayMs);
+    if (delayMs >= options.retryDeadlineMs - Date.now()) break;
+    await waitForFontFetchRetry(delayMs, options.abortSignal);
+  }
+
+  throw fontFetchError(familyName, url, what, lastCause, true);
 }
 
 /**
@@ -714,17 +899,20 @@ async function ensureWoff2DataUri(
 
   const woff2What = `Google Fonts woff2 (${weight}/${style})` as const;
   try {
-    const fontRes = await options.fetchImpl(woff2Url);
-    if (!fontRes.ok) {
-      if (fontRes.status >= 500 && options.failClosedFontFetch) {
-        throw fontFetchError(familyName, woff2Url, woff2What, { status: fontRes.status });
-      }
-      return null;
-    }
+    const fontResult = await fetchFontResource(
+      woff2Url,
+      undefined,
+      (response) => response.arrayBuffer(),
+      familyName,
+      woff2What,
+      options,
+    );
+    if (!fontResult.ok) return null;
     // wx = O_CREAT|O_EXCL: atomic create, rejects symlinks, fails with
     // EEXIST if a concurrent call cached it between our read and write.
-    writeFileSync(cachePath, Buffer.from(await fontRes.arrayBuffer()), { flag: "wx", mode: 0o644 });
+    writeFileSync(cachePath, Buffer.from(fontResult.body), { flag: "wx", mode: 0o644 });
   } catch (err) {
+    throwIfCallerAborted(options.abortSignal);
     if (err instanceof FontFetchError) throw err;
     if ((err as NodeJS.ErrnoException).code === "EEXIST") {
       // Concurrent call wrote it — read their result below.
@@ -755,10 +943,15 @@ async function fetchGoogleFont(
 
   let cssText: string;
   try {
-    const res = await options.fetchImpl(url, {
-      headers: { "User-Agent": WOFF2_USER_AGENT },
-    });
-    if (!res.ok) {
+    const cssResult = await fetchFontResource(
+      url,
+      { headers: { "User-Agent": WOFF2_USER_AGENT } },
+      (response) => response.text(),
+      familyName,
+      "Google Fonts CSS",
+      options,
+    );
+    if (!cssResult.ok) {
       // 4xx is a *deterministic* answer from Google Fonts that this
       // family is not served (e.g. HTTP 400 for "Segoe UI", "Arial",
       // "Futura" — names absent from Google's catalog) or is misnamed.
@@ -767,16 +960,14 @@ async function fetchGoogleFont(
       // transient upstream failures) could return faces on retry, which
       // would break the byte-identical-retry contract distributed
       // renders rely on — those still fail closed when requested.
-      if (res.status >= 500 && options.failClosedFontFetch) {
-        throw fontFetchError(familyName, url, "Google Fonts CSS", { status: res.status });
-      }
       return [];
     }
-    cssText = await res.text();
+    cssText = cssResult.body;
   } catch (err) {
     // Rethrow typed error untouched. Network / DNS / fetch-throws are
     // non-deterministic infrastructure failures — wrapped when failClosed
     // is on, swallowed otherwise.
+    throwIfCallerAborted(options.abortSignal);
     if (err instanceof FontFetchError) throw err;
     if (options.failClosedFontFetch) {
       throw fontFetchError(familyName, url, "Google Fonts CSS", { error: err });
@@ -829,16 +1020,16 @@ async function fetchGoogleFont(
  */
 export interface InjectDeterministicFontFacesOptions {
   /**
-   * When `true`, any external font fetch failure (Google Fonts CSS or
-   * woff2) throws {@link FontFetchError} with code `FONT_FETCH_FAILED`.
+   * When `true`, exhausted transient fetch failures throw
+   * {@link FontFetchUnavailableError} with code `FONT_FETCH_UNAVAILABLE`;
+   * deterministic resolution failures retain `FONT_FETCH_FAILED`.
    *
    * Default `false`: failed fetches are silently swallowed; the composition
    * falls back to system fonts via `warnUnresolvedFonts`. This preserves the
    * in-process behavior.
    *
    * Distributed callers pass `true` so font availability is part of the
-   * planDir's content-addressed hash and fetch failures surface as typed
-   * non-retryable errors.
+   * planDir's content-addressed hash and failures surface as typed errors.
    */
   failClosedFontFetch?: boolean;
   /**
@@ -847,6 +1038,13 @@ export interface InjectDeterministicFontFacesOptions {
    * network.
    */
   fetchImpl?: typeof fetch;
+  /** Caller cancellation propagated through fetch attempts and retry waits. */
+  abortSignal?: AbortSignal;
+  /**
+   * Optional retry tuning. Defaults are deliberately bounded for composition
+   * planning; tests may lower delays and timeouts without replacing timers.
+   */
+  fontFetchRetryPolicy?: Partial<FontFetchRetryPolicy>;
   /**
    * When `true` (default for local renders), fonts that aren't resolved by
    * the bundled alias map or Google Fonts are located on the local filesystem,
@@ -873,6 +1071,29 @@ function extractGoogleFontsText(html: string): string | undefined {
     : undefined;
 }
 
+function resolveFontFetchRetryPolicy(
+  configured: Partial<FontFetchRetryPolicy> | undefined,
+): FontFetchRetryPolicy {
+  return {
+    maxAttempts: Math.max(
+      1,
+      Math.floor(configured?.maxAttempts ?? DEFAULT_FONT_FETCH_RETRY_POLICY.maxAttempts),
+    ),
+    attemptTimeoutMs: Math.max(
+      1,
+      configured?.attemptTimeoutMs ?? DEFAULT_FONT_FETCH_RETRY_POLICY.attemptTimeoutMs,
+    ),
+    maxElapsedMs: Math.max(
+      1,
+      configured?.maxElapsedMs ?? DEFAULT_FONT_FETCH_RETRY_POLICY.maxElapsedMs,
+    ),
+    baseDelayMs: Math.max(
+      0,
+      configured?.baseDelayMs ?? DEFAULT_FONT_FETCH_RETRY_POLICY.baseDelayMs,
+    ),
+  };
+}
+
 export async function injectDeterministicFontFaces(
   html: string,
   options: InjectDeterministicFontFacesOptions = {},
@@ -880,10 +1101,14 @@ export async function injectDeterministicFontFaces(
   const failClosedFontFetch = options.failClosedFontFetch === true;
   const fetchImpl = options.fetchImpl ?? fetch;
   const allowSystemFontCapture = options.allowSystemFontCapture !== false;
+  const retryPolicy = resolveFontFetchRetryPolicy(options.fontFetchRetryPolicy);
   const fetchOptions: InternalFontFetchOptions = {
     failClosedFontFetch,
     fetchImpl,
     allowSystemFontCapture,
+    abortSignal: options.abortSignal,
+    retryPolicy,
+    retryDeadlineMs: Date.now() + retryPolicy.maxElapsedMs,
   };
 
   const existingFaces = extractExistingFontFaces(html);

packages/producer/src/services/distributed/plan.ts

@@ -911,6 +911,7 @@ export async function plan(
     needsAlpha,
     log,
     assertNotAborted,
+    abortSignal,
     // Distributed renders fail closed on font-fetch errors so the planDir
     // is content-addressed against deterministic fonts only.
     failClosedFontFetch: config.failClosedFontFetch !== false,

packages/producer/src/services/htmlCompiler.ts

@@ -1714,11 +1714,11 @@ export interface CompileForRenderOptions {
   log?: ProducerLogger;
   /**
    * Threaded through to {@link injectDeterministicFontFaces}. When `true`,
-   * any external font fetch failure throws `FontFetchError` instead of
-   * silently falling back to system fonts. Distributed `plan()` sets this
-   * to `true` so font availability is part of the planDir's content-addressed
-   * hash and fetch failures surface as typed non-retryable errors. Default
-   * `false` preserves the in-process behavior.
+   * deterministic font resolution and exhausted transient fetch failures
+   * surface as typed errors instead of silently falling back to system fonts.
+   * Distributed `plan()` sets this to `true` so font availability is part of
+   * the planDir's content-addressed hash. Default `false` preserves the
+   * in-process behavior.
    */
   failClosedFontFetch?: boolean;
   /**
@@ -1728,6 +1728,8 @@ export interface CompileForRenderOptions {
    * prevent host-specific font capture from leaking into the planDir.
    */
   allowSystemFontCapture?: boolean;
+  /** Caller cancellation propagated through compile-time font fetches. */
+  abortSignal?: AbortSignal;
   /**
    * Optional persistent cache directory for prep-time animated GIF → WebM
    * transcodes. When omitted, the render's downloadDir is used.
@@ -1839,6 +1841,7 @@ export async function compileForRender(
   const coalescedHtml = await injectDeterministicFontFaces(normalizedFontHtml, {
     failClosedFontFetch: options.failClosedFontFetch === true,
     allowSystemFontCapture: options.allowSystemFontCapture,
+    abortSignal: options.abortSignal,
   });
 
   // Download CDN scripts and inline them AFTER coalescing. This order matters:

packages/producer/src/services/render/stages/compileStage.test.ts

@@ -112,6 +112,16 @@ const IFRAME_HTML = `<!doctype html>
 </body>
 </html>`;
 
+const UNRESOLVED_FONT_HTML = `<!doctype html>
+<html>
+<head><style>body { font-family: "CompileAbortFont", sans-serif; }</style></head>
+<body>
+  <div data-composition-id="root" data-width="1920" data-height="1080" data-duration="1">
+    <p>cancelled font compile</p>
+  </div>
+</body>
+</html>`;
+
 // Portrait/square/landscape fixture template — same shape as PLAIN_HTML but
 // with caller-supplied composition dimensions. Consumed by the aspect-agnostic
 // re-map tests below (and reachable from any sibling describe block, unlike a
@@ -232,6 +242,40 @@ describe("runCompileStage — forceScreenshot snapshot", () => {
   });
 });
 
+describe("runCompileStage — font fetch cancellation", () => {
+  let fixture: CompileFixture | null = null;
+
+  afterEach(() => {
+    fixture?.cleanup();
+    fixture = null;
+  });
+
+  it("propagates the caller abort signal through compileForRender to font fetching", async () => {
+    fixture = setupFixture(UNRESOLVED_FONT_HTML);
+    const projectDir = join(fixture.workDir, "project");
+    const controller = new AbortController();
+    const cancellation = new Error("cancel distributed compile");
+    controller.abort(cancellation);
+
+    const result = runCompileStage({
+      projectDir,
+      workDir: fixture.workDir,
+      htmlPath: fixture.htmlPath,
+      entryFile: "index.html",
+      job: createJob(),
+      cfg: createCfg(),
+      needsAlpha: false,
+      log: noopLog,
+      assertNotAborted: () => {},
+      abortSignal: controller.signal,
+      failClosedFontFetch: true,
+      allowSystemFontCapture: false,
+    });
+
+    await expect(result).rejects.toBe(cancellation);
+  });
+});
+
 /**
  * Tests for the aspect-agnostic --resolution re-map in `runCompileStage`.
  *

packages/producer/src/services/render/stages/compileStage.ts

@@ -71,12 +71,14 @@ export interface CompileStageInput {
   log: ProducerLogger;
   /** Cooperative-cancellation probe; throws `RenderCancelledError` when aborted. */
   assertNotAborted: () => void;
+  /** Caller cancellation propagated through compile-time network requests. */
+  abortSignal?: AbortSignal;
   /**
    * When `true`, `compileForRender` threads through to
-   * `injectDeterministicFontFaces` and any external font fetch failure
-   * throws `FontFetchError` instead of silently falling back to system
-   * fonts. Distributed `plan()` passes `true`; the in-process renderer
-   * leaves it `undefined` to preserve current behavior.
+   * `injectDeterministicFontFaces` so deterministic resolution and exhausted
+   * transient fetch failures surface as typed errors instead of silently
+   * falling back to system fonts. Distributed `plan()` passes `true`; the
+   * in-process renderer leaves it `undefined` to preserve current behavior.
    */
   failClosedFontFetch?: boolean;
   /**
@@ -158,6 +160,7 @@ export async function runCompileStage(input: CompileStageInput): Promise<Compile
     assertNotAborted,
     failClosedFontFetch,
     allowSystemFontCapture,
+    abortSignal,
   } = input;
 
   const compileStart = Date.now();
@@ -165,6 +168,7 @@ export async function runCompileStage(input: CompileStageInput): Promise<Compile
     log,
     failClosedFontFetch: failClosedFontFetch === true,
     allowSystemFontCapture,
+    abortSignal,
     variables: input.variables,
     animatedGifCacheDir: cfg.extractCacheDir
       ? join(cfg.extractCacheDir, "animated-gif")