fix(cli): recover transient publish failures

2026-09-05 00:56:23 +00:00 · 2026-08-07 14:53:20 +00:00
parent c44371a298
commit f2d6ce3245
2 changed files with 242 additions and 42 deletions
@@ -146,6 +146,12 @@ function directFetch(completeData?: Record<string, unknown>) {
    .mockResolvedValueOnce(publishedResponse(completeData));
 }

+function networkFailure(code: string, message: string): TypeError {
+  return new TypeError("fetch failed", {
+    cause: Object.assign(new Error(message), { code }),
+  });
+}
+
 /** Asserts the Nth fetch call, always requiring an AbortSignal alongside the given init. */
 function expectFetchCall(
  fetchMock: ReturnType<typeof vi.fn>,
@@ -729,6 +735,125 @@ describe("publishProjectArchive", () => {
      rmSync(dir, { recursive: true, force: true });
    }
  });
+
+  it("retries a transient presigned upload network failure once", async () => {
+    const dir = makeProjectDir();
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(uploadResponse())
+      .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected"))
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+      .mockResolvedValueOnce(publishedResponse());
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      const result = await publishProjectArchive(dir);
+
+      expect(result.projectId).toBe("hfp_123");
+      expect(fetchMock).toHaveBeenCalledTimes(4);
+      expect(fetchMock.mock.calls[1]![0]).toBe("https://s3.example.com/upload");
+      expect(fetchMock.mock.calls[2]![0]).toBe("https://s3.example.com/upload");
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("reports the upload stage and transport cause after the retry is exhausted", async () => {
+    const dir = makeProjectDir();
+    const signedUrl =
+      "https://s3.example.com/upload?X-Amz-Credential=secret&X-Amz-Signature=do-not-print";
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(uploadResponse({ upload_url: signedUrl }))
+      .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com"))
+      .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com"));
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      const promise = publishProjectArchive(dir);
+      await expect(promise).rejects.toThrow(
+        "Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN: getaddrinfo EAI_AGAIN s3.example.com)",
+      );
+      await expect(promise).rejects.not.toThrow("do-not-print");
+      expect(fetchMock).toHaveBeenCalledTimes(3);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("reports the prepare stage and explains disabled Node proxy support", async () => {
+    const dir = makeProjectDir();
+    vi.stubEnv("HTTPS_PROXY", "http://proxy.example.com:8080");
+    vi.stubEnv("NODE_USE_ENV_PROXY", "");
+    const fetchMock = vi
+      .fn()
+      .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable"))
+      .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable"));
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      await expect(publishProjectArchive(dir)).rejects.toThrow(
+        "Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set, but Node fetch proxy support is disabled; retry with NODE_USE_ENV_PROXY=1",
+      );
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("retries a transient prepare-upload network failure once", async () => {
+    const dir = makeProjectDir();
+    const fetchMock = vi
+      .fn()
+      .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure"))
+      .mockResolvedValueOnce(uploadResponse())
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+      .mockResolvedValueOnce(publishedResponse());
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      const result = await publishProjectArchive(dir);
+
+      expect(result.projectId).toBe("hfp_123");
+      expect(fetchMock).toHaveBeenCalledTimes(4);
+      expect(fetchMock.mock.calls[0]![0]).toBe(
+        "https://api2.heygen.com/v1/hyperframes/projects/publish/upload",
+      );
+      expect(fetchMock.mock.calls[1]![0]).toBe(
+        "https://api2.heygen.com/v1/hyperframes/projects/publish/upload",
+      );
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("reports the finalize stage and transport cause", async () => {
+    const dir = makeProjectDir();
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(uploadResponse())
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+      .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected"));
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      await expect(publishProjectArchive(dir)).rejects.toThrow(
+        "Failed to finalize project publish: fetch failed (ECONNRESET: socket disconnected)",
+      );
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
 });

 /** Staged flow returning an already-owned, already-claimed stable project. */
@@ -15,6 +15,7 @@ const DEFAULT_PROJECT_IGNORE = ["/renders/", "/snapshots/"];
 const PUBLISH_CONTENT_TYPE = "application/zip";
 const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
 const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
+const PUBLISH_TRANSPORT_ATTEMPTS = 2;
 // Conservative floor — most connections are faster, but this prevents
 // premature aborts on slow/unstable networks (hotel wifi, tethering).
 const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000;
@@ -166,6 +167,62 @@ async function readErrorMessage(response: Response, fallback: string): Promise<s
  return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
 }

+function errorCode(value: unknown): string | null {
+  if (!isRecord(value)) return null;
+  return typeof value["code"] === "string" ? value["code"] : null;
+}
+
+function redactUrlQuery(message: string): string {
+  return message.replace(/(https?:\/\/[^\s?]+)\?[^\s]+/gu, "$1?[redacted]");
+}
+
+function proxySupportHint(): string {
+  const proxyConfigured = ["HTTPS_PROXY", "HTTP_PROXY", "https_proxy", "http_proxy"].some((key) =>
+    Boolean(process.env[key]?.trim()),
+  );
+  const proxyEnabled =
+    process.env["NODE_USE_ENV_PROXY"] === "1" ||
+    process.execArgv.includes("--use-env-proxy") ||
+    process.env["NODE_OPTIONS"]?.split(/\s+/u).includes("--use-env-proxy") === true;
+  if (!proxyConfigured || proxyEnabled) return "";
+  return (
+    ". Proxy variables are set, but Node fetch proxy support is disabled; retry with " +
+    "NODE_USE_ENV_PROXY=1 (Node 22.21+)"
+  );
+}
+
+function describeFetchFailure(error: unknown): string {
+  const message = error instanceof Error ? error.message : String(error);
+  const cause = error instanceof Error ? error.cause : undefined;
+  const causeMessage = cause instanceof Error ? cause.message : "";
+  const code = errorCode(cause) ?? errorCode(error);
+  const detail = [code, causeMessage && causeMessage !== message ? causeMessage : ""]
+    .filter(Boolean)
+    .join(": ");
+  return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`;
+}
+
+async function fetchForPublish(
+  input: string,
+  createInit: () => RequestInit,
+  failureStage: string,
+  attempts = 1,
+): Promise<Response> {
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      return await fetch(input, createInit());
+    } catch (error) {
+      if (attempt === attempts) {
+        const attemptDetail = attempts > 1 ? ` after ${attempts} attempts` : "";
+        throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(error)}`, {
+          cause: error instanceof Error ? error : undefined,
+        });
+      }
+    }
+  }
+  throw new Error(failureStage);
+}
+
 export function uploadTimeoutMs(byteLength: number): number {
  return Math.max(
    PUBLISH_UPLOAD_MIN_TIMEOUT_MS,
@@ -483,12 +540,16 @@ async function publishProjectArchiveDirect(
    heygen_route: "canary",
  };

-  const response = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish`, {
-    method: "POST",
-    body,
-    headers,
-    signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
-  });
+  const response = await fetchForPublish(
+    `${apiBaseUrl}/v1/hyperframes/projects/publish`,
+    () => ({
+      method: "POST",
+      body,
+      headers,
+      signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
+    }),
+    "Failed to publish project",
+  );

  const payload = await readJson(response);
  const publishedProject = parsePublishedProjectResponse(payload);
@@ -504,14 +565,19 @@ async function uploadArchiveToPresignedUrl(
  archive: PublishArchiveResult,
 ): Promise<void> {
  const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS;
-  const s3Response = await fetch(stagedUpload.uploadUrl, {
-    method: "PUT",
-    body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
-    headers: stagedUpload.uploadHeaders,
-    signal: AbortSignal.timeout(
-      Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
-    ),
-  });
+  const s3Response = await fetchForPublish(
+    stagedUpload.uploadUrl,
+    () => ({
+      method: "PUT",
+      body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
+      headers: stagedUpload.uploadHeaders,
+      signal: AbortSignal.timeout(
+        Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
+      ),
+    }),
+    "Failed to upload project archive",
+    PUBLISH_TRANSPORT_ATTEMPTS,
+  );
  if (!s3Response.ok) {
    throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive"));
  }
@@ -526,20 +592,25 @@ async function publishProjectArchiveStaged(
  projectId: string | undefined,
 ): Promise<PublishedProjectResponse | null> {
  const fileName = `${title}.zip`;
-  const uploadResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, {
-    method: "POST",
-    body: JSON.stringify({
-      file_name: fileName,
-      content_type: PUBLISH_CONTENT_TYPE,
-      content_length: archive.buffer.byteLength,
+  const uploadResponse = await fetchForPublish(
+    `${apiBaseUrl}/v1/hyperframes/projects/publish/upload`,
+    () => ({
+      method: "POST",
+      body: JSON.stringify({
+        file_name: fileName,
+        content_type: PUBLISH_CONTENT_TYPE,
+        content_length: archive.buffer.byteLength,
+      }),
+      headers: {
+        ...authHeaders,
+        "content-type": "application/json",
+        heygen_route: "canary",
+      },
+      signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
    }),
-    headers: {
-      ...authHeaders,
-      "content-type": "application/json",
-      heygen_route: "canary",
-    },
-    signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
-  });
+    "Failed to prepare project upload",
+    PUBLISH_TRANSPORT_ATTEMPTS,
+  );

  if (uploadResponse.status === 404 || uploadResponse.status === 405) {
    return null;
@@ -553,22 +624,26 @@ async function publishProjectArchiveStaged(

  await uploadArchiveToPresignedUrl(stagedUpload, archive);

-  const completeResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, {
-    method: "POST",
-    body: JSON.stringify({
-      upload_key: stagedUpload.uploadKey,
-      file_name: fileName,
-      title,
-      ...(isPublic ? { is_public: true } : {}),
-      ...(projectId ? { project_id: projectId } : {}),
+  const completeResponse = await fetchForPublish(
+    `${apiBaseUrl}/v1/hyperframes/projects/publish/complete`,
+    () => ({
+      method: "POST",
+      body: JSON.stringify({
+        upload_key: stagedUpload.uploadKey,
+        file_name: fileName,
+        title,
+        ...(isPublic ? { is_public: true } : {}),
+        ...(projectId ? { project_id: projectId } : {}),
+      }),
+      headers: {
+        ...authHeaders,
+        "content-type": "application/json",
+        heygen_route: "canary",
+      },
+      signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
    }),
-    headers: {
-      ...authHeaders,
-      "content-type": "application/json",
-      heygen_route: "canary",
-    },
-    signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
-  });
+    "Failed to finalize project publish",
+  );

  const completePayload = await readJson(completeResponse);
  const publishedProject = parsePublishedProjectResponse(completePayload);