Merge pull request #3087 from heygen-com/fix/publish-fetch-failed

fix(cli): recover transient publish failures
2026-11-15 13:58:20 +00:00 · 2026-08-07 08:29:21 -07:00
parent c44371a298 dd629697d3
commit b043e072b1
2 changed files with 333 additions and 42 deletions
@@ -146,6 +146,16 @@ function directFetch(completeData?: Record<string, unknown>) {
    .mockResolvedValueOnce(publishedResponse(completeData));
 }

+function networkFailure(
+  code: string,
+  message: string,
+  metadata: Record<string, unknown> = {},
+): TypeError {
+  return new TypeError("fetch failed", {
+    cause: Object.assign(new Error(message), { code, ...metadata }),
+  });
+}
+
 /** Asserts the Nth fetch call, always requiring an AbortSignal alongside the given init. */
 function expectFetchCall(
  fetchMock: ReturnType<typeof vi.fn>,
@@ -729,6 +739,190 @@ describe("publishProjectArchive", () => {
      rmSync(dir, { recursive: true, force: true });
    }
  });
+
+  it("retries a transient presigned upload network failure once", async () => {
+    const dir = makeProjectDir();
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(uploadResponse())
+      .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected"))
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+      .mockResolvedValueOnce(publishedResponse());
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      const result = await publishProjectArchive(dir);
+
+      expect(result.projectId).toBe("hfp_123");
+      expect(fetchMock).toHaveBeenCalledTimes(4);
+      expect(fetchMock.mock.calls[1]![0]).toBe("https://s3.example.com/upload");
+      expect(fetchMock.mock.calls[2]![0]).toBe("https://s3.example.com/upload");
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("waits briefly before retrying a transport failure", async () => {
+    vi.useFakeTimers();
+    const dir = makeProjectDir();
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(uploadResponse())
+      .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure"))
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+      .mockResolvedValueOnce(publishedResponse());
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      const publish = publishProjectArchive(dir);
+      await vi.advanceTimersByTimeAsync(0);
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+
+      await vi.advanceTimersByTimeAsync(199);
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+
+      await vi.advanceTimersByTimeAsync(1);
+      await expect(publish).resolves.toMatchObject({ projectId: "hfp_123" });
+      expect(fetchMock).toHaveBeenCalledTimes(4);
+    } finally {
+      await vi.runAllTimersAsync();
+      vi.useRealTimers();
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("reports the upload stage and transport cause after the retry is exhausted", async () => {
+    const dir = makeProjectDir();
+    const signedUrl =
+      "https://s3.example.com/upload?X-Amz-Credential=secret&X-Amz-Signature=do-not-print";
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(uploadResponse({ upload_url: signedUrl }))
+      .mockRejectedValueOnce(
+        networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com", {
+          errno: -3001,
+          syscall: "getaddrinfo",
+        }),
+      )
+      .mockRejectedValueOnce(
+        networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com", {
+          errno: -3001,
+          syscall: "getaddrinfo",
+        }),
+      );
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      const promise = publishProjectArchive(dir);
+      await expect(promise).rejects.toThrow(
+        "Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN, syscall=getaddrinfo, errno=-3001: getaddrinfo EAI_AGAIN s3.example.com)",
+      );
+      await expect(promise).rejects.not.toThrow("do-not-print");
+      expect(fetchMock).toHaveBeenCalledTimes(3);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("reports the prepare stage and explains disabled Node proxy support", async () => {
+    const dir = makeProjectDir();
+    vi.stubEnv("HTTPS_PROXY", "http://proxy.example.com:8080");
+    vi.stubEnv("NODE_USE_ENV_PROXY", "");
+    const fetchMock = vi
+      .fn()
+      .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable"))
+      .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable"));
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      await expect(publishProjectArchive(dir)).rejects.toThrow(
+        "Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set but ignored by Node fetch; if this network requires them, retry with NODE_USE_ENV_PROXY=1",
+      );
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("retries a transient prepare-upload network failure once", async () => {
+    const dir = makeProjectDir();
+    const fetchMock = vi
+      .fn()
+      .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure"))
+      .mockResolvedValueOnce(uploadResponse())
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+      .mockResolvedValueOnce(publishedResponse());
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      const result = await publishProjectArchive(dir);
+
+      expect(result.projectId).toBe("hfp_123");
+      expect(fetchMock).toHaveBeenCalledTimes(4);
+      expect(fetchMock.mock.calls[0]![0]).toBe(
+        "https://api2.heygen.com/v1/hyperframes/projects/publish/upload",
+      );
+      expect(fetchMock.mock.calls[1]![0]).toBe(
+        "https://api2.heygen.com/v1/hyperframes/projects/publish/upload",
+      );
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("does not retry a request that reached its timeout", async () => {
+    const dir = makeProjectDir();
+    const fetchMock = vi
+      .fn()
+      .mockRejectedValueOnce(
+        new DOMException("The operation was aborted due to timeout", "TimeoutError"),
+      )
+      .mockResolvedValueOnce(uploadResponse())
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+      .mockResolvedValueOnce(publishedResponse());
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      await expect(publishProjectArchive(dir)).rejects.toThrow(
+        "Failed to prepare project upload: The operation was aborted due to timeout",
+      );
+      expect(fetchMock).toHaveBeenCalledTimes(1);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("reports the finalize stage and transport cause", async () => {
+    const dir = makeProjectDir();
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(uploadResponse())
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+      .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected"));
+    vi.stubGlobal("fetch", fetchMock);
+
+    try {
+      writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
+
+      await expect(publishProjectArchive(dir)).rejects.toThrow(
+        "Failed to finalize project publish: fetch failed (ECONNRESET: socket disconnected)",
+      );
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
 });

 /** Staged flow returning an already-owned, already-claimed stable project. */
@@ -15,6 +15,8 @@ const DEFAULT_PROJECT_IGNORE = ["/renders/", "/snapshots/"];
 const PUBLISH_CONTENT_TYPE = "application/zip";
 const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
 const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
+const PUBLISH_TRANSPORT_ATTEMPTS = 2;
+const PUBLISH_RETRY_DELAY_MS = 200;
 // Conservative floor — most connections are faster, but this prevents
 // premature aborts on slow/unstable networks (hotel wifi, tethering).
 const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000;
@@ -166,6 +168,83 @@ async function readErrorMessage(response: Response, fallback: string): Promise<s
  return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
 }

+function systemErrorMetadata(value: unknown): string[] {
+  if (!isRecord(value)) return [];
+  const metadata: string[] = [];
+  if (typeof value["code"] === "string") metadata.push(value["code"]);
+  if (typeof value["syscall"] === "string") metadata.push(`syscall=${value["syscall"]}`);
+  if (typeof value["errno"] === "string" || typeof value["errno"] === "number") {
+    metadata.push(`errno=${value["errno"]}`);
+  }
+  return metadata;
+}
+
+function redactUrlQuery(message: string): string {
+  return message.replace(/(https?:\/\/[^\s?]+)\?[^\s]+/gu, "$1?[redacted]");
+}
+
+function proxySupportHint(): string {
+  const proxyConfigured = ["HTTPS_PROXY", "HTTP_PROXY", "https_proxy", "http_proxy"].some((key) =>
+    Boolean(process.env[key]?.trim()),
+  );
+  const proxyEnabled =
+    process.env["NODE_USE_ENV_PROXY"] === "1" ||
+    process.execArgv.includes("--use-env-proxy") ||
+    process.env["NODE_OPTIONS"]?.split(/\s+/u).includes("--use-env-proxy") === true;
+  if (!proxyConfigured || proxyEnabled) return "";
+  return (
+    ". Proxy variables are set but ignored by Node fetch; if this network requires them, retry with " +
+    "NODE_USE_ENV_PROXY=1 (Node 22.21+)"
+  );
+}
+
+function describeFetchFailure(error: unknown): string {
+  const message = error instanceof Error ? error.message : String(error);
+  const cause = error instanceof Error ? error.cause : undefined;
+  const causeMessage = cause instanceof Error ? cause.message : "";
+  const metadata = [...systemErrorMetadata(cause), ...systemErrorMetadata(error)].filter(
+    (value, index, all) => all.indexOf(value) === index,
+  );
+  const distinctCauseMessage = causeMessage && causeMessage !== message ? causeMessage : "";
+  const detail = [metadata.join(", "), distinctCauseMessage].filter(Boolean).join(": ");
+  return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`;
+}
+
+function isRequestTimeout(error: unknown): boolean {
+  return (
+    error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError")
+  );
+}
+
+function waitBeforePublishRetry(): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, PUBLISH_RETRY_DELAY_MS));
+}
+
+async function fetchForPublish(
+  input: string,
+  createInit: () => RequestInit,
+  failureStage: string,
+  attempts = 1,
+): Promise<Response> {
+  if (attempts < 1) throw new RangeError("Publish fetch attempts must be at least 1");
+  let lastError: unknown;
+  let attemptsMade = 0;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    attemptsMade = attempt;
+    try {
+      return await fetch(input, createInit());
+    } catch (error) {
+      lastError = error;
+      if (isRequestTimeout(error) || attempt === attempts) break;
+      await waitBeforePublishRetry();
+    }
+  }
+  const attemptDetail = attemptsMade > 1 ? ` after ${attemptsMade} attempts` : "";
+  throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(lastError)}`, {
+    cause: lastError instanceof Error ? lastError : undefined,
+  });
+}
+
 export function uploadTimeoutMs(byteLength: number): number {
  return Math.max(
    PUBLISH_UPLOAD_MIN_TIMEOUT_MS,
@@ -483,12 +562,16 @@ async function publishProjectArchiveDirect(
    heygen_route: "canary",
  };

-  const response = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish`, {
-    method: "POST",
-    body,
-    headers,
-    signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
-  });
+  const response = await fetchForPublish(
+    `${apiBaseUrl}/v1/hyperframes/projects/publish`,
+    () => ({
+      method: "POST",
+      body,
+      headers,
+      signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
+    }),
+    "Failed to publish project",
+  );

  const payload = await readJson(response);
  const publishedProject = parsePublishedProjectResponse(payload);
@@ -504,14 +587,19 @@ async function uploadArchiveToPresignedUrl(
  archive: PublishArchiveResult,
 ): Promise<void> {
  const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS;
-  const s3Response = await fetch(stagedUpload.uploadUrl, {
-    method: "PUT",
-    body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
-    headers: stagedUpload.uploadHeaders,
-    signal: AbortSignal.timeout(
-      Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
-    ),
-  });
+  const s3Response = await fetchForPublish(
+    stagedUpload.uploadUrl,
+    () => ({
+      method: "PUT",
+      body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
+      headers: stagedUpload.uploadHeaders,
+      signal: AbortSignal.timeout(
+        Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
+      ),
+    }),
+    "Failed to upload project archive",
+    PUBLISH_TRANSPORT_ATTEMPTS,
+  );
  if (!s3Response.ok) {
    throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive"));
  }
@@ -526,20 +614,25 @@ async function publishProjectArchiveStaged(
  projectId: string | undefined,
 ): Promise<PublishedProjectResponse | null> {
  const fileName = `${title}.zip`;
-  const uploadResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, {
-    method: "POST",
-    body: JSON.stringify({
-      file_name: fileName,
-      content_type: PUBLISH_CONTENT_TYPE,
-      content_length: archive.buffer.byteLength,
+  const uploadResponse = await fetchForPublish(
+    `${apiBaseUrl}/v1/hyperframes/projects/publish/upload`,
+    () => ({
+      method: "POST",
+      body: JSON.stringify({
+        file_name: fileName,
+        content_type: PUBLISH_CONTENT_TYPE,
+        content_length: archive.buffer.byteLength,
+      }),
+      headers: {
+        ...authHeaders,
+        "content-type": "application/json",
+        heygen_route: "canary",
+      },
+      signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
    }),
-    headers: {
-      ...authHeaders,
-      "content-type": "application/json",
-      heygen_route: "canary",
-    },
-    signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
-  });
+    "Failed to prepare project upload",
+    PUBLISH_TRANSPORT_ATTEMPTS,
+  );

  if (uploadResponse.status === 404 || uploadResponse.status === 405) {
    return null;
@@ -553,22 +646,26 @@ async function publishProjectArchiveStaged(

  await uploadArchiveToPresignedUrl(stagedUpload, archive);

-  const completeResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, {
-    method: "POST",
-    body: JSON.stringify({
-      upload_key: stagedUpload.uploadKey,
-      file_name: fileName,
-      title,
-      ...(isPublic ? { is_public: true } : {}),
-      ...(projectId ? { project_id: projectId } : {}),
+  const completeResponse = await fetchForPublish(
+    `${apiBaseUrl}/v1/hyperframes/projects/publish/complete`,
+    () => ({
+      method: "POST",
+      body: JSON.stringify({
+        upload_key: stagedUpload.uploadKey,
+        file_name: fileName,
+        title,
+        ...(isPublic ? { is_public: true } : {}),
+        ...(projectId ? { project_id: projectId } : {}),
+      }),
+      headers: {
+        ...authHeaders,
+        "content-type": "application/json",
+        heygen_route: "canary",
+      },
+      signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
    }),
-    headers: {
-      ...authHeaders,
-      "content-type": "application/json",
-      heygen_route: "canary",
-    },
-    signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
-  });
+    "Failed to finalize project publish",
+  );

  const completePayload = await readJson(completeResponse);
  const publishedProject = parsePublishedProjectResponse(completePayload);