From f2d6ce32459fc4c6be604a152e8707df03ce96e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 7 Aug 2026 14:53:20 +0000 Subject: [PATCH 1/2] fix(cli): recover transient publish failures --- packages/cli/src/utils/publishProject.test.ts | 125 ++++++++++++++ packages/cli/src/utils/publishProject.ts | 159 +++++++++++++----- 2 files changed, 242 insertions(+), 42 deletions(-) diff --git a/packages/cli/src/utils/publishProject.test.ts b/packages/cli/src/utils/publishProject.test.ts index 65255cc57a..81227cb24d 100644 --- a/packages/cli/src/utils/publishProject.test.ts +++ b/packages/cli/src/utils/publishProject.test.ts @@ -146,6 +146,12 @@ function directFetch(completeData?: Record) { .mockResolvedValueOnce(publishedResponse(completeData)); } +function networkFailure(code: string, message: string): TypeError { + return new TypeError("fetch failed", { + cause: Object.assign(new Error(message), { code }), + }); +} + /** Asserts the Nth fetch call, always requiring an AbortSignal alongside the given init. */ function expectFetchCall( fetchMock: ReturnType, @@ -729,6 +735,125 @@ describe("publishProjectArchive", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it("retries a transient presigned upload network failure once", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse()) + .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected")) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const result = await publishProjectArchive(dir); + + expect(result.projectId).toBe("hfp_123"); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(fetchMock.mock.calls[1]![0]).toBe("https://s3.example.com/upload"); + expect(fetchMock.mock.calls[2]![0]).toBe("https://s3.example.com/upload"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the upload stage and transport cause after the retry is exhausted", async () => { + const dir = makeProjectDir(); + const signedUrl = + "https://s3.example.com/upload?X-Amz-Credential=secret&X-Amz-Signature=do-not-print"; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse({ upload_url: signedUrl })) + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com")) + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com")); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const promise = publishProjectArchive(dir); + await expect(promise).rejects.toThrow( + "Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN: getaddrinfo EAI_AGAIN s3.example.com)", + ); + await expect(promise).rejects.not.toThrow("do-not-print"); + expect(fetchMock).toHaveBeenCalledTimes(3); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the prepare stage and explains disabled Node proxy support", async () => { + const dir = makeProjectDir(); + vi.stubEnv("HTTPS_PROXY", "http://proxy.example.com:8080"); + vi.stubEnv("NODE_USE_ENV_PROXY", ""); + const fetchMock = vi + .fn() + .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable")) + .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable")); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + await expect(publishProjectArchive(dir)).rejects.toThrow( + "Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set, but Node fetch proxy support is disabled; retry with NODE_USE_ENV_PROXY=1", + ); + expect(fetchMock).toHaveBeenCalledTimes(2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("retries a transient prepare-upload network failure once", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure")) + .mockResolvedValueOnce(uploadResponse()) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const result = await publishProjectArchive(dir); + + expect(result.projectId).toBe("hfp_123"); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(fetchMock.mock.calls[0]![0]).toBe( + "https://api2.heygen.com/v1/hyperframes/projects/publish/upload", + ); + expect(fetchMock.mock.calls[1]![0]).toBe( + "https://api2.heygen.com/v1/hyperframes/projects/publish/upload", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the finalize stage and transport cause", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse()) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected")); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + await expect(publishProjectArchive(dir)).rejects.toThrow( + "Failed to finalize project publish: fetch failed (ECONNRESET: socket disconnected)", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); /** Staged flow returning an already-owned, already-claimed stable project. */ diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index f9bf5f4851..4aaee4fa6a 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -15,6 +15,7 @@ const DEFAULT_PROJECT_IGNORE = ["/renders/", "/snapshots/"]; const PUBLISH_CONTENT_TYPE = "application/zip"; const PUBLISH_METADATA_TIMEOUT_MS = 30_000; const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000; +const PUBLISH_TRANSPORT_ATTEMPTS = 2; // Conservative floor — most connections are faster, but this prevents // premature aborts on slow/unstable networks (hotel wifi, tethering). const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000; @@ -166,6 +167,62 @@ async function readErrorMessage(response: Response, fallback: string): Promise + Boolean(process.env[key]?.trim()), + ); + const proxyEnabled = + process.env["NODE_USE_ENV_PROXY"] === "1" || + process.execArgv.includes("--use-env-proxy") || + process.env["NODE_OPTIONS"]?.split(/\s+/u).includes("--use-env-proxy") === true; + if (!proxyConfigured || proxyEnabled) return ""; + return ( + ". Proxy variables are set, but Node fetch proxy support is disabled; retry with " + + "NODE_USE_ENV_PROXY=1 (Node 22.21+)" + ); +} + +function describeFetchFailure(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + const cause = error instanceof Error ? error.cause : undefined; + const causeMessage = cause instanceof Error ? cause.message : ""; + const code = errorCode(cause) ?? errorCode(error); + const detail = [code, causeMessage && causeMessage !== message ? causeMessage : ""] + .filter(Boolean) + .join(": "); + return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`; +} + +async function fetchForPublish( + input: string, + createInit: () => RequestInit, + failureStage: string, + attempts = 1, +): Promise { + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + return await fetch(input, createInit()); + } catch (error) { + if (attempt === attempts) { + const attemptDetail = attempts > 1 ? ` after ${attempts} attempts` : ""; + throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(error)}`, { + cause: error instanceof Error ? error : undefined, + }); + } + } + } + throw new Error(failureStage); +} + export function uploadTimeoutMs(byteLength: number): number { return Math.max( PUBLISH_UPLOAD_MIN_TIMEOUT_MS, @@ -483,12 +540,16 @@ async function publishProjectArchiveDirect( heygen_route: "canary", }; - const response = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish`, { - method: "POST", - body, - headers, - signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), - }); + const response = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish`, + () => ({ + method: "POST", + body, + headers, + signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), + }), + "Failed to publish project", + ); const payload = await readJson(response); const publishedProject = parsePublishedProjectResponse(payload); @@ -504,14 +565,19 @@ async function uploadArchiveToPresignedUrl( archive: PublishArchiveResult, ): Promise { const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS; - const s3Response = await fetch(stagedUpload.uploadUrl, { - method: "PUT", - body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }), - headers: stagedUpload.uploadHeaders, - signal: AbortSignal.timeout( - Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs), - ), - }); + const s3Response = await fetchForPublish( + stagedUpload.uploadUrl, + () => ({ + method: "PUT", + body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }), + headers: stagedUpload.uploadHeaders, + signal: AbortSignal.timeout( + Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs), + ), + }), + "Failed to upload project archive", + PUBLISH_TRANSPORT_ATTEMPTS, + ); if (!s3Response.ok) { throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive")); } @@ -526,20 +592,25 @@ async function publishProjectArchiveStaged( projectId: string | undefined, ): Promise { const fileName = `${title}.zip`; - const uploadResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, { - method: "POST", - body: JSON.stringify({ - file_name: fileName, - content_type: PUBLISH_CONTENT_TYPE, - content_length: archive.buffer.byteLength, + const uploadResponse = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, + () => ({ + method: "POST", + body: JSON.stringify({ + file_name: fileName, + content_type: PUBLISH_CONTENT_TYPE, + content_length: archive.buffer.byteLength, + }), + headers: { + ...authHeaders, + "content-type": "application/json", + heygen_route: "canary", + }, + signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS), }), - headers: { - ...authHeaders, - "content-type": "application/json", - heygen_route: "canary", - }, - signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS), - }); + "Failed to prepare project upload", + PUBLISH_TRANSPORT_ATTEMPTS, + ); if (uploadResponse.status === 404 || uploadResponse.status === 405) { return null; @@ -553,22 +624,26 @@ async function publishProjectArchiveStaged( await uploadArchiveToPresignedUrl(stagedUpload, archive); - const completeResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, { - method: "POST", - body: JSON.stringify({ - upload_key: stagedUpload.uploadKey, - file_name: fileName, - title, - ...(isPublic ? { is_public: true } : {}), - ...(projectId ? { project_id: projectId } : {}), + const completeResponse = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, + () => ({ + method: "POST", + body: JSON.stringify({ + upload_key: stagedUpload.uploadKey, + file_name: fileName, + title, + ...(isPublic ? { is_public: true } : {}), + ...(projectId ? { project_id: projectId } : {}), + }), + headers: { + ...authHeaders, + "content-type": "application/json", + heygen_route: "canary", + }, + signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), }), - headers: { - ...authHeaders, - "content-type": "application/json", - heygen_route: "canary", - }, - signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), - }); + "Failed to finalize project publish", + ); const completePayload = await readJson(completeResponse); const publishedProject = parsePublishedProjectResponse(completePayload); From dd629697d3356971e50132703b8a897c1a79bfdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 7 Aug 2026 15:13:04 +0000 Subject: [PATCH 2/2] fix(cli): harden publish retry behavior --- packages/cli/src/utils/publishProject.test.ts | 81 +++++++++++++++++-- packages/cli/src/utils/publishProject.ts | 52 ++++++++---- 2 files changed, 112 insertions(+), 21 deletions(-) diff --git a/packages/cli/src/utils/publishProject.test.ts b/packages/cli/src/utils/publishProject.test.ts index 81227cb24d..8cff808289 100644 --- a/packages/cli/src/utils/publishProject.test.ts +++ b/packages/cli/src/utils/publishProject.test.ts @@ -146,9 +146,13 @@ function directFetch(completeData?: Record) { .mockResolvedValueOnce(publishedResponse(completeData)); } -function networkFailure(code: string, message: string): TypeError { +function networkFailure( + code: string, + message: string, + metadata: Record = {}, +): TypeError { return new TypeError("fetch failed", { - cause: Object.assign(new Error(message), { code }), + cause: Object.assign(new Error(message), { code, ...metadata }), }); } @@ -760,6 +764,37 @@ describe("publishProjectArchive", () => { } }); + it("waits briefly before retrying a transport failure", async () => { + vi.useFakeTimers(); + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse()) + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure")) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const publish = publishProjectArchive(dir); + await vi.advanceTimersByTimeAsync(0); + expect(fetchMock).toHaveBeenCalledTimes(2); + + await vi.advanceTimersByTimeAsync(199); + expect(fetchMock).toHaveBeenCalledTimes(2); + + await vi.advanceTimersByTimeAsync(1); + await expect(publish).resolves.toMatchObject({ projectId: "hfp_123" }); + expect(fetchMock).toHaveBeenCalledTimes(4); + } finally { + await vi.runAllTimersAsync(); + vi.useRealTimers(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it("reports the upload stage and transport cause after the retry is exhausted", async () => { const dir = makeProjectDir(); const signedUrl = @@ -767,8 +802,18 @@ describe("publishProjectArchive", () => { const fetchMock = vi .fn() .mockResolvedValueOnce(uploadResponse({ upload_url: signedUrl })) - .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com")) - .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com")); + .mockRejectedValueOnce( + networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com", { + errno: -3001, + syscall: "getaddrinfo", + }), + ) + .mockRejectedValueOnce( + networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com", { + errno: -3001, + syscall: "getaddrinfo", + }), + ); vi.stubGlobal("fetch", fetchMock); try { @@ -776,7 +821,7 @@ describe("publishProjectArchive", () => { const promise = publishProjectArchive(dir); await expect(promise).rejects.toThrow( - "Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN: getaddrinfo EAI_AGAIN s3.example.com)", + "Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN, syscall=getaddrinfo, errno=-3001: getaddrinfo EAI_AGAIN s3.example.com)", ); await expect(promise).rejects.not.toThrow("do-not-print"); expect(fetchMock).toHaveBeenCalledTimes(3); @@ -799,7 +844,7 @@ describe("publishProjectArchive", () => { writeFileSync(join(dir, "index.html"), "", "utf-8"); await expect(publishProjectArchive(dir)).rejects.toThrow( - "Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set, but Node fetch proxy support is disabled; retry with NODE_USE_ENV_PROXY=1", + "Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set but ignored by Node fetch; if this network requires them, retry with NODE_USE_ENV_PROXY=1", ); expect(fetchMock).toHaveBeenCalledTimes(2); } finally { @@ -835,6 +880,30 @@ describe("publishProjectArchive", () => { } }); + it("does not retry a request that reached its timeout", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockRejectedValueOnce( + new DOMException("The operation was aborted due to timeout", "TimeoutError"), + ) + .mockResolvedValueOnce(uploadResponse()) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + await expect(publishProjectArchive(dir)).rejects.toThrow( + "Failed to prepare project upload: The operation was aborted due to timeout", + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("reports the finalize stage and transport cause", async () => { const dir = makeProjectDir(); const fetchMock = vi diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index 4aaee4fa6a..89c2585c80 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -16,6 +16,7 @@ const PUBLISH_CONTENT_TYPE = "application/zip"; const PUBLISH_METADATA_TIMEOUT_MS = 30_000; const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000; const PUBLISH_TRANSPORT_ATTEMPTS = 2; +const PUBLISH_RETRY_DELAY_MS = 200; // Conservative floor — most connections are faster, but this prevents // premature aborts on slow/unstable networks (hotel wifi, tethering). const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000; @@ -167,9 +168,15 @@ async function readErrorMessage(response: Response, fallback: string): Promise all.indexOf(value) === index, + ); + const distinctCauseMessage = causeMessage && causeMessage !== message ? causeMessage : ""; + const detail = [metadata.join(", "), distinctCauseMessage].filter(Boolean).join(": "); return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`; } +function isRequestTimeout(error: unknown): boolean { + return ( + error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError") + ); +} + +function waitBeforePublishRetry(): Promise { + return new Promise((resolve) => setTimeout(resolve, PUBLISH_RETRY_DELAY_MS)); +} + async function fetchForPublish( input: string, createInit: () => RequestInit, failureStage: string, attempts = 1, ): Promise { + if (attempts < 1) throw new RangeError("Publish fetch attempts must be at least 1"); + let lastError: unknown; + let attemptsMade = 0; for (let attempt = 1; attempt <= attempts; attempt += 1) { + attemptsMade = attempt; try { return await fetch(input, createInit()); } catch (error) { - if (attempt === attempts) { - const attemptDetail = attempts > 1 ? ` after ${attempts} attempts` : ""; - throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(error)}`, { - cause: error instanceof Error ? error : undefined, - }); - } + lastError = error; + if (isRequestTimeout(error) || attempt === attempts) break; + await waitBeforePublishRetry(); } } - throw new Error(failureStage); + const attemptDetail = attemptsMade > 1 ? ` after ${attemptsMade} attempts` : ""; + throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(lastError)}`, { + cause: lastError instanceof Error ? lastError : undefined, + }); } export function uploadTimeoutMs(byteLength: number): number {