diff --git a/packages/cli/src/utils/publishProject.test.ts b/packages/cli/src/utils/publishProject.test.ts index 65255cc57..81227cb24 100644 --- a/packages/cli/src/utils/publishProject.test.ts +++ b/packages/cli/src/utils/publishProject.test.ts @@ -146,6 +146,12 @@ function directFetch(completeData?: Record) { .mockResolvedValueOnce(publishedResponse(completeData)); } +function networkFailure(code: string, message: string): TypeError { + return new TypeError("fetch failed", { + cause: Object.assign(new Error(message), { code }), + }); +} + /** Asserts the Nth fetch call, always requiring an AbortSignal alongside the given init. */ function expectFetchCall( fetchMock: ReturnType, @@ -729,6 +735,125 @@ describe("publishProjectArchive", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it("retries a transient presigned upload network failure once", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse()) + .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected")) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const result = await publishProjectArchive(dir); + + expect(result.projectId).toBe("hfp_123"); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(fetchMock.mock.calls[1]![0]).toBe("https://s3.example.com/upload"); + expect(fetchMock.mock.calls[2]![0]).toBe("https://s3.example.com/upload"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the upload stage and transport cause after the retry is exhausted", async () => { + const dir = makeProjectDir(); + const signedUrl = + "https://s3.example.com/upload?X-Amz-Credential=secret&X-Amz-Signature=do-not-print"; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse({ upload_url: signedUrl })) + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com")) + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com")); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const promise = publishProjectArchive(dir); + await expect(promise).rejects.toThrow( + "Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN: getaddrinfo EAI_AGAIN s3.example.com)", + ); + await expect(promise).rejects.not.toThrow("do-not-print"); + expect(fetchMock).toHaveBeenCalledTimes(3); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the prepare stage and explains disabled Node proxy support", async () => { + const dir = makeProjectDir(); + vi.stubEnv("HTTPS_PROXY", "http://proxy.example.com:8080"); + vi.stubEnv("NODE_USE_ENV_PROXY", ""); + const fetchMock = vi + .fn() + .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable")) + .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable")); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + await expect(publishProjectArchive(dir)).rejects.toThrow( + "Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set, but Node fetch proxy support is disabled; retry with NODE_USE_ENV_PROXY=1", + ); + expect(fetchMock).toHaveBeenCalledTimes(2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("retries a transient prepare-upload network failure once", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure")) + .mockResolvedValueOnce(uploadResponse()) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const result = await publishProjectArchive(dir); + + expect(result.projectId).toBe("hfp_123"); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(fetchMock.mock.calls[0]![0]).toBe( + "https://api2.heygen.com/v1/hyperframes/projects/publish/upload", + ); + expect(fetchMock.mock.calls[1]![0]).toBe( + "https://api2.heygen.com/v1/hyperframes/projects/publish/upload", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the finalize stage and transport cause", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse()) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected")); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + await expect(publishProjectArchive(dir)).rejects.toThrow( + "Failed to finalize project publish: fetch failed (ECONNRESET: socket disconnected)", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); /** Staged flow returning an already-owned, already-claimed stable project. */ diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index f9bf5f485..4aaee4fa6 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -15,6 +15,7 @@ const DEFAULT_PROJECT_IGNORE = ["/renders/", "/snapshots/"]; const PUBLISH_CONTENT_TYPE = "application/zip"; const PUBLISH_METADATA_TIMEOUT_MS = 30_000; const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000; +const PUBLISH_TRANSPORT_ATTEMPTS = 2; // Conservative floor — most connections are faster, but this prevents // premature aborts on slow/unstable networks (hotel wifi, tethering). const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000; @@ -166,6 +167,62 @@ async function readErrorMessage(response: Response, fallback: string): Promise + Boolean(process.env[key]?.trim()), + ); + const proxyEnabled = + process.env["NODE_USE_ENV_PROXY"] === "1" || + process.execArgv.includes("--use-env-proxy") || + process.env["NODE_OPTIONS"]?.split(/\s+/u).includes("--use-env-proxy") === true; + if (!proxyConfigured || proxyEnabled) return ""; + return ( + ". Proxy variables are set, but Node fetch proxy support is disabled; retry with " + + "NODE_USE_ENV_PROXY=1 (Node 22.21+)" + ); +} + +function describeFetchFailure(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + const cause = error instanceof Error ? error.cause : undefined; + const causeMessage = cause instanceof Error ? cause.message : ""; + const code = errorCode(cause) ?? errorCode(error); + const detail = [code, causeMessage && causeMessage !== message ? causeMessage : ""] + .filter(Boolean) + .join(": "); + return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`; +} + +async function fetchForPublish( + input: string, + createInit: () => RequestInit, + failureStage: string, + attempts = 1, +): Promise { + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + return await fetch(input, createInit()); + } catch (error) { + if (attempt === attempts) { + const attemptDetail = attempts > 1 ? ` after ${attempts} attempts` : ""; + throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(error)}`, { + cause: error instanceof Error ? error : undefined, + }); + } + } + } + throw new Error(failureStage); +} + export function uploadTimeoutMs(byteLength: number): number { return Math.max( PUBLISH_UPLOAD_MIN_TIMEOUT_MS, @@ -483,12 +540,16 @@ async function publishProjectArchiveDirect( heygen_route: "canary", }; - const response = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish`, { - method: "POST", - body, - headers, - signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), - }); + const response = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish`, + () => ({ + method: "POST", + body, + headers, + signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), + }), + "Failed to publish project", + ); const payload = await readJson(response); const publishedProject = parsePublishedProjectResponse(payload); @@ -504,14 +565,19 @@ async function uploadArchiveToPresignedUrl( archive: PublishArchiveResult, ): Promise { const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS; - const s3Response = await fetch(stagedUpload.uploadUrl, { - method: "PUT", - body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }), - headers: stagedUpload.uploadHeaders, - signal: AbortSignal.timeout( - Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs), - ), - }); + const s3Response = await fetchForPublish( + stagedUpload.uploadUrl, + () => ({ + method: "PUT", + body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }), + headers: stagedUpload.uploadHeaders, + signal: AbortSignal.timeout( + Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs), + ), + }), + "Failed to upload project archive", + PUBLISH_TRANSPORT_ATTEMPTS, + ); if (!s3Response.ok) { throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive")); } @@ -526,20 +592,25 @@ async function publishProjectArchiveStaged( projectId: string | undefined, ): Promise { const fileName = `${title}.zip`; - const uploadResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, { - method: "POST", - body: JSON.stringify({ - file_name: fileName, - content_type: PUBLISH_CONTENT_TYPE, - content_length: archive.buffer.byteLength, + const uploadResponse = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, + () => ({ + method: "POST", + body: JSON.stringify({ + file_name: fileName, + content_type: PUBLISH_CONTENT_TYPE, + content_length: archive.buffer.byteLength, + }), + headers: { + ...authHeaders, + "content-type": "application/json", + heygen_route: "canary", + }, + signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS), }), - headers: { - ...authHeaders, - "content-type": "application/json", - heygen_route: "canary", - }, - signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS), - }); + "Failed to prepare project upload", + PUBLISH_TRANSPORT_ATTEMPTS, + ); if (uploadResponse.status === 404 || uploadResponse.status === 405) { return null; @@ -553,22 +624,26 @@ async function publishProjectArchiveStaged( await uploadArchiveToPresignedUrl(stagedUpload, archive); - const completeResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, { - method: "POST", - body: JSON.stringify({ - upload_key: stagedUpload.uploadKey, - file_name: fileName, - title, - ...(isPublic ? { is_public: true } : {}), - ...(projectId ? { project_id: projectId } : {}), + const completeResponse = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, + () => ({ + method: "POST", + body: JSON.stringify({ + upload_key: stagedUpload.uploadKey, + file_name: fileName, + title, + ...(isPublic ? { is_public: true } : {}), + ...(projectId ? { project_id: projectId } : {}), + }), + headers: { + ...authHeaders, + "content-type": "application/json", + heygen_route: "canary", + }, + signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), }), - headers: { - ...authHeaders, - "content-type": "application/json", - heygen_route: "canary", - }, - signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), - }); + "Failed to finalize project publish", + ); const completePayload = await readJson(completeResponse); const publishedProject = parsePublishedProjectResponse(completePayload);