mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
fix(cli): harden publish retry behavior
This commit is contained in:
@@ -146,9 +146,13 @@ function directFetch(completeData?: Record<string, unknown>) {
|
||||
.mockResolvedValueOnce(publishedResponse(completeData));
|
||||
}
|
||||
|
||||
function networkFailure(code: string, message: string): TypeError {
|
||||
function networkFailure(
|
||||
code: string,
|
||||
message: string,
|
||||
metadata: Record<string, unknown> = {},
|
||||
): TypeError {
|
||||
return new TypeError("fetch failed", {
|
||||
cause: Object.assign(new Error(message), { code }),
|
||||
cause: Object.assign(new Error(message), { code, ...metadata }),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -760,6 +764,37 @@ describe("publishProjectArchive", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("waits briefly before retrying a transport failure", async () => {
|
||||
vi.useFakeTimers();
|
||||
const dir = makeProjectDir();
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(uploadResponse())
|
||||
.mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure"))
|
||||
.mockResolvedValueOnce(new Response(null, { status: 200 }))
|
||||
.mockResolvedValueOnce(publishedResponse());
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
try {
|
||||
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
|
||||
|
||||
const publish = publishProjectArchive(dir);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(199);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await expect(publish).resolves.toMatchObject({ projectId: "hfp_123" });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(4);
|
||||
} finally {
|
||||
await vi.runAllTimersAsync();
|
||||
vi.useRealTimers();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reports the upload stage and transport cause after the retry is exhausted", async () => {
|
||||
const dir = makeProjectDir();
|
||||
const signedUrl =
|
||||
@@ -767,8 +802,18 @@ describe("publishProjectArchive", () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(uploadResponse({ upload_url: signedUrl }))
|
||||
.mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com"))
|
||||
.mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com"));
|
||||
.mockRejectedValueOnce(
|
||||
networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com", {
|
||||
errno: -3001,
|
||||
syscall: "getaddrinfo",
|
||||
}),
|
||||
)
|
||||
.mockRejectedValueOnce(
|
||||
networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com", {
|
||||
errno: -3001,
|
||||
syscall: "getaddrinfo",
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
try {
|
||||
@@ -776,7 +821,7 @@ describe("publishProjectArchive", () => {
|
||||
|
||||
const promise = publishProjectArchive(dir);
|
||||
await expect(promise).rejects.toThrow(
|
||||
"Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN: getaddrinfo EAI_AGAIN s3.example.com)",
|
||||
"Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN, syscall=getaddrinfo, errno=-3001: getaddrinfo EAI_AGAIN s3.example.com)",
|
||||
);
|
||||
await expect(promise).rejects.not.toThrow("do-not-print");
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3);
|
||||
@@ -799,7 +844,7 @@ describe("publishProjectArchive", () => {
|
||||
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
|
||||
|
||||
await expect(publishProjectArchive(dir)).rejects.toThrow(
|
||||
"Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set, but Node fetch proxy support is disabled; retry with NODE_USE_ENV_PROXY=1",
|
||||
"Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set but ignored by Node fetch; if this network requires them, retry with NODE_USE_ENV_PROXY=1",
|
||||
);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
@@ -835,6 +880,30 @@ describe("publishProjectArchive", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not retry a request that reached its timeout", async () => {
|
||||
const dir = makeProjectDir();
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(
|
||||
new DOMException("The operation was aborted due to timeout", "TimeoutError"),
|
||||
)
|
||||
.mockResolvedValueOnce(uploadResponse())
|
||||
.mockResolvedValueOnce(new Response(null, { status: 200 }))
|
||||
.mockResolvedValueOnce(publishedResponse());
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
try {
|
||||
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
|
||||
|
||||
await expect(publishProjectArchive(dir)).rejects.toThrow(
|
||||
"Failed to prepare project upload: The operation was aborted due to timeout",
|
||||
);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reports the finalize stage and transport cause", async () => {
|
||||
const dir = makeProjectDir();
|
||||
const fetchMock = vi
|
||||
|
||||
@@ -16,6 +16,7 @@ const PUBLISH_CONTENT_TYPE = "application/zip";
|
||||
const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
|
||||
const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
|
||||
const PUBLISH_TRANSPORT_ATTEMPTS = 2;
|
||||
const PUBLISH_RETRY_DELAY_MS = 200;
|
||||
// Conservative floor — most connections are faster, but this prevents
|
||||
// premature aborts on slow/unstable networks (hotel wifi, tethering).
|
||||
const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000;
|
||||
@@ -167,9 +168,15 @@ async function readErrorMessage(response: Response, fallback: string): Promise<s
|
||||
return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
|
||||
}
|
||||
|
||||
function errorCode(value: unknown): string | null {
|
||||
if (!isRecord(value)) return null;
|
||||
return typeof value["code"] === "string" ? value["code"] : null;
|
||||
function systemErrorMetadata(value: unknown): string[] {
|
||||
if (!isRecord(value)) return [];
|
||||
const metadata: string[] = [];
|
||||
if (typeof value["code"] === "string") metadata.push(value["code"]);
|
||||
if (typeof value["syscall"] === "string") metadata.push(`syscall=${value["syscall"]}`);
|
||||
if (typeof value["errno"] === "string" || typeof value["errno"] === "number") {
|
||||
metadata.push(`errno=${value["errno"]}`);
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
function redactUrlQuery(message: string): string {
|
||||
@@ -186,7 +193,7 @@ function proxySupportHint(): string {
|
||||
process.env["NODE_OPTIONS"]?.split(/\s+/u).includes("--use-env-proxy") === true;
|
||||
if (!proxyConfigured || proxyEnabled) return "";
|
||||
return (
|
||||
". Proxy variables are set, but Node fetch proxy support is disabled; retry with " +
|
||||
". Proxy variables are set but ignored by Node fetch; if this network requires them, retry with " +
|
||||
"NODE_USE_ENV_PROXY=1 (Node 22.21+)"
|
||||
);
|
||||
}
|
||||
@@ -195,32 +202,47 @@ function describeFetchFailure(error: unknown): string {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const cause = error instanceof Error ? error.cause : undefined;
|
||||
const causeMessage = cause instanceof Error ? cause.message : "";
|
||||
const code = errorCode(cause) ?? errorCode(error);
|
||||
const detail = [code, causeMessage && causeMessage !== message ? causeMessage : ""]
|
||||
.filter(Boolean)
|
||||
.join(": ");
|
||||
const metadata = [...systemErrorMetadata(cause), ...systemErrorMetadata(error)].filter(
|
||||
(value, index, all) => all.indexOf(value) === index,
|
||||
);
|
||||
const distinctCauseMessage = causeMessage && causeMessage !== message ? causeMessage : "";
|
||||
const detail = [metadata.join(", "), distinctCauseMessage].filter(Boolean).join(": ");
|
||||
return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`;
|
||||
}
|
||||
|
||||
function isRequestTimeout(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError")
|
||||
);
|
||||
}
|
||||
|
||||
function waitBeforePublishRetry(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, PUBLISH_RETRY_DELAY_MS));
|
||||
}
|
||||
|
||||
async function fetchForPublish(
|
||||
input: string,
|
||||
createInit: () => RequestInit,
|
||||
failureStage: string,
|
||||
attempts = 1,
|
||||
): Promise<Response> {
|
||||
if (attempts < 1) throw new RangeError("Publish fetch attempts must be at least 1");
|
||||
let lastError: unknown;
|
||||
let attemptsMade = 0;
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
attemptsMade = attempt;
|
||||
try {
|
||||
return await fetch(input, createInit());
|
||||
} catch (error) {
|
||||
if (attempt === attempts) {
|
||||
const attemptDetail = attempts > 1 ? ` after ${attempts} attempts` : "";
|
||||
throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(error)}`, {
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
}
|
||||
lastError = error;
|
||||
if (isRequestTimeout(error) || attempt === attempts) break;
|
||||
await waitBeforePublishRetry();
|
||||
}
|
||||
}
|
||||
throw new Error(failureStage);
|
||||
const attemptDetail = attemptsMade > 1 ? ` after ${attemptsMade} attempts` : "";
|
||||
throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(lastError)}`, {
|
||||
cause: lastError instanceof Error ? lastError : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export function uploadTimeoutMs(byteLength: number): number {
|
||||
|
||||
Reference in New Issue
Block a user