mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(cli): use size-adaptive timeouts for publish uploads (#635)
## Summary - Replace the flat 30s upload timeout with a size-adaptive calculation: `max(120s, bytes / 500KB/s)` - Metadata requests (presigned URL, complete) keep the original 30s timeout - Companion to the backend change removing the 64 MB upload limit in experiment-framework ## Context With the backend size limit removed, large projects (78 MB+) need proportionally longer to upload. A 78 MB project now gets ~164s, a 500 MB project ~17 min. The old 30s timeout would abort any upload over ~15 MB on a typical connection. ## Test plan - [x] All 4 existing vitest tests pass - [x] Build succeeds, no type errors - [x] Lint + format pass (oxlint + oxfmt) - [x] Timeout values verified for 10/78/200/500/1000 MB archives
This commit is contained in:
@@ -7,6 +7,7 @@ import {
|
||||
createPublishArchive,
|
||||
getPublishApiBaseUrl,
|
||||
publishProjectArchive,
|
||||
uploadTimeoutMs,
|
||||
} from "./publishProject.js";
|
||||
|
||||
function makeProjectDir(): string {
|
||||
@@ -35,6 +36,22 @@ describe("createPublishArchive", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("uploadTimeoutMs", () => {
|
||||
it("returns the minimum timeout for small files", () => {
|
||||
expect(uploadTimeoutMs(0)).toBe(120_000);
|
||||
expect(uploadTimeoutMs(50 * 1024 * 1024)).toBe(120_000);
|
||||
});
|
||||
|
||||
it("scales above the floor for large files", () => {
|
||||
expect(uploadTimeoutMs(64 * 1024 * 1024)).toBeGreaterThan(120_000);
|
||||
expect(uploadTimeoutMs(500 * 1024 * 1024)).toBeGreaterThan(900_000);
|
||||
});
|
||||
|
||||
it("returns an integer", () => {
|
||||
expect(Number.isInteger(uploadTimeoutMs(123_456))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("publishProjectArchive", () => {
|
||||
beforeEach(() => {
|
||||
vi.stubEnv("HYPERFRAMES_PUBLISHED_PROJECTS_API_URL", "");
|
||||
|
||||
@@ -5,7 +5,11 @@ import AdmZip from "adm-zip";
|
||||
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
|
||||
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
|
||||
const PUBLISH_CONTENT_TYPE = "application/zip";
|
||||
const PUBLISH_REQUEST_TIMEOUT_MS = 30_000;
|
||||
const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
|
||||
const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
|
||||
// Conservative floor — most connections are faster, but this prevents
|
||||
// premature aborts on slow/unstable networks (hotel wifi, tethering).
|
||||
const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000;
|
||||
|
||||
export interface PublishArchiveResult {
|
||||
buffer: Buffer;
|
||||
@@ -25,6 +29,7 @@ interface StagedUploadResponse {
|
||||
uploadKey: string;
|
||||
contentType: string;
|
||||
uploadHeaders: Record<string, string>;
|
||||
expiresInSeconds: number;
|
||||
}
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
@@ -73,11 +78,14 @@ function parseStagedUploadResponse(
|
||||
const uploadKey = stringField(data, "upload_key");
|
||||
const contentType = stringField(data, "content_type") || PUBLISH_CONTENT_TYPE;
|
||||
if (!uploadUrl || !uploadKey) return null;
|
||||
const rawExpires = data["expires_in_seconds"];
|
||||
const expiresInSeconds = typeof rawExpires === "number" && rawExpires > 0 ? rawExpires : 1800;
|
||||
return {
|
||||
uploadUrl,
|
||||
uploadKey,
|
||||
contentType,
|
||||
uploadHeaders: getUploadHeaders(data, uploadUrl, contentType, archiveByteLength),
|
||||
expiresInSeconds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -142,6 +150,13 @@ async function readErrorMessage(response: Response, fallback: string): Promise<s
|
||||
return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
|
||||
}
|
||||
|
||||
export function uploadTimeoutMs(byteLength: number): number {
|
||||
return Math.max(
|
||||
PUBLISH_UPLOAD_MIN_TIMEOUT_MS,
|
||||
Math.ceil((byteLength / PUBLISH_UPLOAD_BYTES_PER_SECOND) * 1000),
|
||||
);
|
||||
}
|
||||
|
||||
function shouldIgnoreSegment(segment: string): boolean {
|
||||
return segment.startsWith(".") || IGNORED_DIRS.has(segment) || IGNORED_FILES.has(segment);
|
||||
}
|
||||
@@ -214,7 +229,7 @@ async function publishProjectArchiveDirect(
|
||||
method: "POST",
|
||||
body,
|
||||
headers,
|
||||
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
|
||||
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
|
||||
});
|
||||
|
||||
const payload = await readJson(response);
|
||||
@@ -243,7 +258,7 @@ async function publishProjectArchiveStaged(
|
||||
"content-type": "application/json",
|
||||
heygen_route: "canary",
|
||||
},
|
||||
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
|
||||
signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
|
||||
});
|
||||
|
||||
if (uploadResponse.status === 404 || uploadResponse.status === 405) {
|
||||
@@ -256,11 +271,14 @@ async function publishProjectArchiveStaged(
|
||||
throw new Error(await readErrorMessage(uploadResponse, "Failed to prepare project upload"));
|
||||
}
|
||||
|
||||
const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS;
|
||||
const s3Response = await fetch(stagedUpload.uploadUrl, {
|
||||
method: "PUT",
|
||||
body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
|
||||
headers: stagedUpload.uploadHeaders,
|
||||
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
|
||||
signal: AbortSignal.timeout(
|
||||
Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
|
||||
),
|
||||
});
|
||||
if (!s3Response.ok) {
|
||||
throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive"));
|
||||
@@ -277,7 +295,7 @@ async function publishProjectArchiveStaged(
|
||||
"content-type": "application/json",
|
||||
heygen_route: "canary",
|
||||
},
|
||||
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
|
||||
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
|
||||
});
|
||||
|
||||
const completePayload = await readJson(completeResponse);
|
||||
|
||||
Reference in New Issue
Block a user