mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(cli): use size-adaptive timeouts for publish uploads (#635)
## Summary - Replace the flat 30s upload timeout with a size-adaptive calculation: `max(120s, bytes / 500KB/s)` - Metadata requests (presigned URL, complete) keep the original 30s timeout - Companion to the backend change removing the 64 MB upload limit in experiment-framework ## Context With the backend size limit removed, large projects (78 MB+) need proportionally longer to upload. A 78 MB project now gets ~164s, a 500 MB project ~17 min. The old 30s timeout would abort any upload over ~15 MB on a typical connection. ## Test plan - [x] All 4 existing vitest tests pass - [x] Build succeeds, no type errors - [x] Lint + format pass (oxlint + oxfmt) - [x] Timeout values verified for 10/78/200/500/1000 MB archives
This commit is contained in:
@@ -7,6 +7,7 @@ import {
|
|||||||
createPublishArchive,
|
createPublishArchive,
|
||||||
getPublishApiBaseUrl,
|
getPublishApiBaseUrl,
|
||||||
publishProjectArchive,
|
publishProjectArchive,
|
||||||
|
uploadTimeoutMs,
|
||||||
} from "./publishProject.js";
|
} from "./publishProject.js";
|
||||||
|
|
||||||
function makeProjectDir(): string {
|
function makeProjectDir(): string {
|
||||||
@@ -35,6 +36,22 @@ describe("createPublishArchive", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("uploadTimeoutMs", () => {
|
||||||
|
it("returns the minimum timeout for small files", () => {
|
||||||
|
expect(uploadTimeoutMs(0)).toBe(120_000);
|
||||||
|
expect(uploadTimeoutMs(50 * 1024 * 1024)).toBe(120_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("scales above the floor for large files", () => {
|
||||||
|
expect(uploadTimeoutMs(64 * 1024 * 1024)).toBeGreaterThan(120_000);
|
||||||
|
expect(uploadTimeoutMs(500 * 1024 * 1024)).toBeGreaterThan(900_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns an integer", () => {
|
||||||
|
expect(Number.isInteger(uploadTimeoutMs(123_456))).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("publishProjectArchive", () => {
|
describe("publishProjectArchive", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.stubEnv("HYPERFRAMES_PUBLISHED_PROJECTS_API_URL", "");
|
vi.stubEnv("HYPERFRAMES_PUBLISHED_PROJECTS_API_URL", "");
|
||||||
|
|||||||
@@ -5,7 +5,11 @@ import AdmZip from "adm-zip";
|
|||||||
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
|
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
|
||||||
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
|
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
|
||||||
const PUBLISH_CONTENT_TYPE = "application/zip";
|
const PUBLISH_CONTENT_TYPE = "application/zip";
|
||||||
const PUBLISH_REQUEST_TIMEOUT_MS = 30_000;
|
const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
|
||||||
|
const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
|
||||||
|
// Conservative floor — most connections are faster, but this prevents
|
||||||
|
// premature aborts on slow/unstable networks (hotel wifi, tethering).
|
||||||
|
const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000;
|
||||||
|
|
||||||
export interface PublishArchiveResult {
|
export interface PublishArchiveResult {
|
||||||
buffer: Buffer;
|
buffer: Buffer;
|
||||||
@@ -25,6 +29,7 @@ interface StagedUploadResponse {
|
|||||||
uploadKey: string;
|
uploadKey: string;
|
||||||
contentType: string;
|
contentType: string;
|
||||||
uploadHeaders: Record<string, string>;
|
uploadHeaders: Record<string, string>;
|
||||||
|
expiresInSeconds: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
type JsonRecord = Record<string, unknown>;
|
type JsonRecord = Record<string, unknown>;
|
||||||
@@ -73,11 +78,14 @@ function parseStagedUploadResponse(
|
|||||||
const uploadKey = stringField(data, "upload_key");
|
const uploadKey = stringField(data, "upload_key");
|
||||||
const contentType = stringField(data, "content_type") || PUBLISH_CONTENT_TYPE;
|
const contentType = stringField(data, "content_type") || PUBLISH_CONTENT_TYPE;
|
||||||
if (!uploadUrl || !uploadKey) return null;
|
if (!uploadUrl || !uploadKey) return null;
|
||||||
|
const rawExpires = data["expires_in_seconds"];
|
||||||
|
const expiresInSeconds = typeof rawExpires === "number" && rawExpires > 0 ? rawExpires : 1800;
|
||||||
return {
|
return {
|
||||||
uploadUrl,
|
uploadUrl,
|
||||||
uploadKey,
|
uploadKey,
|
||||||
contentType,
|
contentType,
|
||||||
uploadHeaders: getUploadHeaders(data, uploadUrl, contentType, archiveByteLength),
|
uploadHeaders: getUploadHeaders(data, uploadUrl, contentType, archiveByteLength),
|
||||||
|
expiresInSeconds,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,6 +150,13 @@ async function readErrorMessage(response: Response, fallback: string): Promise<s
|
|||||||
return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
|
return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function uploadTimeoutMs(byteLength: number): number {
|
||||||
|
return Math.max(
|
||||||
|
PUBLISH_UPLOAD_MIN_TIMEOUT_MS,
|
||||||
|
Math.ceil((byteLength / PUBLISH_UPLOAD_BYTES_PER_SECOND) * 1000),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function shouldIgnoreSegment(segment: string): boolean {
|
function shouldIgnoreSegment(segment: string): boolean {
|
||||||
return segment.startsWith(".") || IGNORED_DIRS.has(segment) || IGNORED_FILES.has(segment);
|
return segment.startsWith(".") || IGNORED_DIRS.has(segment) || IGNORED_FILES.has(segment);
|
||||||
}
|
}
|
||||||
@@ -214,7 +229,7 @@ async function publishProjectArchiveDirect(
|
|||||||
method: "POST",
|
method: "POST",
|
||||||
body,
|
body,
|
||||||
headers,
|
headers,
|
||||||
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
|
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
|
||||||
});
|
});
|
||||||
|
|
||||||
const payload = await readJson(response);
|
const payload = await readJson(response);
|
||||||
@@ -243,7 +258,7 @@ async function publishProjectArchiveStaged(
|
|||||||
"content-type": "application/json",
|
"content-type": "application/json",
|
||||||
heygen_route: "canary",
|
heygen_route: "canary",
|
||||||
},
|
},
|
||||||
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
|
signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (uploadResponse.status === 404 || uploadResponse.status === 405) {
|
if (uploadResponse.status === 404 || uploadResponse.status === 405) {
|
||||||
@@ -256,11 +271,14 @@ async function publishProjectArchiveStaged(
|
|||||||
throw new Error(await readErrorMessage(uploadResponse, "Failed to prepare project upload"));
|
throw new Error(await readErrorMessage(uploadResponse, "Failed to prepare project upload"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS;
|
||||||
const s3Response = await fetch(stagedUpload.uploadUrl, {
|
const s3Response = await fetch(stagedUpload.uploadUrl, {
|
||||||
method: "PUT",
|
method: "PUT",
|
||||||
body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
|
body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
|
||||||
headers: stagedUpload.uploadHeaders,
|
headers: stagedUpload.uploadHeaders,
|
||||||
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
|
signal: AbortSignal.timeout(
|
||||||
|
Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
|
||||||
|
),
|
||||||
});
|
});
|
||||||
if (!s3Response.ok) {
|
if (!s3Response.ok) {
|
||||||
throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive"));
|
throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive"));
|
||||||
@@ -277,7 +295,7 @@ async function publishProjectArchiveStaged(
|
|||||||
"content-type": "application/json",
|
"content-type": "application/json",
|
||||||
heygen_route: "canary",
|
heygen_route: "canary",
|
||||||
},
|
},
|
||||||
signal: AbortSignal.timeout(PUBLISH_REQUEST_TIMEOUT_MS),
|
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
|
||||||
});
|
});
|
||||||
|
|
||||||
const completePayload = await readJson(completeResponse);
|
const completePayload = await readJson(completeResponse);
|
||||||
|
|||||||
Reference in New Issue
Block a user