mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): cloud render sends one zip when it fits and uploads only the largest media (#7859)
* fix(cli): cloud render sends one zip when it fits and uploads only the largest media * fix(cli): cloud render leaves unused media out only when the zip is over the limit * test(cli): eight-at-once upload test waits for eight in flight, never a timer * test(cli): eight-at-once upload test also fails when more than eight run
This commit is contained in:
@@ -59,7 +59,7 @@ hyperframes cloud render --resolution 4k
|
||||
|
||||
## Check what will upload
|
||||
|
||||
Media files of 5 MB or more that the project names by path upload as HeyGen assets and are referenced by public, unsigned URL, so large footage and audio do not count against the 200 MB project archive limit. Uploads are cached, so re-rendering uploads only new or changed media. Media is streamed from disk as it uploads, so a large file never has to fit in memory; editing one while the command is preparing it stops the run. Inspect what will upload before starting a render:
|
||||
When the project fits under the 200 MB project archive limit, it is sent as one zip. When it does not, media no page, script or variable names is left out first, then the largest media files the project names by path upload as HeyGen assets, eight at a time, just enough for the rest to fit, and are referenced by public, unsigned URL. Uploads are cached, so re-rendering uploads only new or changed media. Media is streamed from disk as it uploads, so a large file never has to fit in memory; editing one while the command is preparing it stops the run. Inspect what will upload before starting a render:
|
||||
|
||||
```bash
|
||||
hyperframes cloud render --dry-run
|
||||
|
||||
@@ -2701,9 +2701,10 @@ Env files (`.env`, `.env.*`, `.envrc`, at any depth) and Studio's private record
|
||||
under `.hyperframes/` are always left out, even when a rule re-includes them, and
|
||||
each one is reported as left out with the reason `private`.
|
||||
|
||||
Media files of 5 MB or more that the project names by path go up as HeyGen
|
||||
assets before zipping, and the zip's copy of each HTML, CSS, script and JSON
|
||||
file points at the asset URL instead; your project files are not changed.
|
||||
When the zip would be over the upload limit, media no page, script or variable
|
||||
names is left out first, then the largest media files the project names by path
|
||||
go up as HeyGen assets, just enough for the rest to fit, and the zip's copy of each HTML, CSS,
|
||||
script and JSON file points at the asset URL instead; your project files are not changed.
|
||||
Asset URLs are public and unsigned: anyone who has one can fetch the
|
||||
media. Uploads are cached by content, so a re-render uploads only what changed.
|
||||
Media the asset API does not take as is is converted on your machine first,
|
||||
|
||||
@@ -30,6 +30,8 @@ vi.mock("node:fs", async (importOriginal) => {
|
||||
});
|
||||
|
||||
const HEAVY = Buffer.alloc(OFFLOAD_MIN_BYTES, 1);
|
||||
// A zip budget only HEAVY stand-ins are over: the tiny files beside them stay in the zip.
|
||||
const ONE_MIB = 1024 * 1024;
|
||||
const dirs: string[] = [];
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -122,6 +124,7 @@ describe("offloadHeavyMedia", () => {
|
||||
|
||||
const report = await offloadHeavyMedia(dir, fileMap, {
|
||||
apiBaseUrl: "https://api.example",
|
||||
zipBudgetBytes: ONE_MIB,
|
||||
upload,
|
||||
cachePath: cachePath(),
|
||||
});
|
||||
@@ -132,7 +135,8 @@ describe("offloadHeavyMedia", () => {
|
||||
["media/clip.mp4", "upload"],
|
||||
["media/voice.mp3", "upload"],
|
||||
]);
|
||||
expect(report.map((item) => item.asset_id)).toEqual(["asset_1", "asset_2", "asset_3"]);
|
||||
// Uploads run together, so which file got which asset number is a race; each got its own.
|
||||
expect(report.map((item) => item.asset_id).sort()).toEqual(["asset_1", "asset_2", "asset_3"]);
|
||||
const index = fileMap.get("index.html")!.toString();
|
||||
expect(index).toContain(`url("${url("media/bg.png")}")`);
|
||||
expect(index).toContain(`src="${url("media/clip.mp4")}" poster="${url("media/bg.png")}"`);
|
||||
@@ -181,6 +185,175 @@ describe("offloadHeavyMedia", () => {
|
||||
expect(second.get("index.html")!.toString()).toContain("https://assets.example/1/clip.mp4");
|
||||
});
|
||||
|
||||
it("with no zip budget, uploads every media file a page uses whatever its size, and leaves unused media out", async () => {
|
||||
// A film cut from many short clips: each clip is far under the old 5 MB floor, together over the upload limit.
|
||||
const small = (name: string) => honest(name).subarray(0, 64 * 1024);
|
||||
const clips = Array.from({ length: 6 }, (_, i) => `assets/clip-${i}.mp4`);
|
||||
const dir = project({
|
||||
"index.html": html(
|
||||
clips.map((clip) => `<video src="${clip}"></video>`).join("") +
|
||||
`<img src="assets/still.png" /><audio src="assets/bed.mp3"></audio>`,
|
||||
),
|
||||
...Object.fromEntries(clips.map((clip) => [clip, small(clip)])),
|
||||
"assets/still.png": small("still.png"),
|
||||
"assets/bed.mp3": small("bed.mp3"),
|
||||
"assets/unused-take.mp4": small("unused-take.mp4"),
|
||||
"assets/notes.txt": "notes",
|
||||
});
|
||||
const fileMap = stagePublishFiles(dir, OFFLOAD_MIN_BYTES);
|
||||
|
||||
const report = await offloadHeavyMedia(dir, fileMap, {
|
||||
apiBaseUrl: "https://api.example",
|
||||
upload: fakeUpload(),
|
||||
cachePath: cachePath(),
|
||||
});
|
||||
|
||||
expect(report.map((item) => item.path).sort()).toEqual(
|
||||
[...clips, "assets/still.png", "assets/bed.mp3"].sort(),
|
||||
);
|
||||
expect([...fileMap.keys()].sort()).toEqual(["assets/notes.txt", "index.html"]);
|
||||
});
|
||||
|
||||
it("uploads up to eight files at once", async () => {
|
||||
const clips = Array.from(
|
||||
{ length: 12 },
|
||||
(_, i) => `assets/clip-${String(i).padStart(2, "0")}.mp4`,
|
||||
);
|
||||
const dir = project({
|
||||
"index.html": html(clips.map((clip) => `<video src="${clip}"></video>`).join("")),
|
||||
// Each clip's bytes differ by its index, so none is another's cached copy.
|
||||
...Object.fromEntries(
|
||||
clips.map((clip, i) => [
|
||||
clip,
|
||||
Buffer.concat([honest(clip).subarray(0, 4096), Buffer.from([i])]),
|
||||
]),
|
||||
),
|
||||
});
|
||||
let inFlight = 0;
|
||||
let peak = 0;
|
||||
let started = 0;
|
||||
let parked: (() => void)[] = [];
|
||||
// Uploads wait until eight are in flight or every file has started, so reaching eight never depends on timing.
|
||||
const upload = vi.fn(async (file: UploadableFile) => {
|
||||
inFlight += 1;
|
||||
started += 1;
|
||||
peak = Math.max(peak, inFlight);
|
||||
const turn = new Promise<void>((resolve) => parked.push(resolve));
|
||||
if (inFlight === 8 || started === clips.length) {
|
||||
const release = parked;
|
||||
parked = [];
|
||||
// Held a moment first: an upload past the eighth would start now and raise the peak.
|
||||
setTimeout(() => release.forEach((go) => go()), 50);
|
||||
}
|
||||
await turn;
|
||||
inFlight -= 1;
|
||||
return { asset_id: file.filename, url: `https://assets.example/${file.filename}` };
|
||||
});
|
||||
|
||||
const report = await offloadHeavyMedia(dir, stagePublishFiles(dir, OFFLOAD_MIN_BYTES), {
|
||||
apiBaseUrl: "https://api.example",
|
||||
upload,
|
||||
cachePath: cachePath(),
|
||||
});
|
||||
|
||||
expect(peak).toBe(8);
|
||||
expect(upload).toHaveBeenCalledTimes(12);
|
||||
expect(report.map((item) => item.path)).toEqual(clips);
|
||||
});
|
||||
|
||||
it("uploads two files with the same bytes once when both start together", async () => {
|
||||
const same = honest("a.mp4").subarray(0, 4096);
|
||||
const dir = project({
|
||||
"index.html": html(`<video src="a.mp4"></video><video src="b.mp4"></video>`),
|
||||
"a.mp4": same,
|
||||
"b.mp4": same,
|
||||
});
|
||||
const upload = fakeUpload();
|
||||
|
||||
const report = await offloadHeavyMedia(dir, stagePublishFiles(dir, OFFLOAD_MIN_BYTES), {
|
||||
apiBaseUrl: "https://api.example",
|
||||
upload,
|
||||
cachePath: cachePath(),
|
||||
});
|
||||
|
||||
expect(upload).toHaveBeenCalledOnce();
|
||||
expect(report.map((item) => item.url)).toEqual([report[0]!.url, report[0]!.url]);
|
||||
});
|
||||
|
||||
describe("with a zip budget", () => {
|
||||
// Clips of 1, 2, 3 and 4 KiB a page plays, a 3 KiB take no page plays, and the page itself.
|
||||
const sized = (name: string, kib: number) => honest(name).subarray(0, kib * 1024);
|
||||
const files = () => ({
|
||||
"index.html": html(
|
||||
[1, 2, 3, 4].map((kib) => `<video src="assets/clip-${kib}.mp4"></video>`).join(""),
|
||||
),
|
||||
...Object.fromEntries(
|
||||
[1, 2, 3, 4].map((kib) => [`assets/clip-${kib}.mp4`, sized("a.mp4", kib)]),
|
||||
),
|
||||
"assets/unused-take.mp4": sized("a.mp4", 3),
|
||||
});
|
||||
const offload = async (zipBudgetBytes: number) => {
|
||||
const dir = project(files());
|
||||
const fileMap = stagePublishFiles(dir, OFFLOAD_MIN_BYTES);
|
||||
const upload = fakeUpload();
|
||||
const report = await offloadHeavyMedia(dir, fileMap, {
|
||||
apiBaseUrl: "https://api.example",
|
||||
upload,
|
||||
cachePath: cachePath(),
|
||||
zipBudgetBytes,
|
||||
});
|
||||
return {
|
||||
uploaded: report.map((item) => item.path),
|
||||
zipped: [...fileMap.keys()].sort(),
|
||||
upload,
|
||||
};
|
||||
};
|
||||
|
||||
it("uploads nothing and keeps every file when the zip fits", async () => {
|
||||
const { upload, zipped } = await offload(14 * 1024);
|
||||
expect(upload).not.toHaveBeenCalled();
|
||||
expect(zipped).toEqual(
|
||||
["assets/unused-take.mp4", "index.html"]
|
||||
.concat([1, 2, 3, 4].map((kib) => `assets/clip-${kib}.mp4`))
|
||||
.sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves unused media out before uploading anything", async () => {
|
||||
const { upload, zipped } = await offload(11 * 1024);
|
||||
expect(upload).not.toHaveBeenCalled();
|
||||
expect(zipped).not.toContain("assets/unused-take.mp4");
|
||||
});
|
||||
|
||||
it("keeps frames a script loads by a built path when the zip fits", async () => {
|
||||
const dir = project({
|
||||
"index.html": html(
|
||||
`<script>for (let i = 1; i <= 3; i++) new Image().src = "frames/frame_" + i + ".png";</script>`,
|
||||
),
|
||||
...Object.fromEntries([1, 2, 3].map((i) => [`frames/frame_${i}.png`, sized("f.png", 1)])),
|
||||
});
|
||||
const fileMap = stagePublishFiles(dir, OFFLOAD_MIN_BYTES);
|
||||
await offloadHeavyMedia(dir, fileMap, {
|
||||
apiBaseUrl: "https://api.example",
|
||||
upload: fakeUpload(),
|
||||
cachePath: cachePath(),
|
||||
zipBudgetBytes: 190 * 1024 * 1024,
|
||||
});
|
||||
expect([...fileMap.keys()].sort()).toEqual([
|
||||
"frames/frame_1.png",
|
||||
"frames/frame_2.png",
|
||||
"frames/frame_3.png",
|
||||
"index.html",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uploads the largest files first, only until the rest fits", async () => {
|
||||
const { uploaded, zipped } = await offload(5 * 1024);
|
||||
expect(uploaded).toEqual(["assets/clip-3.mp4", "assets/clip-4.mp4"]);
|
||||
expect(zipped).toEqual(["assets/clip-1.mp4", "assets/clip-2.mp4", "index.html"]);
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves heavy files nothing references out of the archive, but keeps one a variable names", async () => {
|
||||
const dir = project({
|
||||
"index.html": html(`<video src="clip.mp4"></video>`),
|
||||
@@ -1092,6 +1265,7 @@ describe("offloadHeavyMedia", () => {
|
||||
|
||||
await offloadHeavyMedia(dir, fileMap, {
|
||||
apiBaseUrl: "https://api.example",
|
||||
zipBudgetBytes: ONE_MIB,
|
||||
upload: fakeUpload(),
|
||||
cachePath: cachePath(),
|
||||
});
|
||||
@@ -1138,7 +1312,12 @@ describe("offloadHeavyMedia", () => {
|
||||
preparedGifs: [{ sourceSrc: "media/loop.gif", outputPath }],
|
||||
};
|
||||
});
|
||||
const opts = { apiBaseUrl: "https://api.example", upload: fakeUpload(), gifPrep };
|
||||
const opts = {
|
||||
apiBaseUrl: "https://api.example",
|
||||
upload: fakeUpload(),
|
||||
gifPrep,
|
||||
zipBudgetBytes: ONE_MIB,
|
||||
};
|
||||
|
||||
const dir = project(files);
|
||||
await offloadHeavyMedia(dir, stagePublishFiles(dir, OFFLOAD_MIN_BYTES), {
|
||||
@@ -1190,6 +1369,7 @@ describe("offloadHeavyMedia", () => {
|
||||
|
||||
const report = await offloadHeavyMedia(dir, fileMap, {
|
||||
apiBaseUrl: "https://api.example",
|
||||
zipBudgetBytes: ONE_MIB,
|
||||
upload,
|
||||
gifPrep,
|
||||
cachePath: cachePath(),
|
||||
|
||||
@@ -54,12 +54,15 @@ import {
|
||||
TEXT_FILE_EXTS,
|
||||
} from "../utils/publishProject.js";
|
||||
import { hyperframesCacheDir } from "@hyperframes/core/cache-dir";
|
||||
import { runWithConcurrencyLimit } from "../utils/concurrency.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** Media at or above this size leaves the zip as a HeyGen asset. */
|
||||
/** Staged media this big stays on disk until it uploads; an entry GIF this big becomes a video first. */
|
||||
export const OFFLOAD_MIN_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
const OFFLOAD_CONCURRENCY = 8;
|
||||
|
||||
const MiB = 1024 * 1024;
|
||||
const AUDIO_MAX_BYTES = 1_500_000_000;
|
||||
|
||||
@@ -147,6 +150,8 @@ export interface OffloadOptions {
|
||||
gifPrep?: GifPrep;
|
||||
onFile?: (item: OffloadedMedia) => void;
|
||||
variablesText?: string;
|
||||
/** Media leaves the zip, largest first, only while the zip is over this; absent, all used media leaves. */
|
||||
zipBudgetBytes?: number;
|
||||
}
|
||||
|
||||
type MediaKind = "video" | "audio" | "image";
|
||||
@@ -601,6 +606,7 @@ interface OffloadContext {
|
||||
/** Staged GIFs the entry composition no longer shows as images. */
|
||||
preparedGifSources: string[];
|
||||
fileMap: Map<string, StagedFile>;
|
||||
uploadsInFlightByKey: Map<string, Promise<{ url: string; asset_id: string }>>;
|
||||
}
|
||||
|
||||
interface FilePlan {
|
||||
@@ -634,7 +640,7 @@ async function planFile(
|
||||
file: StagedFile,
|
||||
): Promise<FilePlan | null> {
|
||||
const size = stagedSize(file);
|
||||
let kind = size < OFFLOAD_MIN_BYTES ? null : mediaFileKind(path, file);
|
||||
let kind = mediaFileKind(path, file);
|
||||
if (!kind) return null;
|
||||
const ext = extname(path).toLowerCase();
|
||||
const accepted = ACCEPTED[ext];
|
||||
@@ -675,12 +681,13 @@ async function planFileTrim(
|
||||
kind: MediaKind,
|
||||
): Promise<PlayedSegment[] | null> {
|
||||
const entryPath = ctx.opts.entryPath ?? "index.html";
|
||||
const entryHtml = textOf(entryPath, ctx.fileMap.get(entryPath));
|
||||
const gridPerSecond = entryHtml ? await ctx.reencoder.trimGrid?.(sourcePath, kind) : null;
|
||||
if (!entryHtml || !gridPerSecond) return null;
|
||||
if (!textOf(entryPath, ctx.fileMap.get(entryPath))) return null;
|
||||
const gridPerSecond = await ctx.reencoder.trimGrid?.(sourcePath, kind);
|
||||
const entryHtmlAfterOthersPlanned = textOf(entryPath, ctx.fileMap.get(entryPath));
|
||||
if (!entryHtmlAfterOthersPlanned || !gridPerSecond) return null;
|
||||
const name = basename(path);
|
||||
const plan = planTrim({
|
||||
entryHtml,
|
||||
entryHtml: entryHtmlAfterOthersPlanned,
|
||||
path,
|
||||
gridPerSecond,
|
||||
resolveKey: (raw) => htmlSrcKey(ctx.absDir, entryPath, raw, ctx.fileMap),
|
||||
@@ -724,6 +731,21 @@ async function uploadPlanned(
|
||||
path: string,
|
||||
staged: StagedFile,
|
||||
plan: FilePlan,
|
||||
): Promise<{ url: string; asset_id: string }> {
|
||||
if (plan.key === undefined) return uploadOnce(ctx, upload, path, staged, plan);
|
||||
const pending = ctx.uploadsInFlightByKey.get(plan.key);
|
||||
if (pending) return pending;
|
||||
const started = uploadOnce(ctx, upload, path, staged, plan);
|
||||
ctx.uploadsInFlightByKey.set(plan.key, started);
|
||||
return started;
|
||||
}
|
||||
|
||||
async function uploadOnce(
|
||||
ctx: OffloadContext,
|
||||
upload: NonNullable<OffloadOptions["upload"]>,
|
||||
path: string,
|
||||
staged: StagedFile,
|
||||
plan: FilePlan,
|
||||
): Promise<{ url: string; asset_id: string }> {
|
||||
const filename = basename(path);
|
||||
const file: UploadableFile = plan.preset
|
||||
@@ -800,8 +822,8 @@ function entryGifSources(
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload each heavy media file as an asset (or reuse a cached one still served), point its references at the
|
||||
* asset URL and drop it from `fileMap`. Without `upload`, nothing uploads and only animated GIFs are encoded.
|
||||
* Moves used media out of the zip as assets (cached ones reused), largest first, until the zip fits the budget.
|
||||
* Without `upload`, nothing uploads and only animated GIFs are encoded.
|
||||
*/
|
||||
export async function offloadHeavyMedia(
|
||||
projectDir: string,
|
||||
@@ -826,6 +848,7 @@ export async function offloadHeavyMedia(
|
||||
tempFiles: [],
|
||||
preparedGifSources: [],
|
||||
fileMap,
|
||||
uploadsInFlightByKey: new Map(),
|
||||
};
|
||||
const urls = new Map<string, string>();
|
||||
const report: OffloadedMedia[] = [];
|
||||
@@ -834,39 +857,80 @@ export async function offloadHeavyMedia(
|
||||
await prepareEntryGifs(ctx, fileMap);
|
||||
// Only files a reference points at: one used through render-time --variables must stay in the zip.
|
||||
referenced = referencedFiles(ctx.absDir, fileMap);
|
||||
for (const [path, file] of [...fileMap].sort(([a], [b]) => a.localeCompare(b))) {
|
||||
if (!referenced.has(path)) continue;
|
||||
const item = await offloadFile(ctx, path, file);
|
||||
if (!item) continue;
|
||||
// A dry run rewrites to a stand-in URL so the reported archive size matches the real one.
|
||||
urls.set(path, item.url ?? `https://assets.invalid/${report.length}`);
|
||||
report.push(item);
|
||||
opts.onFile?.(item);
|
||||
}
|
||||
const budget = opts.zipBudgetBytes ?? 0;
|
||||
const zipped = [...fileMap.values()].reduce((sum, file) => sum + stagedSize(file), 0);
|
||||
const left = dropUnusedMediaWhileOver(ctx, referenced, zipped, budget);
|
||||
await uploadLargestWhileOver(ctx, referenced, left, budget, urls, report);
|
||||
report.sort((a, b) => a.path.localeCompare(b.path));
|
||||
} finally {
|
||||
for (const dir of ctx.tempFiles) rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
if (urls.size > 0) rewriteReferences(ctx.absDir, fileMap, (key) => urls.get(key) ?? null);
|
||||
dropReplacedFiles(fileMap, report, ctx.preparedGifSources, [...urls.values()]);
|
||||
dropUnusedHeavyFiles(ctx, referenced, [...urls.values()]);
|
||||
return report;
|
||||
}
|
||||
|
||||
function dropUnusedHeavyFiles(
|
||||
/** Drops media no page, script or variable names, largest first, while the zip is over budget; returns its size. */
|
||||
function dropUnusedMediaWhileOver(
|
||||
ctx: OffloadContext,
|
||||
referenced: Set<string>,
|
||||
assetUrls: string[],
|
||||
): void {
|
||||
zipped: number,
|
||||
budget: number,
|
||||
): number {
|
||||
const { fileMap } = ctx;
|
||||
const withoutUrls = (text: string) =>
|
||||
assetUrls.reduce((acc, url) => acc.replaceAll(url, ""), text);
|
||||
const code = withoutUrls(scriptCode(fileMap));
|
||||
const code = scriptCode(fileMap);
|
||||
const variables = ctx.opts.variablesText ?? "";
|
||||
for (const [path, file] of [...fileMap]) {
|
||||
if (referenced.has(path) || stagedSize(file) < OFFLOAD_MIN_BYTES) continue;
|
||||
if (variables.includes(basename(path))) continue;
|
||||
if (!isStillMentioned(fileMap, path, code, withoutUrls)) fileMap.delete(path);
|
||||
const unused = [...fileMap]
|
||||
.filter(([path]) => !referenced.has(path) && mediaKind(path) !== null)
|
||||
.filter(([path]) => !variables.includes(basename(path)))
|
||||
.filter(([path]) => !isStillMentioned(fileMap, path, code, (text) => text))
|
||||
.sort(byLargest);
|
||||
let left = zipped;
|
||||
for (const [path, file] of unused) {
|
||||
if (left <= budget) break;
|
||||
fileMap.delete(path);
|
||||
left -= stagedSize(file);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
const byLargest = ([a, x]: [string, StagedFile], [b, y]: [string, StagedFile]) =>
|
||||
stagedSize(y) - stagedSize(x) || a.localeCompare(b);
|
||||
|
||||
async function uploadLargestWhileOver(
|
||||
ctx: OffloadContext,
|
||||
referenced: Set<string>,
|
||||
zipped: number,
|
||||
budget: number,
|
||||
urls: Map<string, string>,
|
||||
report: OffloadedMedia[],
|
||||
): Promise<void> {
|
||||
const used = [...ctx.fileMap].filter(([path]) => referenced.has(path)).sort(byLargest);
|
||||
let left = zipped;
|
||||
let next = 0;
|
||||
while (left > budget && next < used.length) {
|
||||
const batch: [string, StagedFile][] = [];
|
||||
for (let after = left; after > budget && next < used.length; next += 1) {
|
||||
batch.push(used[next]!);
|
||||
after -= stagedSize(used[next]![1]);
|
||||
}
|
||||
const items = await runWithConcurrencyLimit(
|
||||
batch,
|
||||
OFFLOAD_CONCURRENCY,
|
||||
async ([path, file]) => {
|
||||
const item = await offloadFile(ctx, path, file);
|
||||
if (item) ctx.opts.onFile?.(item);
|
||||
return item;
|
||||
},
|
||||
);
|
||||
for (const [index, item] of items.entries()) {
|
||||
if (!item) continue;
|
||||
// A dry run rewrites to a stand-in URL so the reported archive size matches the real one.
|
||||
urls.set(item.path, item.url ?? `https://assets.invalid/${report.length}`);
|
||||
report.push(item);
|
||||
left -= stagedSize(batch[index]![1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -180,7 +180,7 @@ describe("cloud render --dry-run of an imported project", () => {
|
||||
});
|
||||
|
||||
describe("cloud render --dry-run with heavy media", () => {
|
||||
it("plans the media as HeyGen asset uploads and leaves it out of the archive", () => {
|
||||
it("sends every file in the zip when the project fits, with no uploads", () => {
|
||||
const dir = writeComposition(1920, 1080);
|
||||
writeFileSync(
|
||||
join(dir, "index.html"),
|
||||
@@ -192,6 +192,7 @@ describe("cloud render --dry-run with heavy media", () => {
|
||||
join(dir, "clip.mp4"),
|
||||
Buffer.concat([mp4Header, Buffer.alloc(6 * 1024 * 1024 - mp4Header.length, 1)]),
|
||||
);
|
||||
writeFileSync(join(dir, "unused-take.mp4"), Buffer.concat([mp4Header, Buffer.alloc(1024, 2)]));
|
||||
const result = spawnSync(
|
||||
"bun",
|
||||
["run", cliEntry, "cloud", "render", dir, "--dry-run", "--json"],
|
||||
@@ -214,13 +215,16 @@ describe("cloud render --dry-run with heavy media", () => {
|
||||
file_count: number;
|
||||
size_bytes: number;
|
||||
offloaded_media: Array<{ path: string; action: string; size_bytes: number }>;
|
||||
largest_files: Array<{ path: string; size_bytes: number }>;
|
||||
};
|
||||
};
|
||||
expect(archive.offloaded_media).toEqual([
|
||||
{ path: "clip.mp4", action: "upload", size_bytes: 6 * 1024 * 1024 },
|
||||
// One zip is faster than any per-file upload, so media leaves it only when the zip would not fit.
|
||||
expect(archive.offloaded_media).toEqual([]);
|
||||
expect(archive.largest_files.map((file) => file.path)).toEqual([
|
||||
"clip.mp4",
|
||||
"unused-take.mp4",
|
||||
"index.html",
|
||||
]);
|
||||
expect(archive.file_count).toBe(1);
|
||||
expect(archive.size_bytes).toBeLessThan(64 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -115,6 +115,7 @@ const VALID_RESOLUTION = ["1080p", "4k"] as const;
|
||||
const VALID_ASPECT_RATIO = ["16:9", "9:16", "1:1"] as const;
|
||||
// Mirrors the binary-byte max_bytes contract used by direct asset uploads.
|
||||
const DIRECT_UPLOAD_LIMIT_BYTES = 200 * 1024 * 1024;
|
||||
const ZIP_BUDGET_BYTES_WITH_ROOM_FOR_ZIP_RECORDS = Math.floor(DIRECT_UPLOAD_LIMIT_BYTES * 0.95);
|
||||
const ARCHIVE_DIAGNOSTIC_THRESHOLD_BYTES = 150 * 1024 * 1024;
|
||||
const LARGEST_FILE_COUNT = 10;
|
||||
|
||||
@@ -831,6 +832,7 @@ async function offloadMedia(
|
||||
upload,
|
||||
entryPath,
|
||||
variablesText,
|
||||
zipBudgetBytes: ZIP_BUDGET_BYTES_WITH_ROOM_FOR_ZIP_RECORDS,
|
||||
onFile: (item) =>
|
||||
(asJson ? console.error : console.log)(c.dim(` ${describeOffload(item, !upload)}`)),
|
||||
});
|
||||
|
||||
@@ -78,6 +78,21 @@ describe("runWithConcurrencyLimit", () => {
|
||||
expect(out).toEqual([]);
|
||||
});
|
||||
|
||||
it("after a rejection starts nothing new and settles only once started work has finished", async () => {
|
||||
const started: number[] = [];
|
||||
let slowDone = false;
|
||||
const run = runWithConcurrencyLimit([1, 2, 3, 4], 2, async (n) => {
|
||||
started.push(n);
|
||||
if (n === 2) throw new Error("boom");
|
||||
await new Promise((r) => setTimeout(r, 30));
|
||||
slowDone = true;
|
||||
return n;
|
||||
});
|
||||
await expect(run).rejects.toThrow(/boom/);
|
||||
expect(started).toEqual([1, 2]);
|
||||
expect(slowDone).toBe(true);
|
||||
});
|
||||
|
||||
it("propagates the first worker rejection", async () => {
|
||||
await expect(
|
||||
runWithConcurrencyLimit([1, 2, 3], 2, async (n) => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/** Runs `worker` over `inputs` with at most `limit` in flight. Results keep input order, and the
|
||||
* first rejection rejects the run. */
|
||||
/** Runs `worker` over `inputs` with at most `limit` in flight. Results keep input order. After the first
|
||||
* rejection no new input starts, and the run rejects with it once the work already started has finished. */
|
||||
export async function runWithConcurrencyLimit<T, R>(
|
||||
inputs: readonly T[],
|
||||
limit: number,
|
||||
@@ -8,14 +8,22 @@ export async function runWithConcurrencyLimit<T, R>(
|
||||
if (limit < 1) throw new Error(`runWithConcurrencyLimit: limit must be ≥ 1, got ${limit}`);
|
||||
const results = new Array<R>(inputs.length);
|
||||
let cursor = 0;
|
||||
let failed = false;
|
||||
const workerCount = Math.min(limit, inputs.length);
|
||||
await Promise.all(
|
||||
const settled = await Promise.allSettled(
|
||||
Array.from({ length: workerCount }, async () => {
|
||||
while (cursor < inputs.length) {
|
||||
while (!failed && cursor < inputs.length) {
|
||||
const idx = cursor++;
|
||||
results[idx] = await worker(inputs[idx]!, idx);
|
||||
try {
|
||||
results[idx] = await worker(inputs[idx]!, idx);
|
||||
} catch (error) {
|
||||
failed = true;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
const rejected = settled.find((s): s is PromiseRejectedResult => s.status === "rejected");
|
||||
if (rejected) throw rejected.reason;
|
||||
return results;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user