mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 19:36:04 +00:00
272 lines
8.2 KiB
JavaScript
272 lines
8.2 KiB
JavaScript
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtemp, open, rm, realpath, symlink, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { createHash } from "node:crypto";
|
|
import { uploadNeeded } from "../src/sources/uploadFiles.mjs";
|
|
import { partDigests, putRange } from "../src/sources/transfer.mjs";
|
|
import fsPromises from "node:fs/promises";
|
|
import { syncBuiltinESMExports } from "node:module";
|
|
import { bindShareHost } from "../src/host.mjs";
|
|
|
|
bindShareHost({ reviewUploadOrigins: ["https://store.test"] });
|
|
|
|
async function folder(t) {
|
|
const root = await realpath(await mkdtemp(join(tmpdir(), "hf-source-upload-")));
|
|
t.after(() => rm(root, { recursive: true, force: true }));
|
|
return root;
|
|
}
|
|
const signal = () => new AbortController().signal;
|
|
|
|
test("multipart retries retain the pinned upload and skip completed parts after restart", async (t) => {
|
|
const root = await folder(t);
|
|
const file = join(root, "media.mp4");
|
|
const size = 65 * 1024 * 1024;
|
|
const handle = await open(file, "w");
|
|
await handle.truncate(size);
|
|
await handle.close();
|
|
const digests = await partDigests(file, size, 64 * 1024 * 1024, signal());
|
|
const inflight = { sent: [], multipart: {} };
|
|
const requests = [];
|
|
let fail = true;
|
|
let persisted = 0;
|
|
const job = {
|
|
api: {
|
|
uploadUrls: async (_space, _comp, _upload, files) => {
|
|
requests.push(...files);
|
|
return files.map((item) => ({
|
|
sha256: item.sha256,
|
|
multipartUploadId: "pinned",
|
|
expiresAt: "2099-01-01T00:00:00Z",
|
|
parts: item.parts.map((part) => ({
|
|
...part,
|
|
url: `https://store.test/${part.partNumber}`,
|
|
headers: { checksum: part.sha256 },
|
|
})),
|
|
}));
|
|
},
|
|
},
|
|
space: "space",
|
|
composition: "comp",
|
|
uploadId: "upload",
|
|
inflight,
|
|
local: new Map([[digests.whole, { path: "media.mp4", full: file }]]),
|
|
needed: [{ sha256: digests.whole, size }],
|
|
fetchImpl: async (url, init) => {
|
|
if (url.endsWith("/2") && fail) {
|
|
await init.body.cancel();
|
|
return new Response(null, { status: 400 });
|
|
}
|
|
const hash = createHash("sha256");
|
|
for await (const chunk of init.body) hash.update(chunk);
|
|
assert.equal(hash.digest("hex"), init.headers.checksum);
|
|
return new Response(null, { status: 200 });
|
|
},
|
|
signal: signal(),
|
|
wait: async () => {},
|
|
now: Date.now,
|
|
persist: () => persisted++,
|
|
onBytes: () => {},
|
|
onFile: () => {},
|
|
};
|
|
await assert.rejects(uploadNeeded(job), /refused/);
|
|
assert.deepEqual(inflight.multipart[digests.whole].partsDone, [1]);
|
|
fail = false;
|
|
await uploadNeeded(job);
|
|
assert.equal(requests[1].multipartUploadId, "pinned");
|
|
assert.deepEqual(
|
|
requests[1].parts.map((p) => p.partNumber),
|
|
[2],
|
|
);
|
|
assert.deepEqual(inflight.sent, [digests.whole]);
|
|
assert.ok(persisted >= 4);
|
|
});
|
|
|
|
test("signed URL expiry refreshes the grant without changing the uploaded content", async (t) => {
|
|
const root = await folder(t);
|
|
const file = join(root, "index.html");
|
|
await writeFile(file, "source");
|
|
const digest = createHash("sha256").update("source").digest("hex");
|
|
let grants = 0;
|
|
let puts = 0;
|
|
await uploadNeeded({
|
|
api: {
|
|
uploadUrls: async () => {
|
|
grants++;
|
|
return [
|
|
{
|
|
sha256: digest,
|
|
url: "https://store.test/file",
|
|
headers: {},
|
|
expiresAt: "2099-01-01T00:00:00Z",
|
|
},
|
|
];
|
|
},
|
|
},
|
|
space: "s",
|
|
composition: "c",
|
|
uploadId: "u",
|
|
inflight: { sent: [] },
|
|
local: new Map([[digest, { path: "index.html", full: file }]]),
|
|
needed: [{ sha256: digest, size: 6 }],
|
|
fetchImpl: async (_url, init) => {
|
|
puts++;
|
|
await new Response(init.body).arrayBuffer();
|
|
return new Response(null, { status: puts === 1 ? 403 : 200 });
|
|
},
|
|
signal: signal(),
|
|
wait: async () => {},
|
|
now: Date.now,
|
|
persist: () => {},
|
|
onBytes: () => {},
|
|
onFile: () => {},
|
|
});
|
|
assert.equal(grants, 2);
|
|
assert.equal(puts, 2);
|
|
});
|
|
|
|
test("storage PUT refuses a file swapped for a symlink before reading bytes", async (t) => {
|
|
const root = await folder(t);
|
|
const target = join(root, "private.txt");
|
|
const file = join(root, "asset.txt");
|
|
await writeFile(target, "secret");
|
|
await symlink(target, file);
|
|
let sent = false;
|
|
await assert.rejects(
|
|
putRange({
|
|
file,
|
|
path: "asset.txt",
|
|
start: 0,
|
|
length: 6,
|
|
sha256: "wrong",
|
|
signal: signal(),
|
|
url: "https://store.test",
|
|
headers: {},
|
|
fetchImpl: async () => {
|
|
sent = true;
|
|
return new Response();
|
|
},
|
|
}),
|
|
);
|
|
assert.equal(sent, false);
|
|
});
|
|
|
|
test("source PUT refuses untrusted or redirected targets before it opens a file", async () => {
|
|
for (const url of [
|
|
"https://untrusted.test/source",
|
|
"http://127.0.0.1:18080/source",
|
|
"https://store.test.untrusted.test/source",
|
|
"https://user:secret@store.test/source",
|
|
]) {
|
|
await assert.rejects(
|
|
putRange({
|
|
file: "/missing-source",
|
|
path: "index.html",
|
|
start: 0,
|
|
length: 6,
|
|
sha256: "unused",
|
|
signal: signal(),
|
|
url,
|
|
headers: {},
|
|
fetchImpl: async () => assert.fail("Must not send bytes"),
|
|
}),
|
|
{ kind: "refused" },
|
|
);
|
|
}
|
|
});
|
|
|
|
async function sourcePut(t, overrides = {}) {
|
|
const root = await folder(t);
|
|
const file = join(root, "index.html");
|
|
await writeFile(file, "source");
|
|
return {
|
|
file,
|
|
path: "index.html",
|
|
start: 0,
|
|
length: 6,
|
|
sha256: createHash("sha256").update("source").digest("hex"),
|
|
signal: signal(),
|
|
url: "https://store.test/file",
|
|
headers: {},
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
for (const consume of [false, true]) {
|
|
test(`source PUT times out when ${consume ? "the storage response" : "body consumption"} stalls`, async (t) => {
|
|
let requestSignal;
|
|
const put = await sourcePut(t, {
|
|
stallMs: 20,
|
|
signal: AbortSignal.timeout(2_000),
|
|
fetchImpl: async (_url, init) => {
|
|
requestSignal = init.signal;
|
|
if (consume) await new Response(init.body).arrayBuffer();
|
|
init.signal.throwIfAborted();
|
|
return new Promise((_resolve, reject) => {
|
|
init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true });
|
|
});
|
|
},
|
|
});
|
|
await assert.rejects(putRange(put), { kind: "transient" });
|
|
assert.equal(requestSignal.aborted, true);
|
|
});
|
|
}
|
|
|
|
test("source PUT preserves caller cancellation instead of retrying it", async (t) => {
|
|
const controller = new AbortController();
|
|
const cancelled = new DOMException("Sharing cancelled", "AbortError");
|
|
await assert.rejects(
|
|
putRange(
|
|
await sourcePut(t, {
|
|
signal: controller.signal,
|
|
fetchImpl: async (_url, init) => {
|
|
assert.equal(init.redirect, "error");
|
|
await new Response(init.body).arrayBuffer();
|
|
controller.abort(cancelled);
|
|
throw init.signal.reason;
|
|
},
|
|
}),
|
|
),
|
|
(error) => error === cancelled,
|
|
);
|
|
});
|
|
|
|
test("successful source PUT clears its stall timer", async (t) => {
|
|
let requestSignal;
|
|
await putRange(
|
|
await sourcePut(t, {
|
|
stallMs: 200,
|
|
fetchImpl: async (_url, init) => {
|
|
requestSignal = init.signal;
|
|
await new Response(init.body).arrayBuffer();
|
|
return new Response(null, { status: 200 });
|
|
},
|
|
}),
|
|
);
|
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
assert.equal(requestSignal.aborted, false);
|
|
});
|
|
|
|
test("source PUT closes its file when resolving its parent fails", async (t) => {
|
|
const put = await sourcePut(t, { fetchImpl: async () => assert.fail("Must not send bytes") });
|
|
const originalOpen = fsPromises.open;
|
|
let opened;
|
|
t.mock.method(fsPromises, "open", async (...args) => {
|
|
opened = await originalOpen(...args);
|
|
return opened;
|
|
});
|
|
t.mock.method(fsPromises, "realpath", async () => {
|
|
throw Object.assign(new Error("Parent directory disappeared"), { code: "ENOENT" });
|
|
});
|
|
syncBuiltinESMExports();
|
|
try {
|
|
await assert.rejects(putRange(put), { code: "ENOENT" });
|
|
assert.equal(opened.fd, -1, "source file handle was left open");
|
|
} finally {
|
|
await opened?.close();
|
|
t.mock.restoreAll();
|
|
syncBuiltinESMExports();
|
|
}
|
|
});
|