Files
hyperframes/packages/share/tests/sourceUpload.test.mjs
T

272 lines
8.2 KiB
JavaScript

import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, open, rm, realpath, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createHash } from "node:crypto";
import { uploadNeeded } from "../src/sources/uploadFiles.mjs";
import { partDigests, putRange } from "../src/sources/transfer.mjs";
import fsPromises from "node:fs/promises";
import { syncBuiltinESMExports } from "node:module";
import { bindShareHost } from "../src/host.mjs";
bindShareHost({ reviewUploadOrigins: ["https://store.test"] });
async function folder(t) {
const root = await realpath(await mkdtemp(join(tmpdir(), "hf-source-upload-")));
t.after(() => rm(root, { recursive: true, force: true }));
return root;
}
const signal = () => new AbortController().signal;
test("multipart retries retain the pinned upload and skip completed parts after restart", async (t) => {
const root = await folder(t);
const file = join(root, "media.mp4");
const size = 65 * 1024 * 1024;
const handle = await open(file, "w");
await handle.truncate(size);
await handle.close();
const digests = await partDigests(file, size, 64 * 1024 * 1024, signal());
const inflight = { sent: [], multipart: {} };
const requests = [];
let fail = true;
let persisted = 0;
const job = {
api: {
uploadUrls: async (_space, _comp, _upload, files) => {
requests.push(...files);
return files.map((item) => ({
sha256: item.sha256,
multipartUploadId: "pinned",
expiresAt: "2099-01-01T00:00:00Z",
parts: item.parts.map((part) => ({
...part,
url: `https://store.test/${part.partNumber}`,
headers: { checksum: part.sha256 },
})),
}));
},
},
space: "space",
composition: "comp",
uploadId: "upload",
inflight,
local: new Map([[digests.whole, { path: "media.mp4", full: file }]]),
needed: [{ sha256: digests.whole, size }],
fetchImpl: async (url, init) => {
if (url.endsWith("/2") && fail) {
await init.body.cancel();
return new Response(null, { status: 400 });
}
const hash = createHash("sha256");
for await (const chunk of init.body) hash.update(chunk);
assert.equal(hash.digest("hex"), init.headers.checksum);
return new Response(null, { status: 200 });
},
signal: signal(),
wait: async () => {},
now: Date.now,
persist: () => persisted++,
onBytes: () => {},
onFile: () => {},
};
await assert.rejects(uploadNeeded(job), /refused/);
assert.deepEqual(inflight.multipart[digests.whole].partsDone, [1]);
fail = false;
await uploadNeeded(job);
assert.equal(requests[1].multipartUploadId, "pinned");
assert.deepEqual(
requests[1].parts.map((p) => p.partNumber),
[2],
);
assert.deepEqual(inflight.sent, [digests.whole]);
assert.ok(persisted >= 4);
});
test("signed URL expiry refreshes the grant without changing the uploaded content", async (t) => {
const root = await folder(t);
const file = join(root, "index.html");
await writeFile(file, "source");
const digest = createHash("sha256").update("source").digest("hex");
let grants = 0;
let puts = 0;
await uploadNeeded({
api: {
uploadUrls: async () => {
grants++;
return [
{
sha256: digest,
url: "https://store.test/file",
headers: {},
expiresAt: "2099-01-01T00:00:00Z",
},
];
},
},
space: "s",
composition: "c",
uploadId: "u",
inflight: { sent: [] },
local: new Map([[digest, { path: "index.html", full: file }]]),
needed: [{ sha256: digest, size: 6 }],
fetchImpl: async (_url, init) => {
puts++;
await new Response(init.body).arrayBuffer();
return new Response(null, { status: puts === 1 ? 403 : 200 });
},
signal: signal(),
wait: async () => {},
now: Date.now,
persist: () => {},
onBytes: () => {},
onFile: () => {},
});
assert.equal(grants, 2);
assert.equal(puts, 2);
});
test("storage PUT refuses a file swapped for a symlink before reading bytes", async (t) => {
const root = await folder(t);
const target = join(root, "private.txt");
const file = join(root, "asset.txt");
await writeFile(target, "secret");
await symlink(target, file);
let sent = false;
await assert.rejects(
putRange({
file,
path: "asset.txt",
start: 0,
length: 6,
sha256: "wrong",
signal: signal(),
url: "https://store.test",
headers: {},
fetchImpl: async () => {
sent = true;
return new Response();
},
}),
);
assert.equal(sent, false);
});
test("source PUT refuses untrusted or redirected targets before it opens a file", async () => {
for (const url of [
"https://untrusted.test/source",
"http://127.0.0.1:18080/source",
"https://store.test.untrusted.test/source",
"https://user:secret@store.test/source",
]) {
await assert.rejects(
putRange({
file: "/missing-source",
path: "index.html",
start: 0,
length: 6,
sha256: "unused",
signal: signal(),
url,
headers: {},
fetchImpl: async () => assert.fail("Must not send bytes"),
}),
{ kind: "refused" },
);
}
});
async function sourcePut(t, overrides = {}) {
const root = await folder(t);
const file = join(root, "index.html");
await writeFile(file, "source");
return {
file,
path: "index.html",
start: 0,
length: 6,
sha256: createHash("sha256").update("source").digest("hex"),
signal: signal(),
url: "https://store.test/file",
headers: {},
...overrides,
};
}
for (const consume of [false, true]) {
test(`source PUT times out when ${consume ? "the storage response" : "body consumption"} stalls`, async (t) => {
let requestSignal;
const put = await sourcePut(t, {
stallMs: 20,
signal: AbortSignal.timeout(2_000),
fetchImpl: async (_url, init) => {
requestSignal = init.signal;
if (consume) await new Response(init.body).arrayBuffer();
init.signal.throwIfAborted();
return new Promise((_resolve, reject) => {
init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true });
});
},
});
await assert.rejects(putRange(put), { kind: "transient" });
assert.equal(requestSignal.aborted, true);
});
}
test("source PUT preserves caller cancellation instead of retrying it", async (t) => {
const controller = new AbortController();
const cancelled = new DOMException("Sharing cancelled", "AbortError");
await assert.rejects(
putRange(
await sourcePut(t, {
signal: controller.signal,
fetchImpl: async (_url, init) => {
assert.equal(init.redirect, "error");
await new Response(init.body).arrayBuffer();
controller.abort(cancelled);
throw init.signal.reason;
},
}),
),
(error) => error === cancelled,
);
});
test("successful source PUT clears its stall timer", async (t) => {
let requestSignal;
await putRange(
await sourcePut(t, {
stallMs: 200,
fetchImpl: async (_url, init) => {
requestSignal = init.signal;
await new Response(init.body).arrayBuffer();
return new Response(null, { status: 200 });
},
}),
);
await new Promise((resolve) => setTimeout(resolve, 250));
assert.equal(requestSignal.aborted, false);
});
test("source PUT closes its file when resolving its parent fails", async (t) => {
const put = await sourcePut(t, { fetchImpl: async () => assert.fail("Must not send bytes") });
const originalOpen = fsPromises.open;
let opened;
t.mock.method(fsPromises, "open", async (...args) => {
opened = await originalOpen(...args);
return opened;
});
t.mock.method(fsPromises, "realpath", async () => {
throw Object.assign(new Error("Parent directory disappeared"), { code: "ENOENT" });
});
syncBuiltinESMExports();
try {
await assert.rejects(putRange(put), { code: "ENOENT" });
assert.equal(opened.fd, -1, "source file handle was left open");
} finally {
await opened?.close();
t.mock.restoreAll();
syncBuiltinESMExports();
}
});