mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix: check, render and downloads work on a drive without hard links (#7737)
* fix(studio-server): check and render keep their history on a drive without hard links The history lock had its own list of the errors that mean a drive cannot hard-link, and it lacked EISDIR, which Windows returns on a Google Drive folder. Every check report and render record there failed. The lock now uses core's list, the one owner, which also gains ENOSYS from the lock. * fix(engine): downloads and plan blobs publish on a drive without hard links The producer's plan publisher kept a third list of no-hard-link errors without EISDIR, and the downloader had no fallback, so both failed on a Google Drive folder. Both now use core's cannotHardLink; the downloader copies into an exclusively created file, so it still never overwrites. * fix(engine): a download on a drive without hard links publishes atomically Instead of copying into an exclusively created file, the fallback renames the staged download into place under the cache path lock when nothing is there yet, so a killed run never leaves a truncated file to reuse.
This commit is contained in:
@@ -262,7 +262,7 @@ describe("createFileAtomically", () => {
|
||||
};
|
||||
}
|
||||
|
||||
it.each(["EPERM", "ENOTSUP", "EOPNOTSUPP", "EXDEV", "EISDIR", "EACCES"])(
|
||||
it.each(["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EXDEV", "EISDIR", "EACCES"])(
|
||||
"writes directly on a volume without hard links (%s)",
|
||||
(code) => {
|
||||
const dir = tempDir();
|
||||
|
||||
@@ -9,11 +9,23 @@ type DirectFileSystem = SiblingFileSystem & Pick<typeof fs, "openSync" | "closeS
|
||||
|
||||
// Codes a volume without hard links (FAT, exFAT, some network shares, Android app data) answers link() with.
|
||||
// EISDIR: libuv maps Windows ERROR_INVALID_FUNCTION (FAT/exFAT refusing a link) to it; nodejs/node#65817.
|
||||
const NO_HARD_LINKS = new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "EXDEV", "EISDIR", "EACCES"]);
|
||||
const NO_HARD_LINKS = new Set([
|
||||
"EPERM",
|
||||
"ENOTSUP",
|
||||
"EOPNOTSUPP",
|
||||
"ENOSYS",
|
||||
"EXDEV",
|
||||
"EISDIR",
|
||||
"EACCES",
|
||||
]);
|
||||
const MAX_LINK_HOPS = 40;
|
||||
const TEMP_NAME_TRIES = 3;
|
||||
const TEMP_SUFFIX = /\.hf[0-9a-f]{6}\.tmp$/;
|
||||
|
||||
export function cannotHardLink(error: unknown): boolean {
|
||||
return NO_HARD_LINKS.has(errorCode(error));
|
||||
}
|
||||
|
||||
/** A fresh temp sibling for a write that publishes `filePath`. 13 bytes, so a 242-byte name still fits. */
|
||||
export function atomicTempPath(filePath: string): string {
|
||||
return `${filePath}.hf${randomBytes(3).toString("hex")}.tmp`;
|
||||
@@ -73,7 +85,7 @@ export function createFileAtomically(
|
||||
try {
|
||||
operations.linkSync(tempPath, filePath);
|
||||
} catch (error) {
|
||||
if (!NO_HARD_LINKS.has(errorCode(error))) throw error;
|
||||
if (!cannotHardLink(error)) throw error;
|
||||
createDirectly(filePath, content, operations);
|
||||
}
|
||||
try {
|
||||
|
||||
@@ -29,6 +29,7 @@ const fsRaceControls = vi.hoisted(() => ({
|
||||
deleteBeforeLstatPath: undefined as string | undefined,
|
||||
deleteInjectedWinnerBeforeLstatPath: undefined as string | undefined,
|
||||
injectRaceAtLinkPath: undefined as string | undefined,
|
||||
failLinkWith: undefined as string | undefined,
|
||||
deleteBeforeReadPath: undefined as string | undefined,
|
||||
replaceStaleLockAfterObservationPath: undefined as string | undefined,
|
||||
replaceLockOnReleasePath: undefined as string | undefined,
|
||||
@@ -77,6 +78,11 @@ vi.mock("fs", async (importOriginal) => {
|
||||
fsRaceControls.injectRaceAtLinkPath = undefined;
|
||||
actual.writeFileSync(destination, "concurrent-winner");
|
||||
}
|
||||
if (fsRaceControls.failLinkWith) {
|
||||
throw Object.assign(new Error("hard links are not supported"), {
|
||||
code: fsRaceControls.failLinkWith,
|
||||
});
|
||||
}
|
||||
return Reflect.apply(actual.linkSync, actual, args);
|
||||
}) as typeof actual.linkSync,
|
||||
createReadStream: ((...args: unknown[]) => {
|
||||
@@ -156,6 +162,7 @@ afterEach(() => {
|
||||
fsRaceControls.deleteBeforeLstatPath = undefined;
|
||||
fsRaceControls.deleteInjectedWinnerBeforeLstatPath = undefined;
|
||||
fsRaceControls.injectRaceAtLinkPath = undefined;
|
||||
fsRaceControls.failLinkWith = undefined;
|
||||
fsRaceControls.deleteBeforeReadPath = undefined;
|
||||
fsRaceControls.replaceStaleLockAfterObservationPath = undefined;
|
||||
fsRaceControls.replaceLockOnReleasePath = undefined;
|
||||
@@ -1527,6 +1534,33 @@ describe("downloadToTemp atomic publication and bounded retry", () => {
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// fails without the fix: on a drive without hard links (EISDIR on a Google Drive folder) every download failed.
|
||||
it("publishes on a drive without hard links", async () => {
|
||||
const url = "https://cdn.example/no-hard-links.mp4";
|
||||
const dir = makeTempDir();
|
||||
fsRaceControls.failLinkWith = "EISDIR";
|
||||
stubFetch(vi.fn().mockImplementation(() => Promise.resolve(new Response("complete"))));
|
||||
|
||||
const path = await downloadToTemp(url, dir, 1_000);
|
||||
|
||||
expect(readFileSync(path, "utf8")).toBe("complete");
|
||||
expect(temporaryDownloadEntries(dir)).toEqual([]);
|
||||
});
|
||||
|
||||
it("still never overwrites a race winner on a drive without hard links", async () => {
|
||||
const url = "https://cdn.example/no-hard-links-race.mp4";
|
||||
const dir = makeTempDir();
|
||||
const cacheName = `download_${createHash("md5").update(url).digest("hex").slice(0, 12)}.mp4`;
|
||||
const cachePath = join(dir, cacheName);
|
||||
fsRaceControls.failLinkWith = "EISDIR";
|
||||
fsRaceControls.injectRaceAtLinkPath = cachePath;
|
||||
stubFetch(vi.fn().mockImplementation(() => Promise.resolve(new Response("complete"))));
|
||||
|
||||
await downloadToTemp(url, dir, 1_000).catch(() => undefined);
|
||||
|
||||
expect(readFileSync(cachePath, "utf8")).toBe("concurrent-winner");
|
||||
});
|
||||
|
||||
it("locally refetches when a concurrent race winner disappears before validation", async () => {
|
||||
const url = "https://cdn.example/vanishing-race-winner.mp4";
|
||||
const dir = makeTempDir();
|
||||
|
||||
@@ -23,7 +23,8 @@ import { isBlockedNetworkHost } from "./networkPolicy.js";
|
||||
import { dirname, extname, join } from "path";
|
||||
import { Readable, Transform } from "stream";
|
||||
import { pipeline } from "stream/promises";
|
||||
import { createStagingDir } from "@hyperframes/core/atomic-file";
|
||||
import { cannotHardLink, createStagingDir } from "@hyperframes/core/atomic-file";
|
||||
import { renameWithRetry } from "@hyperframes/core/rename-retry";
|
||||
|
||||
const inFlightDownloads = new Map<string, Promise<string>>();
|
||||
const signalScopes = new WeakMap<AbortSignal, number>();
|
||||
@@ -967,10 +968,36 @@ async function fetchToPartial(
|
||||
};
|
||||
}
|
||||
|
||||
function syncAndPublishPartial(
|
||||
async function publishWithoutClobber(
|
||||
partialPath: string,
|
||||
localPath: string,
|
||||
): Extract<UrlDownloadTelemetryOutcome, "published" | "race_reused"> {
|
||||
timeoutMs: number,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<void> {
|
||||
try {
|
||||
linkSync(partialPath, localPath);
|
||||
unlinkSync(partialPath);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!cannotHardLink(error)) throw error;
|
||||
}
|
||||
const releaseLock = await acquireCachePathLock(localPath, timeoutMs, signal);
|
||||
try {
|
||||
if (lstatSync(localPath, { throwIfNoEntry: false })) {
|
||||
throw Object.assign(new Error(`${localPath} already exists`), { code: "EEXIST" });
|
||||
}
|
||||
renameWithRetry(partialPath, localPath);
|
||||
} finally {
|
||||
releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
async function syncAndPublishPartial(
|
||||
partialPath: string,
|
||||
localPath: string,
|
||||
timeoutMs: number,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<Extract<UrlDownloadTelemetryOutcome, "published" | "race_reused">> {
|
||||
// Windows rejects fsync on a read-only handle (EPERM); the partial is ours
|
||||
// and writable, so r+ preserves the same flush semantics cross-platform.
|
||||
const fd = openSync(partialPath, "r+");
|
||||
@@ -980,11 +1007,10 @@ function syncAndPublishPartial(
|
||||
closeSync(fd);
|
||||
}
|
||||
|
||||
// A hard-link publish is atomic and no-clobber: unlike rename(), it cannot
|
||||
// replace a path that another successful caller has already returned.
|
||||
// Atomic and no-clobber: a hard link, or a rename under the cache path lock where
|
||||
// the drive has no hard links, never replaces a path another caller already returned.
|
||||
try {
|
||||
linkSync(partialPath, localPath);
|
||||
unlinkSync(partialPath);
|
||||
await publishWithoutClobber(partialPath, localPath, timeoutMs, signal);
|
||||
return "published";
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
@@ -1057,7 +1083,7 @@ async function runDownloadAttempt(
|
||||
throw new UrlDownloadError("cancelled", false, "Download cancelled");
|
||||
}
|
||||
const integrity = await fetchToPartial(url, partialPath, controller, options, extendTimeout);
|
||||
let outcome = syncAndPublishPartial(partialPath, localPath);
|
||||
let outcome = await syncAndPublishPartial(partialPath, localPath, timeoutMs, signal);
|
||||
let publishedIntegrity = integrity;
|
||||
if (outcome === "race_reused") {
|
||||
let inspection: Awaited<ReturnType<typeof inspectExistingFile>>;
|
||||
|
||||
@@ -1005,35 +1005,39 @@ describe("Plan v2 artifact publisher", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to an atomic copy when hard-linking is unavailable", async () => {
|
||||
const root = tempPath("hf-plan-v2-publisher-copy-");
|
||||
const v1 = createV1Plan(root);
|
||||
const destination = join(root, "v2");
|
||||
const publisher = new LocalPlanV2ArtifactPublisher(destination, {
|
||||
linkFile() {
|
||||
throw Object.assign(new Error("cross-device link"), { code: "EXDEV" });
|
||||
},
|
||||
});
|
||||
const manifest = await publishPlanV2FromV1(v1, publisher);
|
||||
const artifact = manifest.artifacts.find(
|
||||
(candidate) => candidate.path === "compiled/asset.txt",
|
||||
);
|
||||
if (artifact === undefined) throw new Error("test fixture is missing compiled/asset.txt");
|
||||
const sourcePath = join(v1, artifact.path);
|
||||
const blobPath = join(
|
||||
destination,
|
||||
"artifacts",
|
||||
"sha256",
|
||||
artifact.sha256.slice(0, 2),
|
||||
artifact.sha256,
|
||||
);
|
||||
// fails without the fix: EISDIR (Windows on a Google Drive folder) failed the publish instead of copying.
|
||||
it.each(["EXDEV", "EISDIR", "ENOSYS", "EOPNOTSUPP"])(
|
||||
"falls back to an atomic copy when hard-linking fails with %s",
|
||||
async (code) => {
|
||||
const root = tempPath("hf-plan-v2-publisher-copy-");
|
||||
const v1 = createV1Plan(root);
|
||||
const destination = join(root, "v2");
|
||||
const publisher = new LocalPlanV2ArtifactPublisher(destination, {
|
||||
linkFile() {
|
||||
throw Object.assign(new Error("hard links are not supported"), { code });
|
||||
},
|
||||
});
|
||||
const manifest = await publishPlanV2FromV1(v1, publisher);
|
||||
const artifact = manifest.artifacts.find(
|
||||
(candidate) => candidate.path === "compiled/asset.txt",
|
||||
);
|
||||
if (artifact === undefined) throw new Error("test fixture is missing compiled/asset.txt");
|
||||
const sourcePath = join(v1, artifact.path);
|
||||
const blobPath = join(
|
||||
destination,
|
||||
"artifacts",
|
||||
"sha256",
|
||||
artifact.sha256.slice(0, 2),
|
||||
artifact.sha256,
|
||||
);
|
||||
|
||||
expect(readFileSync(blobPath)).toEqual(readFileSync(sourcePath));
|
||||
expect({ dev: statSync(blobPath).dev, ino: statSync(blobPath).ino }).not.toEqual({
|
||||
dev: statSync(sourcePath).dev,
|
||||
ino: statSync(sourcePath).ino,
|
||||
});
|
||||
});
|
||||
expect(readFileSync(blobPath)).toEqual(readFileSync(sourcePath));
|
||||
expect({ dev: statSync(blobPath).dev, ino: statSync(blobPath).ino }).not.toEqual({
|
||||
dev: statSync(sourcePath).dev,
|
||||
ino: statSync(sourcePath).ino,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("produces byte-identical local CAS output through both publication paths", async () => {
|
||||
const root = tempPath("hf-plan-v2-publisher-parity-");
|
||||
|
||||
@@ -11,7 +11,7 @@ import { renameWithRetry } from "@hyperframes/engine";
|
||||
import { dirname, join } from "node:path";
|
||||
import { PlanV2IntegrityError } from "./planV2Errors.js";
|
||||
import { assertPlanV2Sha256, planV2BlobPath } from "./planV2Layout.js";
|
||||
import { createStagingDir } from "@hyperframes/core/atomic-file";
|
||||
import { cannotHardLink, createStagingDir } from "@hyperframes/core/atomic-file";
|
||||
|
||||
export interface PlanV2PublishBlob {
|
||||
readonly sourcePath: string;
|
||||
@@ -33,12 +33,6 @@ export interface PlanV2ArtifactPublisher {
|
||||
abort(): Promise<void>;
|
||||
}
|
||||
|
||||
function canFallbackToCopy(error: unknown): boolean {
|
||||
if (!(error instanceof Error) || !("code" in error)) return false;
|
||||
const code = error.code;
|
||||
return code === "EXDEV" || code === "EPERM" || code === "EACCES" || code === "ENOTSUP";
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
@@ -108,7 +102,7 @@ export class LocalPlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
|
||||
try {
|
||||
this.#linkFile(blob.sourcePath, temporaryPath);
|
||||
} catch (error) {
|
||||
if (!canFallbackToCopy(error)) throw error;
|
||||
if (!cannotHardLink(error)) throw error;
|
||||
copyFileSync(blob.sourcePath, temporaryPath);
|
||||
}
|
||||
renameWithRetry(temporaryPath, destinationPath);
|
||||
|
||||
@@ -95,7 +95,8 @@ function withoutHardLinks(code = "ENOTSUP"): void {
|
||||
}
|
||||
|
||||
describe("takeHistoryOwnership on a drive without hard links", () => {
|
||||
it.each(["ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EPERM", "EACCES"])(
|
||||
// fails without the fix: EISDIR (Windows on a Google Drive folder) and EXDEV threw instead.
|
||||
it.each(["ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EPERM", "EACCES", "EISDIR", "EXDEV"])(
|
||||
"takes the lock when linking fails with %s",
|
||||
async (code) => {
|
||||
withoutHardLinks(code);
|
||||
|
||||
@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { linkSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { uptime } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
import { cannotHardLink, replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
|
||||
export class HistoryBusyError extends Error {
|
||||
constructor(
|
||||
@@ -16,7 +16,6 @@ export class HistoryBusyError extends Error {
|
||||
}
|
||||
|
||||
const WINDOWS_BUSY = new Set(["EPERM", "EBUSY", "EACCES"]);
|
||||
const NO_HARD_LINKS = new Set(["ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EPERM", "EACCES"]);
|
||||
const TAKING_MS = 2_000;
|
||||
const BOOT_SLACK_MS = 60_000;
|
||||
|
||||
@@ -116,9 +115,8 @@ function claim(file: string): boolean {
|
||||
try {
|
||||
linkSync(draft, file);
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code ?? "";
|
||||
if (code === "EEXIST") return false;
|
||||
if (!NO_HARD_LINKS.has(code)) throw error;
|
||||
if ((error as NodeJS.ErrnoException).code === "EEXIST") return false;
|
||||
if (!cannotHardLink(error)) throw error;
|
||||
return createExclusively(file);
|
||||
}
|
||||
return recordOwnership(file);
|
||||
|
||||
Reference in New Issue
Block a user