diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 71a0b9134b..a42b3b7719 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2017,8 +2017,11 @@ A composition that needs the left-out script stops with a script-load error. A received-files record lists the files in a folder that came from someone else. When a record in the project, or in a folder above it, lists anything but media, fonts and images, the render leaves out every file from outside the -project folder and warns with each one's name. A `--docker` render makes this check on your -machine, where the folders above the project are visible, and passes +project folder and warns with each one's name. When you delete a listed page, +stylesheet or script in Studio, its line stays, so whatever is later written in +its place still counts as received; take the line out of +`.hyperframes/received-files.json` yourself once you know the content is gone. +A `--docker` render makes this check on your machine, where the folders above the project are visible, and passes `--refuse-outside-files` into the container, so it leaves out the same files a local render would. diff --git a/packages/cli/src/server/followReceivedRecords.ts b/packages/cli/src/server/followReceivedRecords.ts index e3c853f592..97a5322c71 100644 --- a/packages/cli/src/server/followReceivedRecords.ts +++ b/packages/cli/src/server/followReceivedRecords.ts @@ -72,7 +72,7 @@ async function startedChange( * Follows a Studio rename, move or delete in the received-files records. Before the files move, every file listed in * a record inside a moved or deleted folder is listed in the project's own record, the received content leaving a * listed path is kept, and a renamed file is listed under its new path; when that fails, the request is refused. - * Once the files have moved, their old lines go. + * Afterwards old lines go, but a non-media file deleted or renamed to a media name stays listed, still received. */ function followMovesAndDeletes(app: Hono, projectDir: string): void { app.on(["PATCH", "DELETE"], "/api/projects/:id/files/*", async (c, next) => { diff --git a/packages/cli/src/server/studioServer.receivedRecord.test.ts b/packages/cli/src/server/studioServer.receivedRecord.test.ts index 7956aa7774..db9ffad614 100644 --- a/packages/cli/src/server/studioServer.receivedRecord.test.ts +++ b/packages/cli/src/server/studioServer.receivedRecord.test.ts @@ -80,6 +80,7 @@ const move = (app: StudioServer["app"], from: string, newPath: string) => headers: { "content-type": "application/json" }, body: JSON.stringify({ newPath }), }); +const PNG = Buffer.from("89504e470d0a1a0a0000000d49484452", "hex"); const allowed = (projectDir: string) => outsideAssetsAllowed(projectDir, "index.html", () => {}); describe("a received folder with its own record, in a project with none", () => { @@ -104,19 +105,20 @@ describe("a received folder with its own record, in a project with none", () => }, ); - it("takes its received files along when Studio deletes it, keeping their content", async () => { + it("keeps its received pages listed when Studio deletes it, keeping their content", async () => { const { projectDir, app } = studio(); withReceivedFolder(projectDir); const scene = contentHash(join(projectDir, "sub", "scene.html")); expect((await app.request(fileUrl("sub"), { method: "DELETE" })).status).toBe(200); expect(existsSync(join(projectDir, "sub"))).toBe(false); const own = join(projectDir, ".hyperframes"); - expect(JSON.parse(readFileSync(join(own, "received-files.json"), "utf-8")).files).toEqual([]); + expect(JSON.parse(readFileSync(join(own, "received-files.json"), "utf-8")).files).toEqual([ + "sub/scene.html", + ]); const content = JSON.parse(readFileSync(join(own, "received-content.json"), "utf-8")); expect(content.hashes).toEqual([scene]); - const received = receivedFiles(projectDir); - expect(received.gone).toBeUndefined(); - expect(received("index.html")).toBe(false); + expect(receivedFiles(projectDir).gone).toBe("sub/scene.html"); + expect(allowed(projectDir)).toBe(false); }); it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( @@ -191,10 +193,18 @@ describe("a Studio rename, move or delete in a project with a received-files rec function withRecord(projectDir: string): void { write(join(projectDir, "index.html"), `
`); write(join(projectDir, "scenes", "intro.html"), ``); - write(join(projectDir, "scenes", "logo.png"), "x"); + write(join(projectDir, "scenes", "logo.png"), PNG); write(record(projectDir), JSON.stringify({ files: ["scenes/intro.html", "scenes/logo.png"] })); } + /** Deletes the received page in Studio, then takes its line out by hand; its content stays kept as received. */ + async function deleteAndForget(app: StudioServer["app"], projectDir: string): Promise { + expect((await app.request(fileUrl("scenes/intro.html"), { method: "DELETE" })).status).toBe( + 200, + ); + write(record(projectDir), JSON.stringify({ files: ["scenes/logo.png"] })); + } + it("keeps a renamed received file received under its new name", async () => { const { projectDir, app } = studio(); withRecord(projectDir); @@ -214,13 +224,72 @@ describe("a Studio rename, move or delete in a project with a received-files rec expect(receivedFiles(projectDir)("index.html")).toBe(false); }); - it("takes a deleted received file off the record, so the person's own files stay theirs", async () => { + it("keeps a deleted received page listed, and takes a deleted received image off the record", async () => { const { projectDir, app } = studio(); withRecord(projectDir); - const deleted = await app.request(fileUrl("scenes/intro.html"), { method: "DELETE" }); - expect(deleted.status).toBe(200); - expect(listed(projectDir)).toEqual(["scenes/logo.png"]); - expect(receivedFiles(projectDir)("index.html")).toBe(false); + expect((await app.request(fileUrl("scenes/intro.html"), { method: "DELETE" })).status).toBe( + 200, + ); + expect((await app.request(fileUrl("scenes/logo.png"), { method: "DELETE" })).status).toBe(200); + expect(listed(projectDir)).toEqual(["scenes/intro.html"]); + expect(receivedFiles(projectDir).gone).toBe("scenes/intro.html"); + }); + + it("lets a project whose record lists only media keep reading outside files after Studio deletes one", async () => { + const { projectDir, app } = studio(); + write(join(projectDir, "index.html"), ``); + write(join(projectDir, "logo.png"), PNG); + write(join(projectDir, "intro.png"), PNG); + write(record(projectDir), JSON.stringify({ files: ["intro.png", "logo.png"] })); + expect(allowed(projectDir)).toBe(true); + expect((await app.request(fileUrl("logo.png"), { method: "DELETE" })).status).toBe(200); + expect(listed(projectDir)).toEqual(["intro.png"]); + expect(allowed(projectDir)).toBe(true); + }); + + const create = (app: StudioServer["app"], path: string, body: string) => + app.request(fileUrl(path), { method: "POST", body }); + + it.each([["scenes/intro.html"], ["scenes/other.html"], ["index2.html"]])( + "still refuses outside files after every received page is deleted and new bytes are created at %s", + async (path) => { + const { projectDir, app } = studio(); + withRecord(projectDir); + for (const file of ["scenes/intro.html", "index.html"]) + expect((await app.request(fileUrl(file), { method: "DELETE" })).status).toBe(200); + expect((await create(app, path, ``)).status).toBe( + 201, + ); + expect( + (await create(app, "index.html", `
`)).status, + ).toBe(201); + expect(listed(projectDir)).toContain("scenes/intro.html"); + expect(receivedFiles(projectDir)(path)).toBe(true); + expect(allowed(projectDir)).toBe(false); + }, + ); + + it("still refuses outside files after a received page is renamed to an image name and saved as one", async () => { + const { projectDir, app } = studio(); + withRecord(projectDir); + write(record(projectDir), JSON.stringify({ files: ["scenes/intro.html"] })); + expect((await move(app, "scenes/intro.html", "scenes/intro.png")).status).toBe(200); + expect(listed(projectDir)).toEqual(["scenes/intro.html", "scenes/intro.png"]); + const before = readFileSync(join(projectDir, "scenes", "intro.png")); + const saved = await app.request(fileUrl("scenes/intro.png"), { + method: "PUT", + headers: { "If-Match": fileContentVersion(before) }, + body: PNG, + }); + expect(saved.status).toBe(200); + expect(allowed(projectDir)).toBe(false); + }); + + it("drops the old line when a received image is renamed to another image name", async () => { + const { projectDir, app } = studio(); + withRecord(projectDir); + expect((await move(app, "scenes/logo.png", "scenes/mark.png")).status).toBe(200); + expect(listed(projectDir)).toEqual(["scenes/intro.html", "scenes/mark.png"]); }); it("counts a deleted received file as received again when its content comes back", async () => { @@ -239,7 +308,6 @@ describe("a Studio rename, move or delete in a project with a received-files rec const received = receivedFiles(projectDir); expect(received("scenes/intro.html")).toBe(true); expect(received("restored.html")).toBe(true); - expect(received("index.html")).toBe(false); expect(allowed(projectDir)).toBe(false); }); @@ -326,7 +394,7 @@ describe("a Studio rename, move or delete in a project with a received-files rec expect((await app.request(fileUrl("scenes/intro.html"), { method: "DELETE" })).status).toBe( 200, ); - expect(listed(projectDir)).toEqual(["scenes/logo.png"]); + expect(listed(projectDir)).toEqual(["scenes/intro.html", "scenes/logo.png"]); const restored = await app.request("/api/projects/film/history/restore", { method: "POST", headers: { "content-type": "application/json" }, @@ -344,9 +412,7 @@ describe("a Studio rename, move or delete in a project with a received-files rec withRecord(projectDir); const intro = join(projectDir, "scenes", "intro.html"); const before = readFileSync(intro); - expect((await app.request(fileUrl("scenes/intro.html"), { method: "DELETE" })).status).toBe( - 200, - ); + await deleteAndForget(app, projectDir); writeFileSync(intro, before); const saved = await app.request(fileUrl("scenes/intro.html"), { method: "PUT", @@ -377,9 +443,7 @@ describe("a Studio rename, move or delete in a project with a received-files rec withRecord(projectDir); const intro = join(projectDir, "scenes", "intro.html"); const before = readFileSync(intro); - expect((await app.request(fileUrl("scenes/intro.html"), { method: "DELETE" })).status).toBe( - 200, - ); + await deleteAndForget(app, projectDir); writeFileSync(intro, before); await app.request(`/api/projects/film/file-mutations/${operation}`, { method: "POST", @@ -395,9 +459,7 @@ describe("a Studio rename, move or delete in a project with a received-files rec withRecord(projectDir); const intro = join(projectDir, "scenes", "intro.html"); const before = readFileSync(intro); - expect((await app.request(fileUrl("scenes/intro.html"), { method: "DELETE" })).status).toBe( - 200, - ); + await deleteAndForget(app, projectDir); writeFileSync(intro, before); writeFileSync(`${record(projectDir)}.lock`, ""); const saved = await app.request(fileUrl("scenes/intro.html"), { @@ -437,7 +499,7 @@ describe("a Studio rename, move or delete in a project with a received-files rec expect((await app.request(fileUrl("scenes/intro.html"), { method: "DELETE" })).status).toBe( 200, ); - expect(listed(projectDir)).toEqual(["scenes/logo.png"]); + expect(listed(projectDir)).toEqual(["scenes/intro.html", "scenes/logo.png"]); await server?.shutdown(); const saved = { ...historyDeps }; Object.assign(historyDeps, { @@ -465,11 +527,12 @@ describe("a Studio rename, move or delete in a project with a received-files rec JSON.stringify({ files: ["scenes/intro.html", "scenes/old.html", "parts/gone.html"] }), ); expect((await app.request(fileUrl("scenes"), { method: "DELETE" })).status).toBe(200); - expect(listed(projectDir)).toEqual(["parts/gone.html", "scenes/old.html"]); + const kept = ["parts/gone.html", "scenes/intro.html", "scenes/old.html"]; + expect(listed(projectDir).sort()).toEqual(kept); write(join(projectDir, "draft.html"), "

"); expect((await move(app, "draft.html", "parts/gone.html")).status).toBe(200); - expect(listed(projectDir)).toEqual(["parts/gone.html", "scenes/old.html"]); - expect(receivedFiles(projectDir).gone).toBe("scenes/old.html"); + expect(listed(projectDir).sort()).toEqual(kept); + expect(receivedFiles(projectDir).gone).toBe("scenes/intro.html"); }); it("moves a line its record spells in decomposed Unicode", async () => { diff --git a/packages/cli/src/utils/receivedRecordFollow.ts b/packages/cli/src/utils/receivedRecordFollow.ts index defda9e28d..1d6f72d9b7 100644 --- a/packages/cli/src/utils/receivedRecordFollow.ts +++ b/packages/cli/src/utils/receivedRecordFollow.ts @@ -9,7 +9,7 @@ import { receivedLinesAtOrAbove, } from "./privateProjectFile.js"; import { contentHashAsync, hashable, readReceivedContent } from "./receivedContent.js"; -import { isMediaFontOrImage } from "./receivedRender.js"; +import { isMediaFontOrImage, isMediaName } from "./receivedRender.js"; type Lines = Map; @@ -17,6 +17,7 @@ type Lines = Map; interface Leaving { line: string; hash?: string; + stays?: boolean; } /** Worked out while the files are on disk. A line whose file is already gone stays, as the sign it is gone. */ @@ -111,9 +112,10 @@ export async function keepListedInFolder(projectDir: string, target: string): Pr async function leaving(line: string, path: string): Promise { const stat = await lstat(path).catch(() => null); if (stat === null) return null; - if (isMediaFontOrImage(path) || (stat.isFile() && !hashable(path, stat))) return { line }; + if (isMediaFontOrImage(path)) return { line }; + if (stat.isFile() && !hashable(path, stat)) return { line, stays: true }; const hash = await contentHashAsync(path); - return hash === null ? null : { line, hash }; + return hash === null ? null : { line, hash, stays: true }; } export async function plannedReceivedChange( @@ -129,10 +131,16 @@ export async function plannedReceivedChange( for (const line of lines.filter(at)) { const left = await leaving(line, join(folder, line)); if (left === null) continue; - push(change.leaving, folder, left); - if (to === undefined) continue; + if (to === undefined) { + push(change.leaving, folder, left); + continue; + } const rest = line.normalize("NFC").slice(fromKey.length); const toKey = keyIn(folder, to); + push(change.leaving, folder, { + ...left, + stays: left.stays === true && isMediaName(toKey + rest), + }); if (leavesFolder(toKey)) push(change.arriving, root, keyIn(root, to) + rest); else push(change.arriving, folder, toKey + rest); } @@ -170,7 +178,12 @@ export async function recordReceivedArrivals(change: ReceivedChange): Promise => dropLines( - new Map([...change.leaving].map(([folder, lines]) => [folder, lines.map((left) => left.line)])), + new Map( + [...change.leaving].map(([folder, lines]) => [ + folder, + lines.flatMap((left) => (left.stays ? [] : [left.line])), + ]), + ), ); export const dropReceivedArrivals = (change: ReceivedChange): Promise => diff --git a/packages/cli/src/utils/receivedRender.ts b/packages/cli/src/utils/receivedRender.ts index 871454c32c..20c3a5702a 100644 --- a/packages/cli/src/utils/receivedRender.ts +++ b/packages/cli/src/utils/receivedRender.ts @@ -200,7 +200,8 @@ function listDir(dir: string): Dirent[] | null { } } -const isMediaName = (path: string) => MEDIA_SIGNATURES[extname(path).toLowerCase()] !== undefined; +export const isMediaName = (path: string): boolean => + MEDIA_SIGNATURES[extname(path).toLowerCase()] !== undefined; // A linked folder is checked when it leads to a folder inside the project, which the walk lists under its own name. function linkedFolderChecked(path: string, realProjectDir: string): boolean {