diff --git a/docs/guides/export-and-share.mdx b/docs/guides/export-and-share.mdx index 197ae481f7..07b0d3c0ba 100644 --- a/docs/guides/export-and-share.mdx +++ b/docs/guides/export-and-share.mdx @@ -138,9 +138,10 @@ Local images outside the project referenced by an `` or `` element' `srcset` are included too. Their size or pixel-density descriptors are preserved, and your source files stay unchanged. -Only images, video, audio, fonts, CSS and JS files are copied from outside the -project. Any other outside file, such as a Lottie `.json` or a `.pdf`, is left -out with a warning naming it; move it into the project folder to include it. +Only images, video, audio, fonts and CSS files are copied from outside the +project. Any other outside file, such as a script, a Lottie `.json` or a `.pdf`, +is left out with a warning naming it; move it into the project folder to include +it. Before sharing source, remove secrets and confirm that every included media file can be shared. Tell a reviewer whether you need feedback on story, visuals, diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 18335380e2..22474b78f8 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2720,9 +2720,11 @@ authenticating, uploading, or starting anything. If you are over, add only paths you have verified are unneeded to `.hyperframesignore`. A file outside the project folder that a composition points at is copied into -the zip only if it is an image, video, audio, font, CSS or JS file, judged by -its extension. Any other outside file (a Lottie `.json`, a `.pdf`, a file with -no extension), one in a private location (env files, Git data, hidden, system +the zip only if it is an image, video, audio, font or CSS file, judged by its +extension. A `.ts` or `.mts` file counts as video only when its first bytes are +an MPEG transport stream, so TypeScript source is not copied. Any other outside +file (a script, a Lottie `.json`, a `.pdf`, a file with no extension), one in a +private location (env files, Git data, hidden, system and app-data folders), and one reached from a file that came in with an imported project are left out, with a warning naming each. To include such a file, move it into the project folder. Under `--json`, the result lists those diff --git a/packages/cli/src/utils/outsideFileType.ts b/packages/cli/src/utils/outsideFileType.ts index ba4f8e20ca..2157864568 100644 --- a/packages/cli/src/utils/outsideFileType.ts +++ b/packages/cli/src/utils/outsideFileType.ts @@ -1,4 +1,4 @@ export { - isCopiedOutsideFileType, - OUTSIDE_FILE_EXTENSIONS, + isUploadedOutsideFileType, + UPLOADED_OUTSIDE_FILE_EXTENSIONS, } from "@hyperframes/studio-server/outside-file"; diff --git a/packages/cli/src/utils/publishProject.fileType.test.ts b/packages/cli/src/utils/publishProject.fileType.test.ts index 0af1d2abd6..28821209b6 100644 --- a/packages/cli/src/utils/publishProject.fileType.test.ts +++ b/packages/cli/src/utils/publishProject.fileType.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { LeftOutReference } from "./leftOutReference.js"; -import { OUTSIDE_FILE_EXTENSIONS } from "./outsideFileType.js"; +import { UPLOADED_OUTSIDE_FILE_EXTENSIONS } from "./outsideFileType.js"; import { stagePublishFiles } from "./publishProject.js"; const dirs: string[] = []; @@ -71,7 +71,7 @@ describe("a file outside the project", () => { }); expect(ext).toEqual([]); const tail = - "which is outside the project and is not an image, video, audio, font, CSS or JS file; it was left out of the upload. Move it into the project to include it."; + "which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it."; expect(warnings.sort()).toEqual( [ `index.html references ../shared/scene.json, ${tail}`, @@ -98,6 +98,57 @@ describe("a file outside the project", () => { expect(index).toContain(`src="../shared/scene.json"`); }); + it("is left out when it is a script or TypeScript source, with a warning", () => { + const { ext, warnings, leftOut } = stage({ + "shared/lib.js": "export const key = 1;", + "shared/lib.mjs": "export const key = 1;", + "shared/config.ts": "export const key: number = 1;", + "shared/config.mts": "export const key: number = 1;", + "p/index.html": [ + ``, + ``, + ``, + ].join(""), + }); + expect(ext).toEqual([]); + expect(warnings.sort()).toEqual( + ["lib.js", "lib.mjs", "config.ts", "config.mts"] + .map( + (name) => + `index.html references ../shared/${name}, which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.`, + ) + .sort(), + ); + expect(leftOut.map((one) => one.reason)).toEqual(Array(4).fill("file_type")); + }); + + it("is copied when a .ts or .mts file is a transport stream video", () => { + const packets = (size: number, offset: number) => { + const bytes = Buffer.alloc(size * 3); + for (let at = offset; at < bytes.length; at += size) bytes[at] = 0x47; + return bytes.toString("latin1"); + }; + const root = mkdtempSync(join(tmpdir(), "hf-file-type-")); + dirs.push(root); + mkdirSync(join(root, "shared")); + mkdirSync(join(root, "p")); + writeFileSync(join(root, "shared", "clip.ts"), Buffer.from(packets(188, 0), "latin1")); + writeFileSync(join(root, "shared", "cam.mts"), Buffer.from(packets(192, 4), "latin1")); + writeFileSync( + join(root, "p", "index.html"), + ``, + ); + const warnings: string[] = []; + const staged = stagePublishFiles(join(root, "p"), Infinity, (warning) => + warnings.push(warning), + ); + expect([...staged.keys()].filter((key) => key.startsWith("_ext/")).sort()).toEqual([ + "_ext/shared/cam.mts", + "_ext/shared/clip.ts", + ]); + expect(warnings).toEqual([]); + }); + it("follows the same rule for CSS url() targets and @import", () => { const { ext, warnings, leftOut } = stage({ "shared/theme.css": "body {}", @@ -162,18 +213,19 @@ describe("a file outside the project", () => { }); describe("the outside file types", () => { - it("are images, video, audio, fonts, CSS and JS, and nothing else", () => { + it("are images, video, audio, fonts and CSS, and nothing else", () => { for (const extension of [".png", ".avif", ".svg", ".bmp", ".tif", ".mp4", ".mov", ".webm"]) { - expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension); + expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension); } for (const extension of [".avi", ".ogv", ".mp3", ".wav", ".aiff"]) { - expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension); + expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension); } - for (const extension of [".woff", ".woff2", ".ttf", ".otf", ".css", ".js", ".mjs"]) { - expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension); + for (const extension of [".woff", ".woff2", ".ttf", ".otf", ".css"]) { + expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension); } - for (const extension of [".json", ".html", ".txt", ".md", ".cube", ".pdf", ".key", ""]) { - expect(OUTSIDE_FILE_EXTENSIONS).not.toContain(extension); + const others = [".js", ".mjs", ".json", ".html", ".txt", ".md", ".cube", ".pdf", ".key", ""]; + for (const extension of others) { + expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).not.toContain(extension); } }); }); diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index 65aa25d350..a4a79397aa 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -36,7 +36,7 @@ import { type LeftOutReason, type OnLeftOut, } from "./leftOutReference.js"; -import { isCopiedOutsideFileType } from "./outsideFileType.js"; +import { isUploadedOutsideFileType } from "./outsideFileType.js"; const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]); const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]); @@ -509,7 +509,7 @@ function archiveTargetOf( leaveOut(ctx, options.entryPath, rawPath, absPath); return undefined; } - if (!isCopiedOutsideFileType(absPath)) { + if (!isUploadedOutsideFileType(absPath)) { leaveOut(ctx, options.entryPath, rawPath, absPath, "file_type"); return undefined; } diff --git a/packages/studio-server/src/helpers/mediaExtensions.ts b/packages/studio-server/src/helpers/mediaExtensions.ts index 5d48977f00..6626933903 100644 --- a/packages/studio-server/src/helpers/mediaExtensions.ts +++ b/packages/studio-server/src/helpers/mediaExtensions.ts @@ -40,7 +40,7 @@ export const MEDIA_INPUT_ARGS: readonly string[] = [ const PLAYLIST_START = /^(?:\xEF\xBB\xBF)?\s*(?:#EXTM3U|ffconcat)/; const MARKUP_START = /^(?:\xEF\xBB\xBF)?\s* byte < 0x09 || (byte > 0x0d && byte < 0x20); +export const isControlByte = (byte: number) => byte < 0x09 || (byte > 0x0d && byte < 0x20); export function startsAsMediaPlaylist(head: Buffer): boolean { const text = head.toString("latin1"); diff --git a/packages/studio-server/src/helpers/outsideFile.test.ts b/packages/studio-server/src/helpers/outsideFile.test.ts new file mode 100644 index 0000000000..498f01a413 --- /dev/null +++ b/packages/studio-server/src/helpers/outsideFile.test.ts @@ -0,0 +1,87 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { + isCopiedOutsideFileType, + isUploadedOutsideFileType, + leftOutWarning, +} from "./outsideFile.js"; + +const dir = mkdtempSync(join(tmpdir(), "hf-outside-file-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +function write(name: string, body: string | Buffer): string { + const path = join(dir, name); + writeFileSync(path, body); + return path; +} + +/** Three packets of `size` bytes with the sync byte at `offset` in each. */ +function transportStream(size: number, offset: number): Buffer { + const bytes = Buffer.alloc(size * 3); + for (let at = offset; at < bytes.length; at += size) bytes[at] = 0x47; + return bytes; +} + +describe("outside file types", () => { + it("copies scripts into a local render but not into an upload", () => { + for (const name of ["lib.js", "lib.mjs"]) { + const path = write(name, "export const key = 1;"); + expect(isCopiedOutsideFileType(path)).toBe(true); + expect(isUploadedOutsideFileType(path)).toBe(false); + } + }); + + it("copies stylesheets, fonts and media into both", () => { + for (const name of ["theme.css", "brand.woff2", "clip.mp4", "logo.png"]) { + const path = write(name, "x"); + expect(isCopiedOutsideFileType(path)).toBe(true); + expect(isUploadedOutsideFileType(path)).toBe(true); + } + }); + + it("treats a .ts or .mts file as video only when it is a transport stream", () => { + const source = [write("config.ts", "export const key: number = 1;"), write("config.mts", "G")]; + for (const path of source) { + expect(isCopiedOutsideFileType(path)).toBe(false); + expect(isUploadedOutsideFileType(path)).toBe(false); + } + const video = [ + write("clip.ts", transportStream(188, 0)), + write("cam.mts", transportStream(192, 4)), + write("disc.m2ts", "x"), + ]; + for (const path of video) { + expect(isCopiedOutsideFileType(path)).toBe(true); + expect(isUploadedOutsideFileType(path)).toBe(true); + } + }); + + it("does not count text with a sync byte at every packet as a transport stream", () => { + const text = Array.from({ length: 4 }, () => `G${"/".repeat(187)}`).join(""); + for (const name of ["spaced.ts", "spaced.mts"]) { + expect(isUploadedOutsideFileType(write(name, text))).toBe(false); + } + }); + + it.skipIf(process.platform === "win32")( + "refuses a named pipe without waiting for a writer", + () => { + const fifo = join(dir, "pipe.ts"); + execFileSync("mkfifo", [fifo]); + expect(isCopiedOutsideFileType(fifo)).toBe(false); + expect(isUploadedOutsideFileType(fifo)).toBe(false); + }, + ); + + it("names the copied types for the upload or render that left a file out", () => { + expect(leftOutWarning("index.html", "../lib.js", "file_type")).toBe( + "index.html references ../lib.js, which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.", + ); + expect(leftOutWarning("index.html", "../a.pdf", "file_type", "render")).toBe( + "index.html references ../a.pdf, which is outside the project and is not an image, video, audio, font, CSS or JS file; it was left out of the render. Move it into the project to include it.", + ); + }); +}); diff --git a/packages/studio-server/src/helpers/outsideFile.ts b/packages/studio-server/src/helpers/outsideFile.ts index a1dc324005..2f6726586a 100644 --- a/packages/studio-server/src/helpers/outsideFile.ts +++ b/packages/studio-server/src/helpers/outsideFile.ts @@ -1,9 +1,11 @@ +import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, extname, isAbsolute, join, relative, sep } from "node:path"; import { MIME_TYPES } from "./mime.js"; import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, + isControlByte, MEDIA_INPUT_ARGS, startsAsMediaPlaylist, VIDEO_EXTS, @@ -14,21 +16,76 @@ export { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, MEDIA_INPUT_ARGS, startsAsMediaPlay const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/; -/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types, - * and the media cloud render re-encodes or converts. */ -export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set([ - ...Object.entries(MIME_TYPES) - .filter(([, type]) => COPIED_TYPE.test(type)) - .map(([extension]) => extension), +const extensionsOf = (type: RegExp) => + Object.entries(MIME_TYPES) + .filter(([, mime]) => type.test(mime)) + .map(([extension]) => extension); + +/** What a local render brings in from outside the project: the preview server's image, video, audio, font, CSS and + * JS types, and the media cloud render re-encodes or converts. */ +const OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set([ + ...extensionsOf(COPIED_TYPE), ...VIDEO_EXTS, ...AUDIO_EXTS, ...CONVERTIBLE_IMAGE_EXTS, ]); -const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase()); +const SCRIPT_EXTENSIONS: ReadonlySet = new Set(extensionsOf(/^text\/javascript;/)); + +/** What an upload copies from outside the project: the same types without scripts. */ +export const UPLOADED_OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set( + [...OUTSIDE_FILE_EXTENSIONS].filter((extension) => !SCRIPT_EXTENSIONS.has(extension)), +); + +const TYPESCRIPT_OR_TRANSPORT_STREAM = new Set([".ts", ".mts"]); +const TS_SYNC_BYTE = 0x47; + +const TS_PACKETS = [ + { size: 188, offset: 0 }, + { size: 192, offset: 4 }, +]; + +function readRegularFileHead(path: string, length: number): Buffer | null { + let fd: number; + try { + // A FIFO would wait for a writer on a blocking open; fstat refuses it first. + fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK); + } catch { + return null; + } + try { + if (!fstatSync(fd).isFile()) return null; + const head = Buffer.alloc(length); + return head.subarray(0, readSync(fd, head, 0, length, 0)); + } catch { + return null; + } finally { + closeSync(fd); + } +} + +function startsAsTransportStream(path: string): boolean { + const head = readRegularFileHead(path, 4 * 192); + if (!head?.some(isControlByte)) return false; + return TS_PACKETS.some(({ size, offset }) => { + const starts = []; + for (let at = offset; at < head.length; at += size) starts.push(at); + return starts.length >= 2 && starts.every((at) => head[at] === TS_SYNC_BYTE); + }); +} + +function isCopiedType(absPath: string, extensions: ReadonlySet): boolean { + const names = [absPath, realFilePath(absPath)].map((path) => extname(path).toLowerCase()); + if (!names.every((extension) => extensions.has(extension))) return false; + const ambiguous = names.some((extension) => TYPESCRIPT_OR_TRANSPORT_STREAM.has(extension)); + return !ambiguous || startsAsTransportStream(absPath); +} export const isCopiedOutsideFileType = (absPath: string): boolean => - hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath)); + isCopiedType(absPath, OUTSIDE_FILE_EXTENSIONS); + +export const isUploadedOutsideFileType = (absPath: string): boolean => + isCopiedType(absPath, UPLOADED_OUTSIDE_FILE_EXTENSIONS); const ENV_FILE = /^\.env([._-].*|rc)?$/; @@ -92,11 +149,15 @@ export const linksOutside = (projectDir: string, filePath: string): boolean => /** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */ export type LeftOutReason = "blocklisted" | "received" | "file_type" | "private"; -const WHY: Record = { +const WHY: Record, string> = { private: "which holds secrets or tool data", blocklisted: "which is outside the project in a private location", received: "which leads outside the project and came in with an imported project", - file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file", +}; + +const COPIED_TYPES = { + upload: "an image, video, audio, font or CSS file", + render: "an image, video, audio, font, CSS or JS file", }; export function leftOutWarning( @@ -105,6 +166,10 @@ export function leftOutWarning( reason: LeftOutReason, from: "upload" | "render" = "upload", ): string { - const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the ${from}.`; + const why = + reason === "file_type" + ? `which is outside the project and is not ${COPIED_TYPES[from]}` + : WHY[reason]; + const warning = `${file} references ${reference}, ${why}; it was left out of the ${from}.`; return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning; }