diff --git a/docs/guides/export-and-share.mdx b/docs/guides/export-and-share.mdx
index 197ae481f7..07b0d3c0ba 100644
--- a/docs/guides/export-and-share.mdx
+++ b/docs/guides/export-and-share.mdx
@@ -138,9 +138,10 @@ Local images outside the project referenced by an `` or `` element'
`srcset` are included too. Their size or pixel-density descriptors are preserved,
and your source files stay unchanged.
-Only images, video, audio, fonts, CSS and JS files are copied from outside the
-project. Any other outside file, such as a Lottie `.json` or a `.pdf`, is left
-out with a warning naming it; move it into the project folder to include it.
+Only images, video, audio, fonts and CSS files are copied from outside the
+project. Any other outside file, such as a script, a Lottie `.json` or a `.pdf`,
+is left out with a warning naming it; move it into the project folder to include
+it.
Before sharing source, remove secrets and confirm that every included media file
can be shared. Tell a reviewer whether you need feedback on story, visuals,
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index 18335380e2..22474b78f8 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -2720,9 +2720,11 @@ authenticating, uploading, or starting anything. If you are over, add only
paths you have verified are unneeded to `.hyperframesignore`.
A file outside the project folder that a composition points at is copied into
-the zip only if it is an image, video, audio, font, CSS or JS file, judged by
-its extension. Any other outside file (a Lottie `.json`, a `.pdf`, a file with
-no extension), one in a private location (env files, Git data, hidden, system
+the zip only if it is an image, video, audio, font or CSS file, judged by its
+extension. A `.ts` or `.mts` file counts as video only when its first bytes are
+an MPEG transport stream, so TypeScript source is not copied. Any other outside
+file (a script, a Lottie `.json`, a `.pdf`, a file with no extension), one in a
+private location (env files, Git data, hidden, system
and app-data folders), and one reached from a file that came in with an
imported project are left out, with a warning naming each. To include such a
file, move it into the project folder. Under `--json`, the result lists those
diff --git a/packages/cli/src/utils/outsideFileType.ts b/packages/cli/src/utils/outsideFileType.ts
index ba4f8e20ca..2157864568 100644
--- a/packages/cli/src/utils/outsideFileType.ts
+++ b/packages/cli/src/utils/outsideFileType.ts
@@ -1,4 +1,4 @@
export {
- isCopiedOutsideFileType,
- OUTSIDE_FILE_EXTENSIONS,
+ isUploadedOutsideFileType,
+ UPLOADED_OUTSIDE_FILE_EXTENSIONS,
} from "@hyperframes/studio-server/outside-file";
diff --git a/packages/cli/src/utils/publishProject.fileType.test.ts b/packages/cli/src/utils/publishProject.fileType.test.ts
index 0af1d2abd6..28821209b6 100644
--- a/packages/cli/src/utils/publishProject.fileType.test.ts
+++ b/packages/cli/src/utils/publishProject.fileType.test.ts
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import type { LeftOutReference } from "./leftOutReference.js";
-import { OUTSIDE_FILE_EXTENSIONS } from "./outsideFileType.js";
+import { UPLOADED_OUTSIDE_FILE_EXTENSIONS } from "./outsideFileType.js";
import { stagePublishFiles } from "./publishProject.js";
const dirs: string[] = [];
@@ -71,7 +71,7 @@ describe("a file outside the project", () => {
});
expect(ext).toEqual([]);
const tail =
- "which is outside the project and is not an image, video, audio, font, CSS or JS file; it was left out of the upload. Move it into the project to include it.";
+ "which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.";
expect(warnings.sort()).toEqual(
[
`index.html references ../shared/scene.json, ${tail}`,
@@ -98,6 +98,57 @@ describe("a file outside the project", () => {
expect(index).toContain(`src="../shared/scene.json"`);
});
+ it("is left out when it is a script or TypeScript source, with a warning", () => {
+ const { ext, warnings, leftOut } = stage({
+ "shared/lib.js": "export const key = 1;",
+ "shared/lib.mjs": "export const key = 1;",
+ "shared/config.ts": "export const key: number = 1;",
+ "shared/config.mts": "export const key: number = 1;",
+ "p/index.html": [
+ ``,
+ ``,
+ ``,
+ ].join(""),
+ });
+ expect(ext).toEqual([]);
+ expect(warnings.sort()).toEqual(
+ ["lib.js", "lib.mjs", "config.ts", "config.mts"]
+ .map(
+ (name) =>
+ `index.html references ../shared/${name}, which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.`,
+ )
+ .sort(),
+ );
+ expect(leftOut.map((one) => one.reason)).toEqual(Array(4).fill("file_type"));
+ });
+
+ it("is copied when a .ts or .mts file is a transport stream video", () => {
+ const packets = (size: number, offset: number) => {
+ const bytes = Buffer.alloc(size * 3);
+ for (let at = offset; at < bytes.length; at += size) bytes[at] = 0x47;
+ return bytes.toString("latin1");
+ };
+ const root = mkdtempSync(join(tmpdir(), "hf-file-type-"));
+ dirs.push(root);
+ mkdirSync(join(root, "shared"));
+ mkdirSync(join(root, "p"));
+ writeFileSync(join(root, "shared", "clip.ts"), Buffer.from(packets(188, 0), "latin1"));
+ writeFileSync(join(root, "shared", "cam.mts"), Buffer.from(packets(192, 4), "latin1"));
+ writeFileSync(
+ join(root, "p", "index.html"),
+ ``,
+ );
+ const warnings: string[] = [];
+ const staged = stagePublishFiles(join(root, "p"), Infinity, (warning) =>
+ warnings.push(warning),
+ );
+ expect([...staged.keys()].filter((key) => key.startsWith("_ext/")).sort()).toEqual([
+ "_ext/shared/cam.mts",
+ "_ext/shared/clip.ts",
+ ]);
+ expect(warnings).toEqual([]);
+ });
+
it("follows the same rule for CSS url() targets and @import", () => {
const { ext, warnings, leftOut } = stage({
"shared/theme.css": "body {}",
@@ -162,18 +213,19 @@ describe("a file outside the project", () => {
});
describe("the outside file types", () => {
- it("are images, video, audio, fonts, CSS and JS, and nothing else", () => {
+ it("are images, video, audio, fonts and CSS, and nothing else", () => {
for (const extension of [".png", ".avif", ".svg", ".bmp", ".tif", ".mp4", ".mov", ".webm"]) {
- expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
+ expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
}
for (const extension of [".avi", ".ogv", ".mp3", ".wav", ".aiff"]) {
- expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
+ expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
}
- for (const extension of [".woff", ".woff2", ".ttf", ".otf", ".css", ".js", ".mjs"]) {
- expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
+ for (const extension of [".woff", ".woff2", ".ttf", ".otf", ".css"]) {
+ expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
}
- for (const extension of [".json", ".html", ".txt", ".md", ".cube", ".pdf", ".key", ""]) {
- expect(OUTSIDE_FILE_EXTENSIONS).not.toContain(extension);
+ const others = [".js", ".mjs", ".json", ".html", ".txt", ".md", ".cube", ".pdf", ".key", ""];
+ for (const extension of others) {
+ expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).not.toContain(extension);
}
});
});
diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts
index 65aa25d350..a4a79397aa 100644
--- a/packages/cli/src/utils/publishProject.ts
+++ b/packages/cli/src/utils/publishProject.ts
@@ -36,7 +36,7 @@ import {
type LeftOutReason,
type OnLeftOut,
} from "./leftOutReference.js";
-import { isCopiedOutsideFileType } from "./outsideFileType.js";
+import { isUploadedOutsideFileType } from "./outsideFileType.js";
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
@@ -509,7 +509,7 @@ function archiveTargetOf(
leaveOut(ctx, options.entryPath, rawPath, absPath);
return undefined;
}
- if (!isCopiedOutsideFileType(absPath)) {
+ if (!isUploadedOutsideFileType(absPath)) {
leaveOut(ctx, options.entryPath, rawPath, absPath, "file_type");
return undefined;
}
diff --git a/packages/studio-server/src/helpers/mediaExtensions.ts b/packages/studio-server/src/helpers/mediaExtensions.ts
index 5d48977f00..6626933903 100644
--- a/packages/studio-server/src/helpers/mediaExtensions.ts
+++ b/packages/studio-server/src/helpers/mediaExtensions.ts
@@ -40,7 +40,7 @@ export const MEDIA_INPUT_ARGS: readonly string[] = [
const PLAYLIST_START = /^(?:\xEF\xBB\xBF)?\s*(?:#EXTM3U|ffconcat)/;
const MARKUP_START = /^(?:\xEF\xBB\xBF)?\s*;
-const isControlByte = (byte: number) => byte < 0x09 || (byte > 0x0d && byte < 0x20);
+export const isControlByte = (byte: number) => byte < 0x09 || (byte > 0x0d && byte < 0x20);
export function startsAsMediaPlaylist(head: Buffer): boolean {
const text = head.toString("latin1");
diff --git a/packages/studio-server/src/helpers/outsideFile.test.ts b/packages/studio-server/src/helpers/outsideFile.test.ts
new file mode 100644
index 0000000000..498f01a413
--- /dev/null
+++ b/packages/studio-server/src/helpers/outsideFile.test.ts
@@ -0,0 +1,87 @@
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, describe, expect, it } from "vitest";
+import {
+ isCopiedOutsideFileType,
+ isUploadedOutsideFileType,
+ leftOutWarning,
+} from "./outsideFile.js";
+
+const dir = mkdtempSync(join(tmpdir(), "hf-outside-file-"));
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+function write(name: string, body: string | Buffer): string {
+ const path = join(dir, name);
+ writeFileSync(path, body);
+ return path;
+}
+
+/** Three packets of `size` bytes with the sync byte at `offset` in each. */
+function transportStream(size: number, offset: number): Buffer {
+ const bytes = Buffer.alloc(size * 3);
+ for (let at = offset; at < bytes.length; at += size) bytes[at] = 0x47;
+ return bytes;
+}
+
+describe("outside file types", () => {
+ it("copies scripts into a local render but not into an upload", () => {
+ for (const name of ["lib.js", "lib.mjs"]) {
+ const path = write(name, "export const key = 1;");
+ expect(isCopiedOutsideFileType(path)).toBe(true);
+ expect(isUploadedOutsideFileType(path)).toBe(false);
+ }
+ });
+
+ it("copies stylesheets, fonts and media into both", () => {
+ for (const name of ["theme.css", "brand.woff2", "clip.mp4", "logo.png"]) {
+ const path = write(name, "x");
+ expect(isCopiedOutsideFileType(path)).toBe(true);
+ expect(isUploadedOutsideFileType(path)).toBe(true);
+ }
+ });
+
+ it("treats a .ts or .mts file as video only when it is a transport stream", () => {
+ const source = [write("config.ts", "export const key: number = 1;"), write("config.mts", "G")];
+ for (const path of source) {
+ expect(isCopiedOutsideFileType(path)).toBe(false);
+ expect(isUploadedOutsideFileType(path)).toBe(false);
+ }
+ const video = [
+ write("clip.ts", transportStream(188, 0)),
+ write("cam.mts", transportStream(192, 4)),
+ write("disc.m2ts", "x"),
+ ];
+ for (const path of video) {
+ expect(isCopiedOutsideFileType(path)).toBe(true);
+ expect(isUploadedOutsideFileType(path)).toBe(true);
+ }
+ });
+
+ it("does not count text with a sync byte at every packet as a transport stream", () => {
+ const text = Array.from({ length: 4 }, () => `G${"/".repeat(187)}`).join("");
+ for (const name of ["spaced.ts", "spaced.mts"]) {
+ expect(isUploadedOutsideFileType(write(name, text))).toBe(false);
+ }
+ });
+
+ it.skipIf(process.platform === "win32")(
+ "refuses a named pipe without waiting for a writer",
+ () => {
+ const fifo = join(dir, "pipe.ts");
+ execFileSync("mkfifo", [fifo]);
+ expect(isCopiedOutsideFileType(fifo)).toBe(false);
+ expect(isUploadedOutsideFileType(fifo)).toBe(false);
+ },
+ );
+
+ it("names the copied types for the upload or render that left a file out", () => {
+ expect(leftOutWarning("index.html", "../lib.js", "file_type")).toBe(
+ "index.html references ../lib.js, which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.",
+ );
+ expect(leftOutWarning("index.html", "../a.pdf", "file_type", "render")).toBe(
+ "index.html references ../a.pdf, which is outside the project and is not an image, video, audio, font, CSS or JS file; it was left out of the render. Move it into the project to include it.",
+ );
+ });
+});
diff --git a/packages/studio-server/src/helpers/outsideFile.ts b/packages/studio-server/src/helpers/outsideFile.ts
index a1dc324005..2f6726586a 100644
--- a/packages/studio-server/src/helpers/outsideFile.ts
+++ b/packages/studio-server/src/helpers/outsideFile.ts
@@ -1,9 +1,11 @@
+import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, extname, isAbsolute, join, relative, sep } from "node:path";
import { MIME_TYPES } from "./mime.js";
import {
AUDIO_EXTS,
CONVERTIBLE_IMAGE_EXTS,
+ isControlByte,
MEDIA_INPUT_ARGS,
startsAsMediaPlaylist,
VIDEO_EXTS,
@@ -14,21 +16,76 @@ export { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, MEDIA_INPUT_ARGS, startsAsMediaPlay
const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/;
-/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types,
- * and the media cloud render re-encodes or converts. */
-export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set([
- ...Object.entries(MIME_TYPES)
- .filter(([, type]) => COPIED_TYPE.test(type))
- .map(([extension]) => extension),
+const extensionsOf = (type: RegExp) =>
+ Object.entries(MIME_TYPES)
+ .filter(([, mime]) => type.test(mime))
+ .map(([extension]) => extension);
+
+/** What a local render brings in from outside the project: the preview server's image, video, audio, font, CSS and
+ * JS types, and the media cloud render re-encodes or converts. */
+const OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set([
+ ...extensionsOf(COPIED_TYPE),
...VIDEO_EXTS,
...AUDIO_EXTS,
...CONVERTIBLE_IMAGE_EXTS,
]);
-const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase());
+const SCRIPT_EXTENSIONS: ReadonlySet = new Set(extensionsOf(/^text\/javascript;/));
+
+/** What an upload copies from outside the project: the same types without scripts. */
+export const UPLOADED_OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set(
+ [...OUTSIDE_FILE_EXTENSIONS].filter((extension) => !SCRIPT_EXTENSIONS.has(extension)),
+);
+
+const TYPESCRIPT_OR_TRANSPORT_STREAM = new Set([".ts", ".mts"]);
+const TS_SYNC_BYTE = 0x47;
+
+const TS_PACKETS = [
+ { size: 188, offset: 0 },
+ { size: 192, offset: 4 },
+];
+
+function readRegularFileHead(path: string, length: number): Buffer | null {
+ let fd: number;
+ try {
+ // A FIFO would wait for a writer on a blocking open; fstat refuses it first.
+ fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
+ } catch {
+ return null;
+ }
+ try {
+ if (!fstatSync(fd).isFile()) return null;
+ const head = Buffer.alloc(length);
+ return head.subarray(0, readSync(fd, head, 0, length, 0));
+ } catch {
+ return null;
+ } finally {
+ closeSync(fd);
+ }
+}
+
+function startsAsTransportStream(path: string): boolean {
+ const head = readRegularFileHead(path, 4 * 192);
+ if (!head?.some(isControlByte)) return false;
+ return TS_PACKETS.some(({ size, offset }) => {
+ const starts = [];
+ for (let at = offset; at < head.length; at += size) starts.push(at);
+ return starts.length >= 2 && starts.every((at) => head[at] === TS_SYNC_BYTE);
+ });
+}
+
+function isCopiedType(absPath: string, extensions: ReadonlySet): boolean {
+ const names = [absPath, realFilePath(absPath)].map((path) => extname(path).toLowerCase());
+ if (!names.every((extension) => extensions.has(extension))) return false;
+ const ambiguous = names.some((extension) => TYPESCRIPT_OR_TRANSPORT_STREAM.has(extension));
+ return !ambiguous || startsAsTransportStream(absPath);
+}
export const isCopiedOutsideFileType = (absPath: string): boolean =>
- hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath));
+ isCopiedType(absPath, OUTSIDE_FILE_EXTENSIONS);
+
+export const isUploadedOutsideFileType = (absPath: string): boolean =>
+ isCopiedType(absPath, UPLOADED_OUTSIDE_FILE_EXTENSIONS);
const ENV_FILE = /^\.env([._-].*|rc)?$/;
@@ -92,11 +149,15 @@ export const linksOutside = (projectDir: string, filePath: string): boolean =>
/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */
export type LeftOutReason = "blocklisted" | "received" | "file_type" | "private";
-const WHY: Record = {
+const WHY: Record, string> = {
private: "which holds secrets or tool data",
blocklisted: "which is outside the project in a private location",
received: "which leads outside the project and came in with an imported project",
- file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file",
+};
+
+const COPIED_TYPES = {
+ upload: "an image, video, audio, font or CSS file",
+ render: "an image, video, audio, font, CSS or JS file",
};
export function leftOutWarning(
@@ -105,6 +166,10 @@ export function leftOutWarning(
reason: LeftOutReason,
from: "upload" | "render" = "upload",
): string {
- const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the ${from}.`;
+ const why =
+ reason === "file_type"
+ ? `which is outside the project and is not ${COPIED_TYPES[from]}`
+ : WHY[reason];
+ const warning = `${file} references ${reference}, ${why}; it was left out of the ${from}.`;
return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning;
}