fix(cli): uploads leave out outside scripts; .ts counts as video only when it is one (#7818)

* fix(cli): uploads leave out outside scripts; .ts counts as video only when it is one

* chore(studio-server): raise unpacked size budget for the upload file-type rule

* fix(studio-server): transport stream check never waits on a pipe and needs binary bytes
This commit is contained in:
Michael Wang
2026-11-15 09:52:49 +00:00
committed by GitHub
parent ac02474105
commit 04202e8a4c
8 changed files with 238 additions and 31 deletions
+4 -3
View File
@@ -138,9 +138,10 @@ Local images outside the project referenced by an `<img>` or `<source>` element'
`srcset` are included too. Their size or pixel-density descriptors are preserved,
and your source files stay unchanged.
Only images, video, audio, fonts, CSS and JS files are copied from outside the
project. Any other outside file, such as a Lottie `.json` or a `.pdf`, is left
out with a warning naming it; move it into the project folder to include it.
Only images, video, audio, fonts and CSS files are copied from outside the
project. Any other outside file, such as a script, a Lottie `.json` or a `.pdf`,
is left out with a warning naming it; move it into the project folder to include
it.
Before sharing source, remove secrets and confirm that every included media file
can be shared. Tell a reviewer whether you need feedback on story, visuals,
+5 -3
View File
@@ -2720,9 +2720,11 @@ authenticating, uploading, or starting anything. If you are over, add only
paths you have verified are unneeded to `.hyperframesignore`.
A file outside the project folder that a composition points at is copied into
the zip only if it is an image, video, audio, font, CSS or JS file, judged by
its extension. Any other outside file (a Lottie `.json`, a `.pdf`, a file with
no extension), one in a private location (env files, Git data, hidden, system
the zip only if it is an image, video, audio, font or CSS file, judged by its
extension. A `.ts` or `.mts` file counts as video only when its first bytes are
an MPEG transport stream, so TypeScript source is not copied. Any other outside
file (a script, a Lottie `.json`, a `.pdf`, a file with no extension), one in a
private location (env files, Git data, hidden, system
and app-data folders), and one reached from a file that came in with an
imported project are left out, with a warning naming each. To include such a
file, move it into the project folder. Under `--json`, the result lists those
+2 -2
View File
@@ -1,4 +1,4 @@
export {
isCopiedOutsideFileType,
OUTSIDE_FILE_EXTENSIONS,
isUploadedOutsideFileType,
UPLOADED_OUTSIDE_FILE_EXTENSIONS,
} from "@hyperframes/studio-server/outside-file";
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import type { LeftOutReference } from "./leftOutReference.js";
import { OUTSIDE_FILE_EXTENSIONS } from "./outsideFileType.js";
import { UPLOADED_OUTSIDE_FILE_EXTENSIONS } from "./outsideFileType.js";
import { stagePublishFiles } from "./publishProject.js";
const dirs: string[] = [];
@@ -71,7 +71,7 @@ describe("a file outside the project", () => {
});
expect(ext).toEqual([]);
const tail =
"which is outside the project and is not an image, video, audio, font, CSS or JS file; it was left out of the upload. Move it into the project to include it.";
"which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.";
expect(warnings.sort()).toEqual(
[
`index.html references ../shared/scene.json, ${tail}`,
@@ -98,6 +98,57 @@ describe("a file outside the project", () => {
expect(index).toContain(`src="../shared/scene.json"`);
});
it("is left out when it is a script or TypeScript source, with a warning", () => {
const { ext, warnings, leftOut } = stage({
"shared/lib.js": "export const key = 1;",
"shared/lib.mjs": "export const key = 1;",
"shared/config.ts": "export const key: number = 1;",
"shared/config.mts": "export const key: number = 1;",
"p/index.html": [
`<script src="../shared/lib.js"></script>`,
`<script type="module" src="../shared/lib.mjs"></script>`,
`<video src="../shared/config.ts"></video><link href="../shared/config.mts">`,
].join(""),
});
expect(ext).toEqual([]);
expect(warnings.sort()).toEqual(
["lib.js", "lib.mjs", "config.ts", "config.mts"]
.map(
(name) =>
`index.html references ../shared/${name}, which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.`,
)
.sort(),
);
expect(leftOut.map((one) => one.reason)).toEqual(Array(4).fill("file_type"));
});
it("is copied when a .ts or .mts file is a transport stream video", () => {
const packets = (size: number, offset: number) => {
const bytes = Buffer.alloc(size * 3);
for (let at = offset; at < bytes.length; at += size) bytes[at] = 0x47;
return bytes.toString("latin1");
};
const root = mkdtempSync(join(tmpdir(), "hf-file-type-"));
dirs.push(root);
mkdirSync(join(root, "shared"));
mkdirSync(join(root, "p"));
writeFileSync(join(root, "shared", "clip.ts"), Buffer.from(packets(188, 0), "latin1"));
writeFileSync(join(root, "shared", "cam.mts"), Buffer.from(packets(192, 4), "latin1"));
writeFileSync(
join(root, "p", "index.html"),
`<video src="../shared/clip.ts"></video><video src="../shared/cam.mts"></video>`,
);
const warnings: string[] = [];
const staged = stagePublishFiles(join(root, "p"), Infinity, (warning) =>
warnings.push(warning),
);
expect([...staged.keys()].filter((key) => key.startsWith("_ext/")).sort()).toEqual([
"_ext/shared/cam.mts",
"_ext/shared/clip.ts",
]);
expect(warnings).toEqual([]);
});
it("follows the same rule for CSS url() targets and @import", () => {
const { ext, warnings, leftOut } = stage({
"shared/theme.css": "body {}",
@@ -162,18 +213,19 @@ describe("a file outside the project", () => {
});
describe("the outside file types", () => {
it("are images, video, audio, fonts, CSS and JS, and nothing else", () => {
it("are images, video, audio, fonts and CSS, and nothing else", () => {
for (const extension of [".png", ".avif", ".svg", ".bmp", ".tif", ".mp4", ".mov", ".webm"]) {
expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
}
for (const extension of [".avi", ".ogv", ".mp3", ".wav", ".aiff"]) {
expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
}
for (const extension of [".woff", ".woff2", ".ttf", ".otf", ".css", ".js", ".mjs"]) {
expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
for (const extension of [".woff", ".woff2", ".ttf", ".otf", ".css"]) {
expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
}
for (const extension of [".json", ".html", ".txt", ".md", ".cube", ".pdf", ".key", ""]) {
expect(OUTSIDE_FILE_EXTENSIONS).not.toContain(extension);
const others = [".js", ".mjs", ".json", ".html", ".txt", ".md", ".cube", ".pdf", ".key", ""];
for (const extension of others) {
expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).not.toContain(extension);
}
});
});
+2 -2
View File
@@ -36,7 +36,7 @@ import {
type LeftOutReason,
type OnLeftOut,
} from "./leftOutReference.js";
import { isCopiedOutsideFileType } from "./outsideFileType.js";
import { isUploadedOutsideFileType } from "./outsideFileType.js";
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
@@ -509,7 +509,7 @@ function archiveTargetOf(
leaveOut(ctx, options.entryPath, rawPath, absPath);
return undefined;
}
if (!isCopiedOutsideFileType(absPath)) {
if (!isUploadedOutsideFileType(absPath)) {
leaveOut(ctx, options.entryPath, rawPath, absPath, "file_type");
return undefined;
}
@@ -40,7 +40,7 @@ export const MEDIA_INPUT_ARGS: readonly string[] = [
const PLAYLIST_START = /^(?:\xEF\xBB\xBF)?\s*(?:#EXTM3U|ffconcat)/;
const MARKUP_START = /^(?:\xEF\xBB\xBF)?\s*</;
const isControlByte = (byte: number) => byte < 0x09 || (byte > 0x0d && byte < 0x20);
export const isControlByte = (byte: number) => byte < 0x09 || (byte > 0x0d && byte < 0x20);
export function startsAsMediaPlaylist(head: Buffer): boolean {
const text = head.toString("latin1");
@@ -0,0 +1,87 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import {
isCopiedOutsideFileType,
isUploadedOutsideFileType,
leftOutWarning,
} from "./outsideFile.js";
const dir = mkdtempSync(join(tmpdir(), "hf-outside-file-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
function write(name: string, body: string | Buffer): string {
const path = join(dir, name);
writeFileSync(path, body);
return path;
}
/** Three packets of `size` bytes with the sync byte at `offset` in each. */
function transportStream(size: number, offset: number): Buffer {
const bytes = Buffer.alloc(size * 3);
for (let at = offset; at < bytes.length; at += size) bytes[at] = 0x47;
return bytes;
}
describe("outside file types", () => {
it("copies scripts into a local render but not into an upload", () => {
for (const name of ["lib.js", "lib.mjs"]) {
const path = write(name, "export const key = 1;");
expect(isCopiedOutsideFileType(path)).toBe(true);
expect(isUploadedOutsideFileType(path)).toBe(false);
}
});
it("copies stylesheets, fonts and media into both", () => {
for (const name of ["theme.css", "brand.woff2", "clip.mp4", "logo.png"]) {
const path = write(name, "x");
expect(isCopiedOutsideFileType(path)).toBe(true);
expect(isUploadedOutsideFileType(path)).toBe(true);
}
});
it("treats a .ts or .mts file as video only when it is a transport stream", () => {
const source = [write("config.ts", "export const key: number = 1;"), write("config.mts", "G")];
for (const path of source) {
expect(isCopiedOutsideFileType(path)).toBe(false);
expect(isUploadedOutsideFileType(path)).toBe(false);
}
const video = [
write("clip.ts", transportStream(188, 0)),
write("cam.mts", transportStream(192, 4)),
write("disc.m2ts", "x"),
];
for (const path of video) {
expect(isCopiedOutsideFileType(path)).toBe(true);
expect(isUploadedOutsideFileType(path)).toBe(true);
}
});
it("does not count text with a sync byte at every packet as a transport stream", () => {
const text = Array.from({ length: 4 }, () => `G${"/".repeat(187)}`).join("");
for (const name of ["spaced.ts", "spaced.mts"]) {
expect(isUploadedOutsideFileType(write(name, text))).toBe(false);
}
});
it.skipIf(process.platform === "win32")(
"refuses a named pipe without waiting for a writer",
() => {
const fifo = join(dir, "pipe.ts");
execFileSync("mkfifo", [fifo]);
expect(isCopiedOutsideFileType(fifo)).toBe(false);
expect(isUploadedOutsideFileType(fifo)).toBe(false);
},
);
it("names the copied types for the upload or render that left a file out", () => {
expect(leftOutWarning("index.html", "../lib.js", "file_type")).toBe(
"index.html references ../lib.js, which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.",
);
expect(leftOutWarning("index.html", "../a.pdf", "file_type", "render")).toBe(
"index.html references ../a.pdf, which is outside the project and is not an image, video, audio, font, CSS or JS file; it was left out of the render. Move it into the project to include it.",
);
});
});
@@ -1,9 +1,11 @@
import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, extname, isAbsolute, join, relative, sep } from "node:path";
import { MIME_TYPES } from "./mime.js";
import {
AUDIO_EXTS,
CONVERTIBLE_IMAGE_EXTS,
isControlByte,
MEDIA_INPUT_ARGS,
startsAsMediaPlaylist,
VIDEO_EXTS,
@@ -14,21 +16,76 @@ export { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, MEDIA_INPUT_ARGS, startsAsMediaPlay
const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/;
/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types,
* and the media cloud render re-encodes or converts. */
export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet<string> = new Set([
...Object.entries(MIME_TYPES)
.filter(([, type]) => COPIED_TYPE.test(type))
.map(([extension]) => extension),
const extensionsOf = (type: RegExp) =>
Object.entries(MIME_TYPES)
.filter(([, mime]) => type.test(mime))
.map(([extension]) => extension);
/** What a local render brings in from outside the project: the preview server's image, video, audio, font, CSS and
* JS types, and the media cloud render re-encodes or converts. */
const OUTSIDE_FILE_EXTENSIONS: ReadonlySet<string> = new Set([
...extensionsOf(COPIED_TYPE),
...VIDEO_EXTS,
...AUDIO_EXTS,
...CONVERTIBLE_IMAGE_EXTS,
]);
const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase());
const SCRIPT_EXTENSIONS: ReadonlySet<string> = new Set(extensionsOf(/^text\/javascript;/));
/** What an upload copies from outside the project: the same types without scripts. */
export const UPLOADED_OUTSIDE_FILE_EXTENSIONS: ReadonlySet<string> = new Set(
[...OUTSIDE_FILE_EXTENSIONS].filter((extension) => !SCRIPT_EXTENSIONS.has(extension)),
);
const TYPESCRIPT_OR_TRANSPORT_STREAM = new Set([".ts", ".mts"]);
const TS_SYNC_BYTE = 0x47;
const TS_PACKETS = [
{ size: 188, offset: 0 },
{ size: 192, offset: 4 },
];
function readRegularFileHead(path: string, length: number): Buffer | null {
let fd: number;
try {
// A FIFO would wait for a writer on a blocking open; fstat refuses it first.
fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
} catch {
return null;
}
try {
if (!fstatSync(fd).isFile()) return null;
const head = Buffer.alloc(length);
return head.subarray(0, readSync(fd, head, 0, length, 0));
} catch {
return null;
} finally {
closeSync(fd);
}
}
function startsAsTransportStream(path: string): boolean {
const head = readRegularFileHead(path, 4 * 192);
if (!head?.some(isControlByte)) return false;
return TS_PACKETS.some(({ size, offset }) => {
const starts = [];
for (let at = offset; at < head.length; at += size) starts.push(at);
return starts.length >= 2 && starts.every((at) => head[at] === TS_SYNC_BYTE);
});
}
function isCopiedType(absPath: string, extensions: ReadonlySet<string>): boolean {
const names = [absPath, realFilePath(absPath)].map((path) => extname(path).toLowerCase());
if (!names.every((extension) => extensions.has(extension))) return false;
const ambiguous = names.some((extension) => TYPESCRIPT_OR_TRANSPORT_STREAM.has(extension));
return !ambiguous || startsAsTransportStream(absPath);
}
export const isCopiedOutsideFileType = (absPath: string): boolean =>
hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath));
isCopiedType(absPath, OUTSIDE_FILE_EXTENSIONS);
export const isUploadedOutsideFileType = (absPath: string): boolean =>
isCopiedType(absPath, UPLOADED_OUTSIDE_FILE_EXTENSIONS);
const ENV_FILE = /^\.env([._-].*|rc)?$/;
@@ -92,11 +149,15 @@ export const linksOutside = (projectDir: string, filePath: string): boolean =>
/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */
export type LeftOutReason = "blocklisted" | "received" | "file_type" | "private";
const WHY: Record<LeftOutReason, string> = {
const WHY: Record<Exclude<LeftOutReason, "file_type">, string> = {
private: "which holds secrets or tool data",
blocklisted: "which is outside the project in a private location",
received: "which leads outside the project and came in with an imported project",
file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file",
};
const COPIED_TYPES = {
upload: "an image, video, audio, font or CSS file",
render: "an image, video, audio, font, CSS or JS file",
};
export function leftOutWarning(
@@ -105,6 +166,10 @@ export function leftOutWarning(
reason: LeftOutReason,
from: "upload" | "render" = "upload",
): string {
const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the ${from}.`;
const why =
reason === "file_type"
? `which is outside the project and is not ${COPIED_TYPES[from]}`
: WHY[reason];
const warning = `${file} references ${reference}, ${why}; it was left out of the ${from}.`;
return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning;
}