mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(cli): uploads leave out outside scripts; .ts counts as video only when it is one (#7818)
* fix(cli): uploads leave out outside scripts; .ts counts as video only when it is one * chore(studio-server): raise unpacked size budget for the upload file-type rule * fix(studio-server): transport stream check never waits on a pipe and needs binary bytes
This commit is contained in:
@@ -138,9 +138,10 @@ Local images outside the project referenced by an `<img>` or `<source>` element'
|
||||
`srcset` are included too. Their size or pixel-density descriptors are preserved,
|
||||
and your source files stay unchanged.
|
||||
|
||||
Only images, video, audio, fonts, CSS and JS files are copied from outside the
|
||||
project. Any other outside file, such as a Lottie `.json` or a `.pdf`, is left
|
||||
out with a warning naming it; move it into the project folder to include it.
|
||||
Only images, video, audio, fonts and CSS files are copied from outside the
|
||||
project. Any other outside file, such as a script, a Lottie `.json` or a `.pdf`,
|
||||
is left out with a warning naming it; move it into the project folder to include
|
||||
it.
|
||||
|
||||
Before sharing source, remove secrets and confirm that every included media file
|
||||
can be shared. Tell a reviewer whether you need feedback on story, visuals,
|
||||
|
||||
@@ -2720,9 +2720,11 @@ authenticating, uploading, or starting anything. If you are over, add only
|
||||
paths you have verified are unneeded to `.hyperframesignore`.
|
||||
|
||||
A file outside the project folder that a composition points at is copied into
|
||||
the zip only if it is an image, video, audio, font, CSS or JS file, judged by
|
||||
its extension. Any other outside file (a Lottie `.json`, a `.pdf`, a file with
|
||||
no extension), one in a private location (env files, Git data, hidden, system
|
||||
the zip only if it is an image, video, audio, font or CSS file, judged by its
|
||||
extension. A `.ts` or `.mts` file counts as video only when its first bytes are
|
||||
an MPEG transport stream, so TypeScript source is not copied. Any other outside
|
||||
file (a script, a Lottie `.json`, a `.pdf`, a file with no extension), one in a
|
||||
private location (env files, Git data, hidden, system
|
||||
and app-data folders), and one reached from a file that came in with an
|
||||
imported project are left out, with a warning naming each. To include such a
|
||||
file, move it into the project folder. Under `--json`, the result lists those
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export {
|
||||
isCopiedOutsideFileType,
|
||||
OUTSIDE_FILE_EXTENSIONS,
|
||||
isUploadedOutsideFileType,
|
||||
UPLOADED_OUTSIDE_FILE_EXTENSIONS,
|
||||
} from "@hyperframes/studio-server/outside-file";
|
||||
|
||||
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { LeftOutReference } from "./leftOutReference.js";
|
||||
import { OUTSIDE_FILE_EXTENSIONS } from "./outsideFileType.js";
|
||||
import { UPLOADED_OUTSIDE_FILE_EXTENSIONS } from "./outsideFileType.js";
|
||||
import { stagePublishFiles } from "./publishProject.js";
|
||||
|
||||
const dirs: string[] = [];
|
||||
@@ -71,7 +71,7 @@ describe("a file outside the project", () => {
|
||||
});
|
||||
expect(ext).toEqual([]);
|
||||
const tail =
|
||||
"which is outside the project and is not an image, video, audio, font, CSS or JS file; it was left out of the upload. Move it into the project to include it.";
|
||||
"which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.";
|
||||
expect(warnings.sort()).toEqual(
|
||||
[
|
||||
`index.html references ../shared/scene.json, ${tail}`,
|
||||
@@ -98,6 +98,57 @@ describe("a file outside the project", () => {
|
||||
expect(index).toContain(`src="../shared/scene.json"`);
|
||||
});
|
||||
|
||||
it("is left out when it is a script or TypeScript source, with a warning", () => {
|
||||
const { ext, warnings, leftOut } = stage({
|
||||
"shared/lib.js": "export const key = 1;",
|
||||
"shared/lib.mjs": "export const key = 1;",
|
||||
"shared/config.ts": "export const key: number = 1;",
|
||||
"shared/config.mts": "export const key: number = 1;",
|
||||
"p/index.html": [
|
||||
`<script src="../shared/lib.js"></script>`,
|
||||
`<script type="module" src="../shared/lib.mjs"></script>`,
|
||||
`<video src="../shared/config.ts"></video><link href="../shared/config.mts">`,
|
||||
].join(""),
|
||||
});
|
||||
expect(ext).toEqual([]);
|
||||
expect(warnings.sort()).toEqual(
|
||||
["lib.js", "lib.mjs", "config.ts", "config.mts"]
|
||||
.map(
|
||||
(name) =>
|
||||
`index.html references ../shared/${name}, which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.`,
|
||||
)
|
||||
.sort(),
|
||||
);
|
||||
expect(leftOut.map((one) => one.reason)).toEqual(Array(4).fill("file_type"));
|
||||
});
|
||||
|
||||
it("is copied when a .ts or .mts file is a transport stream video", () => {
|
||||
const packets = (size: number, offset: number) => {
|
||||
const bytes = Buffer.alloc(size * 3);
|
||||
for (let at = offset; at < bytes.length; at += size) bytes[at] = 0x47;
|
||||
return bytes.toString("latin1");
|
||||
};
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-file-type-"));
|
||||
dirs.push(root);
|
||||
mkdirSync(join(root, "shared"));
|
||||
mkdirSync(join(root, "p"));
|
||||
writeFileSync(join(root, "shared", "clip.ts"), Buffer.from(packets(188, 0), "latin1"));
|
||||
writeFileSync(join(root, "shared", "cam.mts"), Buffer.from(packets(192, 4), "latin1"));
|
||||
writeFileSync(
|
||||
join(root, "p", "index.html"),
|
||||
`<video src="../shared/clip.ts"></video><video src="../shared/cam.mts"></video>`,
|
||||
);
|
||||
const warnings: string[] = [];
|
||||
const staged = stagePublishFiles(join(root, "p"), Infinity, (warning) =>
|
||||
warnings.push(warning),
|
||||
);
|
||||
expect([...staged.keys()].filter((key) => key.startsWith("_ext/")).sort()).toEqual([
|
||||
"_ext/shared/cam.mts",
|
||||
"_ext/shared/clip.ts",
|
||||
]);
|
||||
expect(warnings).toEqual([]);
|
||||
});
|
||||
|
||||
it("follows the same rule for CSS url() targets and @import", () => {
|
||||
const { ext, warnings, leftOut } = stage({
|
||||
"shared/theme.css": "body {}",
|
||||
@@ -162,18 +213,19 @@ describe("a file outside the project", () => {
|
||||
});
|
||||
|
||||
describe("the outside file types", () => {
|
||||
it("are images, video, audio, fonts, CSS and JS, and nothing else", () => {
|
||||
it("are images, video, audio, fonts and CSS, and nothing else", () => {
|
||||
for (const extension of [".png", ".avif", ".svg", ".bmp", ".tif", ".mp4", ".mov", ".webm"]) {
|
||||
expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
|
||||
expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
|
||||
}
|
||||
for (const extension of [".avi", ".ogv", ".mp3", ".wav", ".aiff"]) {
|
||||
expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
|
||||
expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
|
||||
}
|
||||
for (const extension of [".woff", ".woff2", ".ttf", ".otf", ".css", ".js", ".mjs"]) {
|
||||
expect(OUTSIDE_FILE_EXTENSIONS).toContain(extension);
|
||||
for (const extension of [".woff", ".woff2", ".ttf", ".otf", ".css"]) {
|
||||
expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).toContain(extension);
|
||||
}
|
||||
for (const extension of [".json", ".html", ".txt", ".md", ".cube", ".pdf", ".key", ""]) {
|
||||
expect(OUTSIDE_FILE_EXTENSIONS).not.toContain(extension);
|
||||
const others = [".js", ".mjs", ".json", ".html", ".txt", ".md", ".cube", ".pdf", ".key", ""];
|
||||
for (const extension of others) {
|
||||
expect(UPLOADED_OUTSIDE_FILE_EXTENSIONS).not.toContain(extension);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -36,7 +36,7 @@ import {
|
||||
type LeftOutReason,
|
||||
type OnLeftOut,
|
||||
} from "./leftOutReference.js";
|
||||
import { isCopiedOutsideFileType } from "./outsideFileType.js";
|
||||
import { isUploadedOutsideFileType } from "./outsideFileType.js";
|
||||
|
||||
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
|
||||
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
|
||||
@@ -509,7 +509,7 @@ function archiveTargetOf(
|
||||
leaveOut(ctx, options.entryPath, rawPath, absPath);
|
||||
return undefined;
|
||||
}
|
||||
if (!isCopiedOutsideFileType(absPath)) {
|
||||
if (!isUploadedOutsideFileType(absPath)) {
|
||||
leaveOut(ctx, options.entryPath, rawPath, absPath, "file_type");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ export const MEDIA_INPUT_ARGS: readonly string[] = [
|
||||
|
||||
const PLAYLIST_START = /^(?:\xEF\xBB\xBF)?\s*(?:#EXTM3U|ffconcat)/;
|
||||
const MARKUP_START = /^(?:\xEF\xBB\xBF)?\s*</;
|
||||
const isControlByte = (byte: number) => byte < 0x09 || (byte > 0x0d && byte < 0x20);
|
||||
export const isControlByte = (byte: number) => byte < 0x09 || (byte > 0x0d && byte < 0x20);
|
||||
|
||||
export function startsAsMediaPlaylist(head: Buffer): boolean {
|
||||
const text = head.toString("latin1");
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
isCopiedOutsideFileType,
|
||||
isUploadedOutsideFileType,
|
||||
leftOutWarning,
|
||||
} from "./outsideFile.js";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-outside-file-"));
|
||||
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
function write(name: string, body: string | Buffer): string {
|
||||
const path = join(dir, name);
|
||||
writeFileSync(path, body);
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Three packets of `size` bytes with the sync byte at `offset` in each. */
|
||||
function transportStream(size: number, offset: number): Buffer {
|
||||
const bytes = Buffer.alloc(size * 3);
|
||||
for (let at = offset; at < bytes.length; at += size) bytes[at] = 0x47;
|
||||
return bytes;
|
||||
}
|
||||
|
||||
describe("outside file types", () => {
|
||||
it("copies scripts into a local render but not into an upload", () => {
|
||||
for (const name of ["lib.js", "lib.mjs"]) {
|
||||
const path = write(name, "export const key = 1;");
|
||||
expect(isCopiedOutsideFileType(path)).toBe(true);
|
||||
expect(isUploadedOutsideFileType(path)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("copies stylesheets, fonts and media into both", () => {
|
||||
for (const name of ["theme.css", "brand.woff2", "clip.mp4", "logo.png"]) {
|
||||
const path = write(name, "x");
|
||||
expect(isCopiedOutsideFileType(path)).toBe(true);
|
||||
expect(isUploadedOutsideFileType(path)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("treats a .ts or .mts file as video only when it is a transport stream", () => {
|
||||
const source = [write("config.ts", "export const key: number = 1;"), write("config.mts", "G")];
|
||||
for (const path of source) {
|
||||
expect(isCopiedOutsideFileType(path)).toBe(false);
|
||||
expect(isUploadedOutsideFileType(path)).toBe(false);
|
||||
}
|
||||
const video = [
|
||||
write("clip.ts", transportStream(188, 0)),
|
||||
write("cam.mts", transportStream(192, 4)),
|
||||
write("disc.m2ts", "x"),
|
||||
];
|
||||
for (const path of video) {
|
||||
expect(isCopiedOutsideFileType(path)).toBe(true);
|
||||
expect(isUploadedOutsideFileType(path)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not count text with a sync byte at every packet as a transport stream", () => {
|
||||
const text = Array.from({ length: 4 }, () => `G${"/".repeat(187)}`).join("");
|
||||
for (const name of ["spaced.ts", "spaced.mts"]) {
|
||||
expect(isUploadedOutsideFileType(write(name, text))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"refuses a named pipe without waiting for a writer",
|
||||
() => {
|
||||
const fifo = join(dir, "pipe.ts");
|
||||
execFileSync("mkfifo", [fifo]);
|
||||
expect(isCopiedOutsideFileType(fifo)).toBe(false);
|
||||
expect(isUploadedOutsideFileType(fifo)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("names the copied types for the upload or render that left a file out", () => {
|
||||
expect(leftOutWarning("index.html", "../lib.js", "file_type")).toBe(
|
||||
"index.html references ../lib.js, which is outside the project and is not an image, video, audio, font or CSS file; it was left out of the upload. Move it into the project to include it.",
|
||||
);
|
||||
expect(leftOutWarning("index.html", "../a.pdf", "file_type", "render")).toBe(
|
||||
"index.html references ../a.pdf, which is outside the project and is not an image, video, audio, font, CSS or JS file; it was left out of the render. Move it into the project to include it.",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,11 @@
|
||||
import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, extname, isAbsolute, join, relative, sep } from "node:path";
|
||||
import { MIME_TYPES } from "./mime.js";
|
||||
import {
|
||||
AUDIO_EXTS,
|
||||
CONVERTIBLE_IMAGE_EXTS,
|
||||
isControlByte,
|
||||
MEDIA_INPUT_ARGS,
|
||||
startsAsMediaPlaylist,
|
||||
VIDEO_EXTS,
|
||||
@@ -14,21 +16,76 @@ export { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, MEDIA_INPUT_ARGS, startsAsMediaPlay
|
||||
|
||||
const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/;
|
||||
|
||||
/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types,
|
||||
* and the media cloud render re-encodes or converts. */
|
||||
export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet<string> = new Set([
|
||||
...Object.entries(MIME_TYPES)
|
||||
.filter(([, type]) => COPIED_TYPE.test(type))
|
||||
.map(([extension]) => extension),
|
||||
const extensionsOf = (type: RegExp) =>
|
||||
Object.entries(MIME_TYPES)
|
||||
.filter(([, mime]) => type.test(mime))
|
||||
.map(([extension]) => extension);
|
||||
|
||||
/** What a local render brings in from outside the project: the preview server's image, video, audio, font, CSS and
|
||||
* JS types, and the media cloud render re-encodes or converts. */
|
||||
const OUTSIDE_FILE_EXTENSIONS: ReadonlySet<string> = new Set([
|
||||
...extensionsOf(COPIED_TYPE),
|
||||
...VIDEO_EXTS,
|
||||
...AUDIO_EXTS,
|
||||
...CONVERTIBLE_IMAGE_EXTS,
|
||||
]);
|
||||
|
||||
const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase());
|
||||
const SCRIPT_EXTENSIONS: ReadonlySet<string> = new Set(extensionsOf(/^text\/javascript;/));
|
||||
|
||||
/** What an upload copies from outside the project: the same types without scripts. */
|
||||
export const UPLOADED_OUTSIDE_FILE_EXTENSIONS: ReadonlySet<string> = new Set(
|
||||
[...OUTSIDE_FILE_EXTENSIONS].filter((extension) => !SCRIPT_EXTENSIONS.has(extension)),
|
||||
);
|
||||
|
||||
const TYPESCRIPT_OR_TRANSPORT_STREAM = new Set([".ts", ".mts"]);
|
||||
const TS_SYNC_BYTE = 0x47;
|
||||
|
||||
const TS_PACKETS = [
|
||||
{ size: 188, offset: 0 },
|
||||
{ size: 192, offset: 4 },
|
||||
];
|
||||
|
||||
function readRegularFileHead(path: string, length: number): Buffer | null {
|
||||
let fd: number;
|
||||
try {
|
||||
// A FIFO would wait for a writer on a blocking open; fstat refuses it first.
|
||||
fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
if (!fstatSync(fd).isFile()) return null;
|
||||
const head = Buffer.alloc(length);
|
||||
return head.subarray(0, readSync(fd, head, 0, length, 0));
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
function startsAsTransportStream(path: string): boolean {
|
||||
const head = readRegularFileHead(path, 4 * 192);
|
||||
if (!head?.some(isControlByte)) return false;
|
||||
return TS_PACKETS.some(({ size, offset }) => {
|
||||
const starts = [];
|
||||
for (let at = offset; at < head.length; at += size) starts.push(at);
|
||||
return starts.length >= 2 && starts.every((at) => head[at] === TS_SYNC_BYTE);
|
||||
});
|
||||
}
|
||||
|
||||
function isCopiedType(absPath: string, extensions: ReadonlySet<string>): boolean {
|
||||
const names = [absPath, realFilePath(absPath)].map((path) => extname(path).toLowerCase());
|
||||
if (!names.every((extension) => extensions.has(extension))) return false;
|
||||
const ambiguous = names.some((extension) => TYPESCRIPT_OR_TRANSPORT_STREAM.has(extension));
|
||||
return !ambiguous || startsAsTransportStream(absPath);
|
||||
}
|
||||
|
||||
export const isCopiedOutsideFileType = (absPath: string): boolean =>
|
||||
hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath));
|
||||
isCopiedType(absPath, OUTSIDE_FILE_EXTENSIONS);
|
||||
|
||||
export const isUploadedOutsideFileType = (absPath: string): boolean =>
|
||||
isCopiedType(absPath, UPLOADED_OUTSIDE_FILE_EXTENSIONS);
|
||||
|
||||
const ENV_FILE = /^\.env([._-].*|rc)?$/;
|
||||
|
||||
@@ -92,11 +149,15 @@ export const linksOutside = (projectDir: string, filePath: string): boolean =>
|
||||
/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */
|
||||
export type LeftOutReason = "blocklisted" | "received" | "file_type" | "private";
|
||||
|
||||
const WHY: Record<LeftOutReason, string> = {
|
||||
const WHY: Record<Exclude<LeftOutReason, "file_type">, string> = {
|
||||
private: "which holds secrets or tool data",
|
||||
blocklisted: "which is outside the project in a private location",
|
||||
received: "which leads outside the project and came in with an imported project",
|
||||
file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file",
|
||||
};
|
||||
|
||||
const COPIED_TYPES = {
|
||||
upload: "an image, video, audio, font or CSS file",
|
||||
render: "an image, video, audio, font, CSS or JS file",
|
||||
};
|
||||
|
||||
export function leftOutWarning(
|
||||
@@ -105,6 +166,10 @@ export function leftOutWarning(
|
||||
reason: LeftOutReason,
|
||||
from: "upload" | "render" = "upload",
|
||||
): string {
|
||||
const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the ${from}.`;
|
||||
const why =
|
||||
reason === "file_type"
|
||||
? `which is outside the project and is not ${COPIED_TYPES[from]}`
|
||||
: WHY[reason];
|
||||
const warning = `${file} references ${reference}, ${why}; it was left out of the ${from}.`;
|
||||
return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user