From e58fb3249b9d74dbe9997d4020e50680ba36a061 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 17 Nov 2026 04:10:20 +0000 Subject: [PATCH] fix(cli): render --docker applies the received-project rule on the host (#8016) * fix(cli): render --docker applies the received-files rule decided on the host The container mounts only the project, so it cannot see a received-files record in a folder above it. The host now makes that check for single and batch Docker renders and passes --refuse-outside-files into the container when it refuses outside files. * fix(cli): a Docker batch decides the received-files rule once A batch render now applies the received-files rule once and hands the result to every row, so it warns once instead of once per row. A test now drives renderDocker itself and checks the docker run arguments it starts. --- docs/packages/cli.mdx | 9 ++ .../cli/src/commands/render.docker.test.ts | 71 +++++++++++ packages/cli/src/commands/render.test.ts | 71 +++++++++++ packages/cli/src/commands/render.ts | 115 ++++++++++++------ .../cli/src/commands/render/execute.test.ts | 63 +++++++++- packages/cli/src/commands/render/execute.ts | 10 ++ packages/cli/src/commands/render/plan.ts | 3 + packages/cli/src/utils/dockerRunArgs.test.ts | 13 ++ packages/cli/src/utils/dockerRunArgs.ts | 2 + 9 files changed, 315 insertions(+), 42 deletions(-) create mode 100644 packages/cli/src/commands/render.docker.test.ts diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 5a4cd8a121..ee7c779eeb 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -1968,6 +1968,7 @@ How it uses the machine, and when it gives up: | `--strict` / `--strict-all` | off | Fail on lint errors, or on errors and warnings | | `--strict-variables` | off | Fail rather than warn on an undeclared or mistyped variable key | | `--include-outside-files` | off | Also run `.js` and `.mjs` scripts the project references from outside its folder. See the note below. | +| `--refuse-outside-files` | off | Leave out every file the project references from outside its folder. See the note below. | | `--batch-concurrency` / `--batch-fail-fast` | `1` / off | Batch rows at once, and whether the first failure stops the rest | | `--debug` | off | Keep intermediates and write diagnostics under the Producer `.debug` directory | | `--quiet` | off | Less output | @@ -1980,6 +1981,14 @@ browser and can read and send anything the render can reach. Move a script you trust into the project, or pass `--include-outside-files` to run it where it is. A composition that needs the left-out script stops with a script-load error. +A received-files record lists the files in a folder that came from someone +else. When a record in the project, or in a folder above it, lists anything but +media, fonts and images, the render leaves out every file from outside the +project folder and warns with each one's name. A `--docker` render makes this check on your +machine, where the folders above the project are visible, and passes +`--refuse-outside-files` into the container, so it leaves out the same files a +local render would. + Renders started side by side starve each other of CPU and GPU until frames stop landing, so a second local or `--docker` render from the same user account waits for the first and says which render it is waiting for (pid and project). The diff --git a/packages/cli/src/commands/render.docker.test.ts b/packages/cli/src/commands/render.docker.test.ts new file mode 100644 index 0000000000..6ccbfef946 --- /dev/null +++ b/packages/cli/src/commands/render.docker.test.ts @@ -0,0 +1,71 @@ +import { EventEmitter } from "node:events"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const dockerRuns = vi.hoisted(() => [] as string[][]); + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + execFileSync: vi.fn(() => Buffer.from("")), + spawn: vi.fn((command: string, args: string[]) => { + if (command === "docker") dockerRuns.push(args); + const child = Object.assign(new EventEmitter(), { stderr: new EventEmitter() }); + queueMicrotask(() => child.emit("close", 0)); + return child; + }), + }; +}); + +const { renderDocker } = await import("./render.js"); + +describe("renderDocker", () => { + const roots: string[] = []; + afterEach(() => { + vi.restoreAllMocks(); + dockerRuns.length = 0; + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); + }); + + async function dockerRunFor(receivedFiles: string[] | undefined): Promise { + const root = mkdtempSync(join(tmpdir(), "hf-render-docker-")); + roots.push(root); + const projectDir = join(root, "scene"); + mkdirSync(projectDir); + mkdirSync(join(root, "output")); + writeFileSync(join(root, "output", "photo.png"), "secret"); + writeFileSync(join(projectDir, "index.html"), ''); + if (receivedFiles) { + mkdirSync(join(root, ".hyperframes")); + writeFileSync( + join(root, ".hyperframes", "received-files.json"), + JSON.stringify({ files: receivedFiles }), + ); + } + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + await renderDocker(projectDir, join(root, "output", "film.mp4"), { + fps: { num: 30, den: 1 }, + quality: "standard", + format: "mp4", + gpu: false, + hdrMode: "auto", + quiet: true, + embedded: true, + }); + const run = dockerRuns.find((args) => args[0] === "run"); + if (!run) throw new Error("renderDocker started no docker run"); + return run; + } + + it("passes the refusal into the container for a project under a record listing received code", async () => { + expect(await dockerRunFor(["scene/index.html"])).toContain("--refuse-outside-files"); + }); + + it("starts the container without the refusal for a project with no record", async () => { + expect(await dockerRunFor(undefined)).not.toContain("--refuse-outside-files"); + }); +}); diff --git a/packages/cli/src/commands/render.test.ts b/packages/cli/src/commands/render.test.ts index fa9ae29ada..7141ae7f07 100644 --- a/packages/cli/src/commands/render.test.ts +++ b/packages/cli/src/commands/render.test.ts @@ -652,6 +652,21 @@ describe("renderLocal browser GPU config", () => { } }); + it("brings in no outside file when the render is told to refuse them", async () => { + await renderLocal(PROJECT_DIR, join(PROJECT_DIR, "out.mp4"), { + fps: { num: 30, den: 1 }, + quality: "standard", + format: "mp4", + gpu: false, + browserGpuMode: "software", + hdrMode: "auto", + quiet: true, + throwOnError: true, + refuseOutsideFiles: true, + }); + expect(producerState.createdJobs.at(-1)?.allowOutsideProjectAssets).toBe(false); + }); + it("cleans orphaned browser trees before starting a local render", async () => { orphanCleanupState.killed = 1; @@ -2478,6 +2493,62 @@ describe("render command batch options", () => { }, 60_000); }); +describe("dockerRenderOptions", () => { + const { dockerRenderOptions } = renderModule; + const options = { + fps: { num: 30, den: 1 }, + quality: "standard", + format: "mp4", + gpu: false, + hdrMode: "auto", + quiet: true, + } as const; + + function receivedProject(files: string[] | undefined): { root: string; projectDir: string } { + const root = mkdtempSync(join(tmpdir(), "hf-docker-received-")); + const projectDir = join(root, "scene"); + mkdirSync(projectDir); + writeFileSync(join(projectDir, "index.html"), ''); + writeFileSync( + join(projectDir, "photo.png"), + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + ); + if (files) { + mkdirSync(join(root, ".hyperframes")); + writeFileSync(join(root, ".hyperframes", "received-files.json"), JSON.stringify({ files })); + } + return { root, projectDir }; + } + + it("refuses outside files when a record in a folder above the project lists its entry", () => { + const { root, projectDir } = receivedProject(["scene/index.html"]); + mkdirSync(join(root, "output")); + writeFileSync(join(root, "output", "photo.png"), "secret"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + expect(dockerRenderOptions(projectDir, options).refuseOutsideFiles).toBe(true); + expect(warn).toHaveBeenCalledWith( + expect.stringMatching(/came from someone else.*photo\.png/), + ); + } finally { + warn.mockRestore(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + { name: "no received-files record", files: undefined }, + { name: "a record listing only an image", files: ["scene/photo.png"] }, + ])("leaves outside files in for $name", ({ files }) => { + const { root, projectDir } = receivedProject(files); + try { + expect(dockerRenderOptions(projectDir, options).refuseOutsideFiles).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + describe("normalizeStageCode", () => { const { normalizeStageCode } = renderModule; diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index ace35967ae..56109fd443 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -121,7 +121,11 @@ import { isRouterOff, } from "./render/deRouterBreaker.js"; import { isDevMode } from "../utils/env.js"; -import { buildDockerRunArgs, resolveDockerPlatform } from "../utils/dockerRunArgs.js"; +import { + buildDockerRunArgs, + resolveDockerPlatform, + type DockerRenderOptions, +} from "../utils/dockerRunArgs.js"; import { createStderrTail, DockerRenderExitError } from "../utils/dockerStderrTail.js"; import type { BrowserInstallFacts } from "../browser/installFacts.js"; import { normalizeErrorMessage } from "../utils/errorMessage.js"; @@ -355,6 +359,14 @@ export default defineCommand({ "with a warning; images, video, audio, fonts and CSS outside the folder are always included.", default: false, }, + "refuse-outside-files": { + type: "boolean", + description: + "Leave out every file the project references from outside its folder. A --docker render passes it " + + "into the container when the received-files rule refuses those files on the host.", + default: false, + hidden: true, + }, "strict-all": { type: "boolean", description: "Fail render on lint errors AND warnings", @@ -594,6 +606,11 @@ export interface RenderOptions { bestEffort?: boolean; /** --include-outside-files: run scripts the project references from outside its folder. */ includeOutsideScripts?: boolean; + /** + * True brings in no file from outside the project folder (--refuse-outside-files). False means + * the caller already applied the received-files rule and it allows them. Absent, the render applies it. + */ + refuseOutsideFiles?: boolean; browserPath?: string; variables?: Record; entryFile?: string; @@ -864,10 +881,59 @@ async function registerRenderOutputOrWarn(projectDir: string, outputPath: string ); } -// Inherited minor finding (CRAP 37.1, cyclomatic 11). This PR only added -// `pageNavigationTimeoutMs` to the options forwarded to `buildDockerRunArgs`. +function outsideAssetsAllowedOrWarn(projectDir: string, entryFile: string | undefined): boolean { + return outsideAssetsAllowed(projectDir, entryFile, (line) => console.warn(c.warn(` ${line}`))); +} + +/** + * The render options passed into the container. The received-files rule is + * decided here on the host, because the container mounts only the project and + * cannot see a record in a folder above it. A container CLI without + * --refuse-outside-files rejects it as an unknown flag and does not render. + */ +export function dockerRenderOptions( + projectDir: string, + options: RenderOptions, +): DockerRenderOptions { + return { + fps: options.fps, + quality: options.quality, + format: options.format, + gifLoop: options.gifLoop, + hlsSegmentSeconds: options.hlsSegmentSeconds, + workers: options.workers, + gpu: options.gpu, + browserGpu: options.browserGpuMode === "hardware", + hdrMode: options.hdrMode, + crf: options.crf, + vp9CpuUsed: options.vp9CpuUsed, + videoBitrate: options.videoBitrate, + keyframeIntervalSeconds: options.keyframeIntervalSeconds, + range: options.range, + proresProfile: options.proresProfile, + maxBitrate: options.maxBitrate, + bufferSize: options.bufferSize, + videoFrameFormat: options.videoFrameFormat, + chromaSubsampling: options.chromaSubsampling, + pageMotionBlur: options.pageMotionBlur, + quiet: options.quiet, + variables: options.variables, + entryFile: options.entryFile, + outputResolution: options.outputResolutionRaw ?? options.outputResolution, + pageSideCompositing: options.pageSideCompositing, + debug: options.debug, + bestEffort: options.bestEffort, + experimentalFastCapture: options.experimentalFastCapture, + pageNavigationTimeoutMs: options.pageNavigationTimeoutMs, + protocolTimeoutMs: options.protocolTimeout, + playerReadyTimeoutMs: options.playerReadyTimeout, + refuseOutsideFiles: + options.refuseOutsideFiles ?? !outsideAssetsAllowedOrWarn(projectDir, options.entryFile), + }; +} + // fallow-ignore-next-line complexity -async function renderDocker( +export async function renderDocker( projectDir: string, outputPath: string, options: RenderOptions, @@ -904,39 +970,7 @@ async function renderDocker( outputFilename, platform, hostStdoutIsTty: process.stdout.isTTY === true, - options: { - fps: options.fps, - quality: options.quality, - format: options.format, - gifLoop: options.gifLoop, - hlsSegmentSeconds: options.hlsSegmentSeconds, - workers: options.workers, - gpu: options.gpu, - browserGpu: options.browserGpuMode === "hardware", - hdrMode: options.hdrMode, - crf: options.crf, - vp9CpuUsed: options.vp9CpuUsed, - videoBitrate: options.videoBitrate, - keyframeIntervalSeconds: options.keyframeIntervalSeconds, - range: options.range, - proresProfile: options.proresProfile, - maxBitrate: options.maxBitrate, - bufferSize: options.bufferSize, - videoFrameFormat: options.videoFrameFormat, - chromaSubsampling: options.chromaSubsampling, - pageMotionBlur: options.pageMotionBlur, - quiet: options.quiet, - variables: options.variables, - entryFile: options.entryFile, - outputResolution: options.outputResolutionRaw ?? options.outputResolution, - pageSideCompositing: options.pageSideCompositing, - debug: options.debug, - bestEffort: options.bestEffort, - experimentalFastCapture: options.experimentalFastCapture, - pageNavigationTimeoutMs: options.pageNavigationTimeoutMs, - protocolTimeoutMs: options.protocolTimeout, - playerReadyTimeoutMs: options.playerReadyTimeout, - }, + options: dockerRenderOptions(projectDir, options), }); if (!options.quiet) { @@ -1161,9 +1195,10 @@ async function executeLocalRender( process.env.HYPERFRAMES_GSAP_VENDOR_DIR = gsapVendorDir; } const producer = await loadProducer(); - const allowOutsideProjectAssets = outsideAssetsAllowed(projectDir, options.entryFile, (line) => - console.warn(c.warn(` ${line}`)), - ); + const allowOutsideProjectAssets = + options.refuseOutsideFiles === undefined + ? outsideAssetsAllowedOrWarn(projectDir, options.entryFile) + : !options.refuseOutsideFiles; const includeOutsideScripts = options.includeOutsideScripts === true; const deParallelRouterActive = options.manageDeParallelRouterBreaker === true diff --git a/packages/cli/src/commands/render/execute.test.ts b/packages/cli/src/commands/render/execute.test.ts index a7d158874a..f40a2e18af 100644 --- a/packages/cli/src/commands/render/execute.test.ts +++ b/packages/cli/src/commands/render/execute.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -12,7 +12,7 @@ import { } from "./execute.js"; import * as manager from "../../browser/manager.js"; import * as system from "../../telemetry/system.js"; -import { wantsDesktopHint, type SingleRenderResult } from "../render.js"; +import { wantsDesktopHint, type RenderOptions, type SingleRenderResult } from "../render.js"; describe("renderOptionsFromPlan", () => { let projectDir: string; @@ -57,6 +57,14 @@ describe("renderOptionsFromPlan", () => { expect(on.browserPath).toBe("/usr/bin/chrome"); expect(on.variables).toEqual({ title: "x" }); }); + + it("carries --refuse-outside-files from the plan into the render options", () => { + const plan = createRenderPlan({ dir: projectDir }); + expect(renderOptionsFromPlan(plan, undefined, undefined).refuseOutsideFiles).toBeUndefined(); + const refused = createRenderPlan({ dir: projectDir, "refuse-outside-files": true }); + expect(renderOptionsFromPlan(refused, undefined, undefined).refuseOutsideFiles).toBe(true); + expect(batchRowRenderOptions(refused, undefined).refuseOutsideFiles).toBe(true); + }); }); describe("the desktop-app line after a render", () => { @@ -114,6 +122,57 @@ describe("a render on a nearly full disk", () => { ); }); +describe("a Docker batch render of a received project", () => { + it("decides the received-files rule once and refuses outside files on every row", async () => { + const root = mkdtempSync(join(tmpdir(), "hf-render-batch-received-")); + const projectDir = join(root, "scene"); + mkdirSync(projectDir); + mkdirSync(join(root, "output")); + writeFileSync(join(root, "output", "photo.png"), "secret"); + writeFileSync( + join(projectDir, "index.html"), + '
', + ); + mkdirSync(join(root, ".hyperframes")); + writeFileSync( + join(root, ".hyperframes", "received-files.json"), + JSON.stringify({ files: ["scene/index.html"] }), + ); + const rowsPath = join(root, "rows.json"); + writeFileSync(rowsPath, "[{}, {}, {}]"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation(() => {}); + const renderDocker = vi.fn(async (_dir: string, _out: string, _options: RenderOptions) => ({ + renderTimeMs: 1, + })); + try { + const plan = createRenderPlan({ + dir: projectDir, + docker: true, + quiet: true, + batch: rowsPath, + output: join(root, "renders", "{index}.mp4"), + }); + await executeRenderPlan(plan, { + renderDocker, + renderLocal: vi.fn(), + checkResolution: async () => undefined, + }); + expect(renderDocker).toHaveBeenCalledTimes(3); + expect(renderDocker.mock.calls.map(([, , options]) => options.refuseOutsideFiles)).toEqual([ + true, + true, + true, + ]); + const warnings = warn.mock.calls.flat().filter((line) => /came from someone else/.test(line)); + expect(warnings).toHaveLength(1); + } finally { + vi.restoreAllMocks(); + rmSync(root, { recursive: true, force: true }); + } + }); +}); + describe("the machine result line after a single render", () => { let projectDir: string; const originalIsTTY = process.stdout.isTTY; diff --git a/packages/cli/src/commands/render/execute.ts b/packages/cli/src/commands/render/execute.ts index e61a727343..f4809ff9b2 100644 --- a/packages/cli/src/commands/render/execute.ts +++ b/packages/cli/src/commands/render/execute.ts @@ -26,6 +26,7 @@ import { applyRenderEnvironment, renderOutputDirectory, type RenderPlan } from " import type { RenderOptions, SingleRenderResult } from "../render.js"; import type { RenderCancellationScope } from "../../utils/renderCancellation.js"; import { runRenderSetupWorker } from "../../utils/cancellableProcess.js"; +import { outsideAssetsAllowed } from "../../utils/receivedRender.js"; type RenderExecutor = ( projectDir: string, @@ -207,6 +208,7 @@ export function renderOptionsFromPlan( keepSegments: plan.keepSegments, bestEffort: plan.bestEffort, includeOutsideScripts: plan.includeOutsideScripts, + refuseOutsideFiles: plan.refuseOutsideFiles ? true : undefined, variables, entryFile: plan.entryFile, outputResolution: plan.outputResolution, @@ -427,6 +429,7 @@ export function batchRowRenderOptions( keepSegments: plan.keepSegments, bestEffort: plan.bestEffort, includeOutsideScripts: plan.includeOutsideScripts, + refuseOutsideFiles: plan.refuseOutsideFiles ? true : undefined, exitAfterComplete: false, throwOnError: true, skipFeedback: true, @@ -435,6 +438,12 @@ export function batchRowRenderOptions( }; } +function outsideAssetsAllowedOnceForEveryRow(plan: RenderPlan): boolean { + return outsideAssetsAllowed(plan.project.dir, plan.entryFile, (line) => + console.warn(c.warn(` ${line}`)), + ); +} + async function executeBatchRender( plan: RenderPlan, browserPath: string | undefined, @@ -445,6 +454,7 @@ async function executeBatchRender( ): Promise { const batchQuiet = plan.quiet || plan.batchJson; const renderOptionsBase = batchRowRenderOptions(plan, browserPath); + renderOptionsBase.refuseOutsideFiles ??= !outsideAssetsAllowedOnceForEveryRow(plan); const manifest = await batchModule.runBatchRender({ prepared: preparedBatch, concurrency: plan.batchConcurrency, diff --git a/packages/cli/src/commands/render/plan.ts b/packages/cli/src/commands/render/plan.ts index 751f583914..6b512f7642 100644 --- a/packages/cli/src/commands/render/plan.ts +++ b/packages/cli/src/commands/render/plan.ts @@ -105,6 +105,7 @@ export interface RenderCommandArgs { strict?: boolean; "strict-all"?: boolean; "include-outside-files"?: boolean; + "refuse-outside-files"?: boolean; "lint-verbose"?: boolean; "max-concurrent-renders"?: string; variables?: string; @@ -170,6 +171,7 @@ export interface RenderPlan { keepSegments: boolean; bestEffort: boolean; includeOutsideScripts: boolean; + refuseOutsideFiles: boolean; waitForRenderSlot: boolean; batchJson: boolean; effectiveQuiet: boolean; @@ -674,6 +676,7 @@ export function createRenderPlan(args: RenderCommandArgs, now = new Date()): Ren keepSegments: args["keep-segments"] ?? false, bestEffort: args["best-effort"] ?? true, includeOutsideScripts: args["include-outside-files"] ?? false, + refuseOutsideFiles: args["refuse-outside-files"] ?? false, waitForRenderSlot: args.wait ?? true, batchJson, effectiveQuiet: quiet || batchJson, diff --git a/packages/cli/src/utils/dockerRunArgs.test.ts b/packages/cli/src/utils/dockerRunArgs.test.ts index 26dff07987..394a708bb4 100644 --- a/packages/cli/src/utils/dockerRunArgs.test.ts +++ b/packages/cli/src/utils/dockerRunArgs.test.ts @@ -530,6 +530,19 @@ describe("buildDockerRunArgs", () => { expect(args).not.toContain("--no-page-side-compositing"); }); + it("passes the host's refusal of outside files into the container", () => { + const refused = buildDockerRunArgs({ + ...FIXED_INPUT, + options: { ...BASE, refuseOutsideFiles: true }, + }); + expect(refused.indexOf("--refuse-outside-files")).toBeGreaterThan( + refused.indexOf(FIXED_INPUT.imageTag), + ); + expect( + buildDockerRunArgs({ ...FIXED_INPUT, options: { ...BASE, refuseOutsideFiles: false } }), + ).toEqual(buildDockerRunArgs({ ...FIXED_INPUT, options: BASE })); + }); + // Regression for #1193: an arm64 host (Apple Silicon) was being pinned to // linux/amd64, which forced qemu emulation of chrome-headless-shell and // produced either navigation timeouts or chrome SEGVs. Each host arch must diff --git a/packages/cli/src/utils/dockerRunArgs.ts b/packages/cli/src/utils/dockerRunArgs.ts index d01f401e36..a8f74f6f7f 100644 --- a/packages/cli/src/utils/dockerRunArgs.ts +++ b/packages/cli/src/utils/dockerRunArgs.ts @@ -81,6 +81,7 @@ export interface DockerRenderOptions { protocolTimeoutMs?: number; /** Player readiness timeout in milliseconds. */ playerReadyTimeoutMs?: number; + refuseOutsideFiles?: boolean; } /** @@ -208,5 +209,6 @@ export function buildDockerRunArgs(input: DockerRunArgsInput): string[] { ...(options.playerReadyTimeoutMs != null ? ["--player-ready-timeout", String(options.playerReadyTimeoutMs)] : []), + ...(options.refuseOutsideFiles ? ["--refuse-outside-files"] : []), ]; }