From 9320c44e130ab2fbd073599f576d0f42aa7e346d Mon Sep 17 00:00:00 2001 From: James Russo Date: Sun, 15 Nov 2026 00:47:08 +0000 Subject: [PATCH] fix: docs image upload publishes only regular files, and CI rejects new symlinks (#7776) scripts/upload-docs-images.sh synced docs/images/ with the AWS CLI's default of following symbolic links, so a link committed under docs/images/ would publish whatever it pointed to on the machine running the upload. The script now stops before any AWS call when docs/images is not a real folder in the checkout or when anything under it is a link or not a regular file, and the sync passes --no-follow-symlinks as well. A new check, scripts/check-symlinks.mjs, fails CI when a tracked symbolic link is not one of the two the repo already means to have. Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 + docs/contributing/catalog.mdx | 4 +- package.json | 3 +- scripts/check-symlinks.mjs | 60 +++++++++++++++++++ scripts/check-symlinks.test.mjs | 72 +++++++++++++++++++++++ scripts/upload-docs-images.sh | 20 +++++++ scripts/upload-docs-images.test.mjs | 91 +++++++++++++++++++++++++++++ 7 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 scripts/check-symlinks.mjs create mode 100644 scripts/check-symlinks.test.mjs create mode 100644 scripts/upload-docs-images.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10c1ddf9a6..e63a3829d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,6 +113,8 @@ jobs: env: BASE_SHA: ${{ steps.base.outputs.sha }} run: sh scripts/check-large-files.sh --range "$BASE_SHA" HEAD + - name: Reject symbolic links outside the allowlist + run: node scripts/check-symlinks.mjs - name: Find a release-only diff id: release uses: ./.github/actions/release-only-diff diff --git a/docs/contributing/catalog.mdx b/docs/contributing/catalog.mdx index 77d26a68c2..d828355ae1 100644 --- a/docs/contributing/catalog.mdx +++ b/docs/contributing/catalog.mdx @@ -207,4 +207,6 @@ Include: External contributors should attach the preview MP4. A maintainer will publish the final Catalog media. HeyGen contributors can use -`scripts/upload-docs-images.sh` after preview review. +`scripts/upload-docs-images.sh` after preview review, from a checkout that holds +only the regenerated media, not from a contributor's branch. The script publishes +regular files only and stops if anything under `docs/images/` is a symbolic link. diff --git a/package.json b/package.json index f31d42d708..c3ca297914 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "check:skill-mirror": "node scripts/check-skill-mirror.mjs", "lint:fix": "oxlint --fix .", "check:tracked-artifacts": "node scripts/check-tracked-artifacts.mjs", + "check:symlinks": "node scripts/check-symlinks.mjs", "check:docs-snippet-motion": "node scripts/check-docs-snippet-motion.mjs", "check:docs-catalog": "node scripts/check-docs-catalog.mjs", "generate:catalog": "bun scripts/generate-catalog.ts", @@ -54,7 +55,7 @@ "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", - "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-cli-lazy-deps.test.mjs scripts/file-url-paths.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/clawhub-unlist-retired.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/ci-cache-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/registry-cdn-pins.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/check-core-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/lefthook-order.test.mjs scripts/inline-gsap.test.mjs scripts/ci/flake-gate.test.mjs scripts/ci/motion-blur-nightly-report.test.mjs scripts/ci/base-file.test.mjs scripts/ci/release-only-diff.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-theme-fallbacks.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs registry/components/echo-smear/echo-smear.test.mjs registry/examples/path-traced-3d/source/path-traced-3d.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", + "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-symlinks.test.mjs scripts/upload-docs-images.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-cli-lazy-deps.test.mjs scripts/file-url-paths.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/clawhub-unlist-retired.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/ci-cache-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/registry-cdn-pins.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/check-core-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/lefthook-order.test.mjs scripts/inline-gsap.test.mjs scripts/ci/flake-gate.test.mjs scripts/ci/motion-blur-nightly-report.test.mjs scripts/ci/base-file.test.mjs scripts/ci/release-only-diff.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-theme-fallbacks.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs registry/components/echo-smear/echo-smear.test.mjs registry/examples/path-traced-3d/source/path-traced-3d.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json", "test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'", "generate:previews": "tsx scripts/generate-template-previews.ts", diff --git a/scripts/check-symlinks.mjs b/scripts/check-symlinks.mjs new file mode 100644 index 0000000000..6665daf5b8 --- /dev/null +++ b/scripts/check-symlinks.mjs @@ -0,0 +1,60 @@ +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * The symbolic links the repo means to track. Any other link fails the check: tooling that publishes or reads + * the tree (the docs image upload, registry and skills packaging) would follow it to whatever it names on the + * machine running it. + */ +export const ALLOWED_SYMLINKS = new Set([ + "packages/cli/src/audio/scripts", + "packages/producer/tests/render-symlinked-assets/src/shared", +]); + +/** Paths of `git ls-files -s -z` entries with the symlink mode (120000). */ +export function symlinkPaths(stagedListing) { + return stagedListing + .split("\0") + .filter(Boolean) + .filter((entry) => entry.startsWith("120000 ")) + .map((entry) => entry.slice(entry.indexOf("\t") + 1)); +} + +export function findUnexpectedSymlinks(paths, allowed = ALLOWED_SYMLINKS) { + return paths.filter((path) => !allowed.has(path)).sort(); +} + +export function readStagedListing(cwd = process.cwd()) { + const result = spawnSync("git", ["ls-files", "-s", "-z"], { + cwd, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(result.stderr.trim() || `git ls-files exited with status ${result.status}`); + } + return result.stdout; +} + +export function checkSymlinks(cwd = process.cwd()) { + return findUnexpectedSymlinks(symlinkPaths(readStagedListing(cwd))); +} + +function main() { + const unexpected = checkSymlinks(); + if (unexpected.length === 0) { + console.log("Symlink check passed."); + return; + } + console.error( + "Symbolic links are tracked that are not on the allowlist in scripts/check-symlinks.mjs:", + ); + for (const path of unexpected) console.error(`- ${path}`); + console.error("Commit the file itself instead of a link to it."); + process.exitCode = 1; +} + +const entryPath = process.argv[1] ? resolve(process.argv[1]) : null; +if (entryPath === fileURLToPath(import.meta.url)) main(); diff --git a/scripts/check-symlinks.test.mjs b/scripts/check-symlinks.test.mjs new file mode 100644 index 0000000000..0990820dff --- /dev/null +++ b/scripts/check-symlinks.test.mjs @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { checkSymlinks, findUnexpectedSymlinks, symlinkPaths } from "./check-symlinks.mjs"; + +/** A throwaway repo with `files` (contents) and `links` (path to target) staged, and no global git config. */ +function repoWith({ files = {}, links = {} }) { + const dir = mkdtempSync(join(tmpdir(), "hf-symlinks-repo-")); + const env = { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }; + spawnSync("git", ["init", "-q"], { cwd: dir, env }); + for (const [name, contents] of Object.entries(files)) { + mkdirSync(join(dir, name, ".."), { recursive: true }); + writeFileSync(join(dir, name), contents); + } + for (const [name, target] of Object.entries(links)) { + mkdirSync(join(dir, name, ".."), { recursive: true }); + symlinkSync(target, join(dir, name)); + } + spawnSync("git", ["add", "-A"], { cwd: dir, env }); + return dir; +} + +describe("symlink check", () => { + it("reads only symlink entries from the staged listing", () => { + const listing = + "100644 aaaa 0\tdocs/a.png\0" + + "120000 bbbb 0\tdocs/images/env\0" + + "100755 cccc 0\tscripts/x.sh\0" + + "120000 dddd 0\tpath with space/link\0"; + assert.deepEqual(symlinkPaths(listing), ["docs/images/env", "path with space/link"]); + }); + + it("passes the links the repo already means to track", () => { + assert.deepEqual( + findUnexpectedSymlinks([ + "packages/cli/src/audio/scripts", + "packages/producer/tests/render-symlinked-assets/src/shared", + ]), + [], + ); + }); + + it("fails a new link under docs/images, sorted", () => { + assert.deepEqual( + findUnexpectedSymlinks([ + "packages/cli/src/audio/scripts", + "registry/x/link", + "docs/images/env", + ]), + ["docs/images/env", "registry/x/link"], + ); + }); + + it("finds a link a pull request stages, and not regular files", () => { + const dir = repoWith({ + files: { "docs/images/a.png": "png" }, + links: { "docs/images/env": "../../.env", "docs/images/up": "../.." }, + }); + try { + assert.deepEqual(checkSymlinks(dir), ["docs/images/env", "docs/images/up"]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("passes this repository as it stands", () => { + assert.deepEqual(checkSymlinks(join(import.meta.dirname, "..")), []); + }); +}); diff --git a/scripts/upload-docs-images.sh b/scripts/upload-docs-images.sh index 3d15b429be..f256ed168a 100755 --- a/scripts/upload-docs-images.sh +++ b/scripts/upload-docs-images.sh @@ -12,6 +12,10 @@ # not reach a reader, see the invalidation step below. # Contributors without AWS access: open a PR with the HTML/MDX changes and a # maintainer will run the generators + this upload before merging. +# +# Run it from a checkout you trust: only regular files inside docs/images/ are +# published, and the script refuses to start if anything there is a symbolic +# link, since a link would publish whatever it points to on this machine. set -euo pipefail @@ -25,8 +29,24 @@ if [ ! -d "$SRC" ]; then exit 0 fi +# docs/images itself (or docs) must be the real folder in this checkout, not a link to elsewhere. +if [ "$(cd "$SRC" && pwd -P)" != "$(cd "$REPO_ROOT" && pwd -P)/docs/images" ]; then + echo "docs/images/ is not a folder inside this checkout; refusing to upload." >&2 + exit 1 +fi + +# Only regular files and folders go up. find does not follow links, so a link +# (or anything else that is not a plain file) is listed rather than read. +UNSAFE="$(find "$SRC" ! -type f ! -type d)" +if [ -n "$UNSAFE" ]; then + echo "Refusing to upload: these are not regular files (symbolic links are never published):" >&2 + printf '%s\n' "$UNSAFE" >&2 + exit 1 +fi + echo "Uploading $SRC → $DEST (profile: $PROFILE)" aws --profile "$PROFILE" s3 sync "$SRC" "$DEST" \ + --no-follow-symlinks \ --cache-control "public, max-age=31536000, immutable" \ --metadata-directive REPLACE diff --git a/scripts/upload-docs-images.test.mjs b/scripts/upload-docs-images.test.mjs new file mode 100644 index 0000000000..c1599ed8e6 --- /dev/null +++ b/scripts/upload-docs-images.test.mjs @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "node:test"; + +const SCRIPT = join(import.meta.dirname, "upload-docs-images.sh"); + +/** + * Runs a copy of the upload script in a throwaway checkout, with `aws` on PATH replaced by a stub that records + * its arguments, so nothing can reach S3. `arrange` lays out the checkout before the run. + */ +function upload(arrange) { + const root = mkdtempSync(join(tmpdir(), "hf-upload-docs-")); + try { + const checkout = join(root, "checkout"); + mkdirSync(join(checkout, "scripts"), { recursive: true }); + copyFileSync(SCRIPT, join(checkout, "scripts", "upload-docs-images.sh")); + const bin = join(root, "bin"); + const log = join(root, "aws.log"); + mkdirSync(bin); + writeFileSync(join(bin, "aws"), `#!/bin/sh\necho "$*" >> "${log}"\n`); + chmodSync(join(bin, "aws"), 0o755); + const secret = join(root, "outside"); + mkdirSync(secret); + writeFileSync(join(secret, ".env"), "SECRET=1"); + arrange({ checkout, images: join(checkout, "docs", "images"), secret }); + const run = spawnSync("bash", [join(checkout, "scripts", "upload-docs-images.sh")], { + env: { ...process.env, PATH: `${bin}:${process.env.PATH}` }, + encoding: "utf8", + }); + return { ...run, aws: existsSync(log) ? readFileSync(log, "utf8") : "" }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +const withImages = (images) => { + mkdirSync(join(images, "catalog"), { recursive: true }); + writeFileSync(join(images, "catalog", "a.png"), "png"); + writeFileSync(join(images, "b.mp4"), "mp4"); +}; + +describe("upload-docs-images.sh", () => { + it("uploads ordinary files without following links", () => { + const run = upload(({ images }) => withImages(images)); + assert.equal(run.status, 0, run.stderr); + assert.match(run.aws, /s3 sync .*--no-follow-symlinks/); + assert.match(run.aws, /cloudfront create-invalidation/); + }); + + it("refuses before any upload when a file under docs/images is a link", () => { + const run = upload(({ images, secret }) => { + withImages(images); + symlinkSync(join(secret, ".env"), join(images, "catalog", "env.png")); + }); + assert.notEqual(run.status, 0); + assert.match(run.stderr, /catalog\/env\.png/); + assert.equal(run.aws, ""); + }); + + it("refuses before any upload when a folder under docs/images is a link", () => { + const run = upload(({ images, secret }) => { + withImages(images); + symlinkSync(secret, join(images, "up")); + }); + assert.notEqual(run.status, 0); + assert.equal(run.aws, ""); + }); + + it("refuses before any upload when docs/images itself is a link", () => { + const run = upload(({ checkout, secret }) => { + mkdirSync(join(checkout, "docs")); + symlinkSync(secret, join(checkout, "docs", "images")); + }); + assert.notEqual(run.status, 0); + assert.match(run.stderr, /not a folder inside this checkout/); + assert.equal(run.aws, ""); + }); +});