mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix: docs image upload publishes only regular files, and CI rejects new symlinks (#7776)
scripts/upload-docs-images.sh synced docs/images/ with the AWS CLI's default of following symbolic links, so a link committed under docs/images/ would publish whatever it pointed to on the machine running the upload. The script now stops before any AWS call when docs/images is not a real folder in the checkout or when anything under it is a link or not a regular file, and the sync passes --no-follow-symlinks as well. A new check, scripts/check-symlinks.mjs, fails CI when a tracked symbolic link is not one of the two the repo already means to have. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
870b1fa4dc
commit
9320c44e13
@@ -113,6 +113,8 @@ jobs:
|
||||
env:
|
||||
BASE_SHA: ${{ steps.base.outputs.sha }}
|
||||
run: sh scripts/check-large-files.sh --range "$BASE_SHA" HEAD
|
||||
- name: Reject symbolic links outside the allowlist
|
||||
run: node scripts/check-symlinks.mjs
|
||||
- name: Find a release-only diff
|
||||
id: release
|
||||
uses: ./.github/actions/release-only-diff
|
||||
|
||||
@@ -207,4 +207,6 @@ Include:
|
||||
|
||||
External contributors should attach the preview MP4. A maintainer will publish
|
||||
the final Catalog media. HeyGen contributors can use
|
||||
`scripts/upload-docs-images.sh` after preview review.
|
||||
`scripts/upload-docs-images.sh` after preview review, from a checkout that holds
|
||||
only the regenerated media, not from a contributor's branch. The script publishes
|
||||
regular files only and stops if anything under `docs/images/` is a symbolic link.
|
||||
|
||||
+2
-1
@@ -32,6 +32,7 @@
|
||||
"check:skill-mirror": "node scripts/check-skill-mirror.mjs",
|
||||
"lint:fix": "oxlint --fix .",
|
||||
"check:tracked-artifacts": "node scripts/check-tracked-artifacts.mjs",
|
||||
"check:symlinks": "node scripts/check-symlinks.mjs",
|
||||
"check:docs-snippet-motion": "node scripts/check-docs-snippet-motion.mjs",
|
||||
"check:docs-catalog": "node scripts/check-docs-catalog.mjs",
|
||||
"generate:catalog": "bun scripts/generate-catalog.ts",
|
||||
@@ -54,7 +55,7 @@
|
||||
"player:perf": "bun run --filter @hyperframes/player perf",
|
||||
"format:check": "oxfmt --check .",
|
||||
"knip": "knip",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-cli-lazy-deps.test.mjs scripts/file-url-paths.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/clawhub-unlist-retired.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/ci-cache-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/registry-cdn-pins.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/check-core-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/lefthook-order.test.mjs scripts/inline-gsap.test.mjs scripts/ci/flake-gate.test.mjs scripts/ci/motion-blur-nightly-report.test.mjs scripts/ci/base-file.test.mjs scripts/ci/release-only-diff.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-theme-fallbacks.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs registry/components/echo-smear/echo-smear.test.mjs registry/examples/path-traced-3d/source/path-traced-3d.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-symlinks.test.mjs scripts/upload-docs-images.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-cli-lazy-deps.test.mjs scripts/file-url-paths.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/clawhub-unlist-retired.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/ci-cache-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/registry-cdn-pins.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/check-core-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/lefthook-order.test.mjs scripts/inline-gsap.test.mjs scripts/ci/flake-gate.test.mjs scripts/ci/motion-blur-nightly-report.test.mjs scripts/ci/base-file.test.mjs scripts/ci/release-only-diff.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-theme-fallbacks.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs registry/components/echo-smear/echo-smear.test.mjs registry/examples/path-traced-3d/source/path-traced-3d.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
|
||||
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
|
||||
"test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'",
|
||||
"generate:previews": "tsx scripts/generate-template-previews.ts",
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
/**
|
||||
* The symbolic links the repo means to track. Any other link fails the check: tooling that publishes or reads
|
||||
* the tree (the docs image upload, registry and skills packaging) would follow it to whatever it names on the
|
||||
* machine running it.
|
||||
*/
|
||||
export const ALLOWED_SYMLINKS = new Set([
|
||||
"packages/cli/src/audio/scripts",
|
||||
"packages/producer/tests/render-symlinked-assets/src/shared",
|
||||
]);
|
||||
|
||||
/** Paths of `git ls-files -s -z` entries with the symlink mode (120000). */
|
||||
export function symlinkPaths(stagedListing) {
|
||||
return stagedListing
|
||||
.split("\0")
|
||||
.filter(Boolean)
|
||||
.filter((entry) => entry.startsWith("120000 "))
|
||||
.map((entry) => entry.slice(entry.indexOf("\t") + 1));
|
||||
}
|
||||
|
||||
export function findUnexpectedSymlinks(paths, allowed = ALLOWED_SYMLINKS) {
|
||||
return paths.filter((path) => !allowed.has(path)).sort();
|
||||
}
|
||||
|
||||
export function readStagedListing(cwd = process.cwd()) {
|
||||
const result = spawnSync("git", ["ls-files", "-s", "-z"], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
if (result.error) throw result.error;
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr.trim() || `git ls-files exited with status ${result.status}`);
|
||||
}
|
||||
return result.stdout;
|
||||
}
|
||||
|
||||
export function checkSymlinks(cwd = process.cwd()) {
|
||||
return findUnexpectedSymlinks(symlinkPaths(readStagedListing(cwd)));
|
||||
}
|
||||
|
||||
function main() {
|
||||
const unexpected = checkSymlinks();
|
||||
if (unexpected.length === 0) {
|
||||
console.log("Symlink check passed.");
|
||||
return;
|
||||
}
|
||||
console.error(
|
||||
"Symbolic links are tracked that are not on the allowlist in scripts/check-symlinks.mjs:",
|
||||
);
|
||||
for (const path of unexpected) console.error(`- ${path}`);
|
||||
console.error("Commit the file itself instead of a link to it.");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
const entryPath = process.argv[1] ? resolve(process.argv[1]) : null;
|
||||
if (entryPath === fileURLToPath(import.meta.url)) main();
|
||||
@@ -0,0 +1,72 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, it } from "node:test";
|
||||
import { checkSymlinks, findUnexpectedSymlinks, symlinkPaths } from "./check-symlinks.mjs";
|
||||
|
||||
/** A throwaway repo with `files` (contents) and `links` (path to target) staged, and no global git config. */
|
||||
function repoWith({ files = {}, links = {} }) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-symlinks-repo-"));
|
||||
const env = { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" };
|
||||
spawnSync("git", ["init", "-q"], { cwd: dir, env });
|
||||
for (const [name, contents] of Object.entries(files)) {
|
||||
mkdirSync(join(dir, name, ".."), { recursive: true });
|
||||
writeFileSync(join(dir, name), contents);
|
||||
}
|
||||
for (const [name, target] of Object.entries(links)) {
|
||||
mkdirSync(join(dir, name, ".."), { recursive: true });
|
||||
symlinkSync(target, join(dir, name));
|
||||
}
|
||||
spawnSync("git", ["add", "-A"], { cwd: dir, env });
|
||||
return dir;
|
||||
}
|
||||
|
||||
describe("symlink check", () => {
|
||||
it("reads only symlink entries from the staged listing", () => {
|
||||
const listing =
|
||||
"100644 aaaa 0\tdocs/a.png\0" +
|
||||
"120000 bbbb 0\tdocs/images/env\0" +
|
||||
"100755 cccc 0\tscripts/x.sh\0" +
|
||||
"120000 dddd 0\tpath with space/link\0";
|
||||
assert.deepEqual(symlinkPaths(listing), ["docs/images/env", "path with space/link"]);
|
||||
});
|
||||
|
||||
it("passes the links the repo already means to track", () => {
|
||||
assert.deepEqual(
|
||||
findUnexpectedSymlinks([
|
||||
"packages/cli/src/audio/scripts",
|
||||
"packages/producer/tests/render-symlinked-assets/src/shared",
|
||||
]),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
it("fails a new link under docs/images, sorted", () => {
|
||||
assert.deepEqual(
|
||||
findUnexpectedSymlinks([
|
||||
"packages/cli/src/audio/scripts",
|
||||
"registry/x/link",
|
||||
"docs/images/env",
|
||||
]),
|
||||
["docs/images/env", "registry/x/link"],
|
||||
);
|
||||
});
|
||||
|
||||
it("finds a link a pull request stages, and not regular files", () => {
|
||||
const dir = repoWith({
|
||||
files: { "docs/images/a.png": "png" },
|
||||
links: { "docs/images/env": "../../.env", "docs/images/up": "../.." },
|
||||
});
|
||||
try {
|
||||
assert.deepEqual(checkSymlinks(dir), ["docs/images/env", "docs/images/up"]);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("passes this repository as it stands", () => {
|
||||
assert.deepEqual(checkSymlinks(join(import.meta.dirname, "..")), []);
|
||||
});
|
||||
});
|
||||
@@ -12,6 +12,10 @@
|
||||
# not reach a reader, see the invalidation step below.
|
||||
# Contributors without AWS access: open a PR with the HTML/MDX changes and a
|
||||
# maintainer will run the generators + this upload before merging.
|
||||
#
|
||||
# Run it from a checkout you trust: only regular files inside docs/images/ are
|
||||
# published, and the script refuses to start if anything there is a symbolic
|
||||
# link, since a link would publish whatever it points to on this machine.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -25,8 +29,24 @@ if [ ! -d "$SRC" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# docs/images itself (or docs) must be the real folder in this checkout, not a link to elsewhere.
|
||||
if [ "$(cd "$SRC" && pwd -P)" != "$(cd "$REPO_ROOT" && pwd -P)/docs/images" ]; then
|
||||
echo "docs/images/ is not a folder inside this checkout; refusing to upload." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Only regular files and folders go up. find does not follow links, so a link
|
||||
# (or anything else that is not a plain file) is listed rather than read.
|
||||
UNSAFE="$(find "$SRC" ! -type f ! -type d)"
|
||||
if [ -n "$UNSAFE" ]; then
|
||||
echo "Refusing to upload: these are not regular files (symbolic links are never published):" >&2
|
||||
printf '%s\n' "$UNSAFE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Uploading $SRC → $DEST (profile: $PROFILE)"
|
||||
aws --profile "$PROFILE" s3 sync "$SRC" "$DEST" \
|
||||
--no-follow-symlinks \
|
||||
--cache-control "public, max-age=31536000, immutable" \
|
||||
--metadata-directive REPLACE
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, it } from "node:test";
|
||||
|
||||
const SCRIPT = join(import.meta.dirname, "upload-docs-images.sh");
|
||||
|
||||
/**
|
||||
* Runs a copy of the upload script in a throwaway checkout, with `aws` on PATH replaced by a stub that records
|
||||
* its arguments, so nothing can reach S3. `arrange` lays out the checkout before the run.
|
||||
*/
|
||||
function upload(arrange) {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-upload-docs-"));
|
||||
try {
|
||||
const checkout = join(root, "checkout");
|
||||
mkdirSync(join(checkout, "scripts"), { recursive: true });
|
||||
copyFileSync(SCRIPT, join(checkout, "scripts", "upload-docs-images.sh"));
|
||||
const bin = join(root, "bin");
|
||||
const log = join(root, "aws.log");
|
||||
mkdirSync(bin);
|
||||
writeFileSync(join(bin, "aws"), `#!/bin/sh\necho "$*" >> "${log}"\n`);
|
||||
chmodSync(join(bin, "aws"), 0o755);
|
||||
const secret = join(root, "outside");
|
||||
mkdirSync(secret);
|
||||
writeFileSync(join(secret, ".env"), "SECRET=1");
|
||||
arrange({ checkout, images: join(checkout, "docs", "images"), secret });
|
||||
const run = spawnSync("bash", [join(checkout, "scripts", "upload-docs-images.sh")], {
|
||||
env: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
|
||||
encoding: "utf8",
|
||||
});
|
||||
return { ...run, aws: existsSync(log) ? readFileSync(log, "utf8") : "" };
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const withImages = (images) => {
|
||||
mkdirSync(join(images, "catalog"), { recursive: true });
|
||||
writeFileSync(join(images, "catalog", "a.png"), "png");
|
||||
writeFileSync(join(images, "b.mp4"), "mp4");
|
||||
};
|
||||
|
||||
describe("upload-docs-images.sh", () => {
|
||||
it("uploads ordinary files without following links", () => {
|
||||
const run = upload(({ images }) => withImages(images));
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
assert.match(run.aws, /s3 sync .*--no-follow-symlinks/);
|
||||
assert.match(run.aws, /cloudfront create-invalidation/);
|
||||
});
|
||||
|
||||
it("refuses before any upload when a file under docs/images is a link", () => {
|
||||
const run = upload(({ images, secret }) => {
|
||||
withImages(images);
|
||||
symlinkSync(join(secret, ".env"), join(images, "catalog", "env.png"));
|
||||
});
|
||||
assert.notEqual(run.status, 0);
|
||||
assert.match(run.stderr, /catalog\/env\.png/);
|
||||
assert.equal(run.aws, "");
|
||||
});
|
||||
|
||||
it("refuses before any upload when a folder under docs/images is a link", () => {
|
||||
const run = upload(({ images, secret }) => {
|
||||
withImages(images);
|
||||
symlinkSync(secret, join(images, "up"));
|
||||
});
|
||||
assert.notEqual(run.status, 0);
|
||||
assert.equal(run.aws, "");
|
||||
});
|
||||
|
||||
it("refuses before any upload when docs/images itself is a link", () => {
|
||||
const run = upload(({ checkout, secret }) => {
|
||||
mkdirSync(join(checkout, "docs"));
|
||||
symlinkSync(secret, join(checkout, "docs", "images"));
|
||||
});
|
||||
assert.notEqual(run.status, 0);
|
||||
assert.match(run.stderr, /not a folder inside this checkout/);
|
||||
assert.equal(run.aws, "");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user