mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 17:27:09 +00:00
fix(registry): verify staged asset content before publishing manifests (#3832)
* fix(registry): verify staged asset content before publishing manifests * test(registry): run asset staging regression in script CI * refactor(registry): separate verified manifest publication
This commit is contained in:
James Russo
+1
-1
@@ -48,7 +48,7 @@
|
||||
"player:perf": "bun run --filter @hyperframes/player perf",
|
||||
"format:check": "oxfmt --check .",
|
||||
"knip": "knip",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/",
|
||||
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
|
||||
"test:skills": "node --test 'skills/**/*.test.mjs'",
|
||||
"generate:previews": "tsx scripts/generate-template-previews.ts",
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const repoRoot = resolve(import.meta.dirname, "..");
|
||||
|
||||
async function fixture() {
|
||||
const root = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "hf-host-assets-")));
|
||||
const item = join(root, "registry/blocks/test-item");
|
||||
fs.mkdirSync(item, { recursive: true });
|
||||
fs.mkdirSync(join(root, "scripts"));
|
||||
for (const name of ["host-registry-assets.ts", "entrypoint.ts"]) {
|
||||
fs.copyFileSync(join(repoRoot, "scripts", name), join(root, "scripts", name));
|
||||
}
|
||||
for (const name of ["packages", "node_modules"]) {
|
||||
fs.symlinkSync(join(repoRoot, name), join(root, name), "junction");
|
||||
}
|
||||
const source = join(item, "asset.png");
|
||||
const manifestPath = join(item, "registry-item.json");
|
||||
const bytes = Buffer.from("original asset bytes");
|
||||
fs.writeFileSync(source, bytes);
|
||||
const manifest = JSON.stringify({ name: "test-item", files: [{ path: "asset.png" }] });
|
||||
fs.writeFileSync(manifestPath, manifest);
|
||||
const { main } = await import(pathToFileURL(join(root, "scripts/host-registry-assets.ts")).href);
|
||||
return { root, source, manifestPath, manifest, bytes, main };
|
||||
}
|
||||
|
||||
test("staged bytes match their key, dry-run is unchanged, and no-upload preserves sources", async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
await f.main(["test-item", "--dry-run"]);
|
||||
assert.equal(fs.readFileSync(f.manifestPath, "utf8"), f.manifest);
|
||||
assert.equal(fs.existsSync(join(f.root, ".registry-assets-staging")), false);
|
||||
await f.main(["test-item", "--no-upload"]);
|
||||
const key = `${createHash("sha256").update(f.bytes).digest("hex").slice(0, 16)}.png`;
|
||||
assert.deepEqual(fs.readFileSync(join(f.root, ".registry-assets-staging", key)), f.bytes);
|
||||
assert.deepEqual(fs.readFileSync(f.source), f.bytes);
|
||||
const manifest = JSON.parse(fs.readFileSync(f.manifestPath, "utf8"));
|
||||
assert.equal(
|
||||
manifest.files[0].url,
|
||||
`https://static.heygen.ai/hyperframes-oss/registry-assets/${key}`,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(f.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("source replacement between planning and staging aborts before manifest publication", async (t) => {
|
||||
const f = await fixture();
|
||||
const copy = fs.copyFileSync;
|
||||
t.mock.method(fs, "copyFileSync", (...args: Parameters<typeof copy>) => {
|
||||
if (args[0] === f.source) fs.writeFileSync(f.source, "replacement bytes");
|
||||
return copy(...args);
|
||||
});
|
||||
syncBuiltinESMExports();
|
||||
try {
|
||||
await assert.rejects(f.main(["test-item", "--no-upload"]), /Asset changed while staging/);
|
||||
assert.equal(fs.readFileSync(f.manifestPath, "utf8"), f.manifest);
|
||||
assert.equal(fs.readFileSync(f.source, "utf8"), "replacement bytes");
|
||||
assert.deepEqual(fs.readdirSync(join(f.root, ".registry-assets-staging")), []);
|
||||
} finally {
|
||||
t.mock.restoreAll();
|
||||
syncBuiltinESMExports();
|
||||
fs.rmSync(f.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -33,12 +33,15 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
copyFileSync,
|
||||
constants,
|
||||
closeSync,
|
||||
fstatSync,
|
||||
openSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
unlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
@@ -87,6 +90,8 @@ interface Plan {
|
||||
source: string;
|
||||
/** Content-addressed object key, without the bucket prefix. */
|
||||
key: string;
|
||||
/** Full digest of the bytes used to derive the key. */
|
||||
digest: string;
|
||||
/** Public URL the manifest will carry. */
|
||||
url: string;
|
||||
}
|
||||
@@ -124,17 +129,27 @@ function needsHosting(file: FileTarget): boolean {
|
||||
/** Point one file at the CDN, and say which bytes have to get there. */
|
||||
function hostFile(itemName: string, itemDir: string, file: FileTarget): Plan {
|
||||
const source = resolve(itemDir, file.path);
|
||||
if (!existsSync(source) || !statSync(source).isFile()) {
|
||||
throw new Error(`${itemName}: files[] declares "${file.path}", which is not on disk.`);
|
||||
const fd = openSync(source, constants.O_RDONLY | constants.O_NONBLOCK);
|
||||
let digest: string;
|
||||
try {
|
||||
if (!fstatSync(fd).isFile()) {
|
||||
throw new Error(`${itemName}: files[] declares "${file.path}", which is not a file.`);
|
||||
}
|
||||
digest = createHash("sha256").update(readFileSync(fd)).digest("hex");
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
const digest = createHash("sha256").update(readFileSync(source)).digest("hex").slice(0, 16);
|
||||
const key = `${digest}${extname(file.path).toLowerCase()}`;
|
||||
const key = `${digest.slice(0, 16)}${extname(file.path).toLowerCase()}`;
|
||||
file.url = `${CDN_BASE}/${key}`;
|
||||
return { source, key, url: file.url };
|
||||
return { source, key, digest, url: file.url };
|
||||
}
|
||||
|
||||
/** Rewrite one item's manifest, and report which bytes have to reach the CDN. */
|
||||
function planItem(itemDir: string): Plan[] {
|
||||
/** Prepare a manifest update and identify the bytes that must reach the CDN. */
|
||||
function planItem(itemDir: string): {
|
||||
plans: Plan[];
|
||||
manifestPath: string;
|
||||
manifest: RegistryItem;
|
||||
} {
|
||||
const manifestPath = join(itemDir, "registry-item.json");
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as RegistryItem;
|
||||
|
||||
@@ -142,10 +157,13 @@ function planItem(itemDir: string): Plan[] {
|
||||
.filter(needsHosting)
|
||||
.map((file) => hostFile(manifest.name, itemDir, file));
|
||||
|
||||
if (plans.length > 0) {
|
||||
return { plans, manifestPath, manifest };
|
||||
}
|
||||
|
||||
function writeManifests(items: ReturnType<typeof planItem>[]): void {
|
||||
for (const { manifestPath, manifest } of items) {
|
||||
writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf-8");
|
||||
}
|
||||
return plans;
|
||||
}
|
||||
|
||||
/** Collect every planned object into one flat directory, deduplicated by key. */
|
||||
@@ -156,7 +174,13 @@ function stage(plans: Plan[]): number {
|
||||
for (const plan of plans) {
|
||||
if (seen.has(plan.key)) continue;
|
||||
seen.add(plan.key);
|
||||
copyFileSync(plan.source, join(STAGING_DIR, plan.key));
|
||||
const stagedPath = join(STAGING_DIR, plan.key);
|
||||
copyFileSync(plan.source, stagedPath);
|
||||
const digest = createHash("sha256").update(readFileSync(stagedPath)).digest("hex");
|
||||
if (digest !== plan.digest) {
|
||||
unlinkSync(stagedPath);
|
||||
throw new Error(`Asset changed while staging: ${plan.source}. No upload was attempted.`);
|
||||
}
|
||||
}
|
||||
return seen.size;
|
||||
}
|
||||
@@ -186,19 +210,10 @@ function upload(): void {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Report what would move, changing nothing.
|
||||
*
|
||||
* Planning writes the manifest, so a dry run restores each one after asking
|
||||
* it. Cheaper than a second code path that could disagree with the real one
|
||||
* about what counts as hostable.
|
||||
*/
|
||||
/** Report what would move, changing nothing. */
|
||||
function reportDryRun(itemDirs: string[]): void {
|
||||
for (const dir of itemDirs) {
|
||||
const manifestPath = join(dir, "registry-item.json");
|
||||
const before = readFileSync(manifestPath, "utf-8");
|
||||
const plans = planItem(dir);
|
||||
writeFileSync(manifestPath, before, "utf-8");
|
||||
const { plans } = planItem(dir);
|
||||
if (plans.length > 0) {
|
||||
console.log(`${dir.replace(`${repoRoot}/`, "")}: ${plans.length} file(s) would be hosted`);
|
||||
}
|
||||
@@ -228,13 +243,15 @@ export async function main(argv: string[]): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
const plans = itemDirs.flatMap(planItem);
|
||||
const items = itemDirs.map(planItem).filter((item) => item.plans.length > 0);
|
||||
const plans = items.flatMap((item) => item.plans);
|
||||
if (plans.length === 0) {
|
||||
console.log("No unhosted binary assets found — nothing to do.");
|
||||
return;
|
||||
}
|
||||
|
||||
const objects = stage(plans);
|
||||
writeManifests(items);
|
||||
console.log(
|
||||
`Staged ${objects} object(s) from ${plans.length} manifest entr(ies) → ${STAGING_DIR}`,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user