mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
ci: generate catalog artifacts through a standing publish PR (#4337)
* ci: generate catalog artifacts through a standing publish PR * fix(catalog): reconcile obsolete publication snapshots * refactor(catalog): report failures after staging cleanup * refactor(catalog): separate generation and publication decisions * fix(catalog): unpublish deleted items with their sources * docs(catalog): keep scaffolding guidance concise
This commit is contained in:
Miguel Ángel
@@ -0,0 +1,41 @@
|
||||
name: Publish catalog
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: catalog-publish
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
HYPERFRAMES_NO_TELEMETRY: "1"
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
- run: bash scripts/ci/install-workspace-dependencies.sh
|
||||
- run: bun run --filter '@hyperframes/{parsers,lint,studio-server}' build
|
||||
- run: bun run --cwd packages/core build
|
||||
- run: bun run --cwd packages/core build:hyperframes-runtime
|
||||
- run: bun run check:catalog-drift
|
||||
- name: Refresh the standing publish PR
|
||||
run: node scripts/publish-catalog.mjs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -80,11 +80,12 @@ jobs:
|
||||
token: ""
|
||||
filters: |
|
||||
catalog_index:
|
||||
- "registry/registry.json"
|
||||
- "registry/**"
|
||||
- "registry/catalog-artifact/**"
|
||||
- "skills/media-use/audio/assets/sfx/**"
|
||||
- "scripts/catalog/check-artifact-coverage.ts"
|
||||
docs_catalog:
|
||||
- "skills/media-use/audio/assets/sfx/**"
|
||||
- "docs/**"
|
||||
- "registry/**"
|
||||
- "scripts/*catalog*"
|
||||
@@ -202,8 +203,8 @@ jobs:
|
||||
node-version: 22
|
||||
- uses: ./.github/actions/prepare-ffmpeg-bin
|
||||
- run: bash scripts/ci/install-workspace-dependencies.sh
|
||||
- run: bun scripts/generate-registry-items.ts
|
||||
- run: bun run lint
|
||||
- run: bun run check:docs-catalog
|
||||
|
||||
# Loads every catalog payload a PR changes in headless Chrome. A static scan of the
|
||||
# markup cannot see an asset a script fetches, and the docs host serves only fonts,
|
||||
@@ -226,8 +227,26 @@ jobs:
|
||||
- run: bun run --filter '@hyperframes/{parsers,lint,studio-server}' build
|
||||
- run: bun run --cwd packages/core build
|
||||
- run: bun run --cwd packages/core build:hyperframes-runtime
|
||||
- name: Keep catalog source PRs source-only
|
||||
if: github.event_name == 'pull_request'
|
||||
run: node scripts/check-catalog-source-pr.mjs origin/main
|
||||
- name: Generate and validate catalog
|
||||
env:
|
||||
PUBLISH_PR: ${{ github.head_ref == 'bot/catalog-publish' }}
|
||||
run: |
|
||||
if [ "$PUBLISH_PR" = "true" ]; then
|
||||
bun run check:catalog-drift --committed
|
||||
else
|
||||
bun run check:catalog-drift
|
||||
fi
|
||||
- run: bun run verify:catalog-payloads --changed origin/main
|
||||
- run: bun run check:catalog-drift
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: catalog-index
|
||||
path: |
|
||||
registry/registry.json
|
||||
registry/catalog-artifact/local-vectors.json
|
||||
registry/catalog-artifact/local-vectors.bin
|
||||
|
||||
# `fallow audit` runs dead-code + complexity + duplication analysis scoped to
|
||||
# the changed files. The default `--gate new-only` means existing legacy
|
||||
@@ -411,6 +430,8 @@ jobs:
|
||||
uses: browser-actions/setup-chrome@c785b87e244131f27c9f19c1a33e2ead956ab7ce # v1
|
||||
with:
|
||||
chrome-version: stable
|
||||
- run: bun run generate:catalog
|
||||
if: needs.changes.outputs.docs_catalog == 'true'
|
||||
- run: bun run test:scripts
|
||||
env:
|
||||
HYPERFRAMES_BROWSER_PATH: ${{ steps.catalog-chrome.outputs.chrome-path }}
|
||||
@@ -525,16 +546,17 @@ jobs:
|
||||
# drift. bun runs the TS script directly, no install needed.
|
||||
catalog-index-coverage:
|
||||
name: "Catalog: search index covers the registry"
|
||||
needs: changes
|
||||
needs: [changes, catalog-payloads]
|
||||
if: needs.changes.outputs.catalog_index == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 3
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
# Comparing two name lists needs neither the embedding model nor a
|
||||
# network call, so this stays a seconds-long gate. Regenerating the
|
||||
# vectors does need the model, which is why CI checks rather than fixes.
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: catalog-index
|
||||
path: registry
|
||||
- name: Verify every searchable registry item has a vector
|
||||
run: bun scripts/catalog/check-artifact-coverage.ts
|
||||
|
||||
|
||||
+29
-31
@@ -99,47 +99,45 @@ before merge, listed at the end.
|
||||
2. For components: include a `demo.html`
|
||||
3. Run `npx hyperframes lint` and `npx hyperframes validate` on your HTML
|
||||
4. Test the install flow: `hyperframes add <name> --dir /tmp/test-project`
|
||||
5. Regenerate the manifest: `npx tsx scripts/generate-registry-items.ts`
|
||||
5. Generate and validate the catalog: `bun run check:catalog-drift`
|
||||
|
||||
`registry/registry.json` is generated from the item directories, so edit it with
|
||||
that script rather than by hand. An entry added by hand survives until the next
|
||||
regeneration and then disappears; entries left behind for directories that no
|
||||
longer exist are worse, because `hyperframes add <name>` resolves the name and
|
||||
then fails on missing files.
|
||||
### Generated catalog files
|
||||
|
||||
### What a maintainer finishes for you
|
||||
Catalog additions and edits commit only the item's source files. Do not commit
|
||||
search vectors, generated docs pages, preview payloads, gallery data, or Catalog
|
||||
navigation. CI builds and validates them from your sources.
|
||||
|
||||
Two things need assets an outside contributor is not expected to install. Open
|
||||
the pull request without them and say so; neither blocks review.
|
||||
Deleting or renaming an item has one exception: remove its old entry from
|
||||
`registry/registry.json` in the same PR that deletes the complete item directory.
|
||||
The CLI reads item files directly from main, so leaving that entry would offer an
|
||||
item whose files no longer exist. Keep every other index entry and all metadata
|
||||
unchanged. Do not add the new entry for a rename or an add-plus-delete PR;
|
||||
automation publishes additions. CI rejects missing removals, added entries,
|
||||
metadata edits, and removals whose item directories still exist.
|
||||
|
||||
| Thing | If you have it | If you do not |
|
||||
| ----------------------------------------------- | --------------------------------------------------------- | ----------------------------------------------------------------------- |
|
||||
| The search index (`registry/catalog-artifact/`) | The pre-commit hook rebuilds and stages it | The hook skips, CI names the gap, a maintainer regenerates before merge |
|
||||
| The catalog preview image | Internal contributors run `scripts/upload-docs-images.sh` | Attach the preview MP4 to the PR instead |
|
||||
|
||||
The search index needs a 32 MB embedding model, which is an opt-in for catalog
|
||||
search rather than a build dependency. Until it is regenerated your item is
|
||||
findable by word search and not by meaning, which is the same state as any item
|
||||
published since a user last refreshed their copy.
|
||||
|
||||
### Auto-generated docs
|
||||
|
||||
When you add a new block or component, its documentation page is generated automatically — you don't need to write MDX by hand.
|
||||
|
||||
Run the codegen script after adding items:
|
||||
To generate the complete catalog locally after installing dependencies and
|
||||
building the workspace packages:
|
||||
|
||||
```bash
|
||||
npx tsx scripts/generate-catalog-pages.ts
|
||||
bun run generate:catalog
|
||||
```
|
||||
|
||||
This produces:
|
||||
This command downloads the pinned embedding model when needed. It updates the
|
||||
registry index, vectors, payloads, pages, gallery and navigation in dependency
|
||||
order. Restore generated files before committing, preserving only the required
|
||||
index-entry removals when you delete item directories.
|
||||
|
||||
- `docs/catalog/blocks/<name>.mdx` — per-block detail page
|
||||
- `docs/catalog/components/<name>.mdx` — per-component detail page
|
||||
- `docs/public/catalog-index.json` — flat manifest for the catalog grid page
|
||||
- Updates `docs/docs.json` navigation with the new pages
|
||||
After source changes merge, automation updates one standing publication PR on
|
||||
`bot/catalog-publish`, titled `chore(catalog): publish generated catalog`.
|
||||
A maintainer approves any waiting workflow runs, reviews it, and merges it after
|
||||
checks pass. New items become discoverable after publication; edits to existing
|
||||
CLI item sources take effect on main immediately. Generated docs and search
|
||||
artifacts update when the publication PR merges. Existing URLs remain unchanged.
|
||||
|
||||
The script wipes `docs/catalog/` before regenerating, so deleted items are automatically cleaned up.
|
||||
Example manifests are authored source. To deliberately scaffold them, use
|
||||
`scripts/scaffold-example-manifests.ts`; indexing never rewrites them.
|
||||
Catalog preview images remain a separate workflow. Attach your preview MP4 to
|
||||
the item PR if you cannot upload its hosted image.
|
||||
|
||||
## Pull Requests
|
||||
|
||||
|
||||
@@ -17,19 +17,6 @@ pre-commit:
|
||||
# enforces the same via the "Skills: manifest in sync" job). Churn-free:
|
||||
# the generator rewrites only when a content hash actually changed.
|
||||
run: bun packages/cli/scripts/gen-skills-manifest.ts && git add skills-manifest.json
|
||||
catalog-index:
|
||||
glob: "registry/*/*/registry-item.json"
|
||||
# Rebuild the on-device search vectors when an item's searchable text
|
||||
# changes, then re-stage them so registry/catalog-artifact/ never drifts
|
||||
# from the registry (CI enforces the same via the "Catalog: search index
|
||||
# covers the registry" job). Churn-free: identical inputs re-embed to
|
||||
# identical bytes, so an unrelated registry edit leaves no diff.
|
||||
# Exit 3 is "no embedding model here", which is the normal case for an
|
||||
# outside contributor. Their commit must not be blocked over a 32 MB
|
||||
# opt-in they were never asked to install; CI names the gap instead.
|
||||
run: |
|
||||
bun scripts/catalog/build-local-vectors.ts || { [ $? -eq 3 ] && exit 0; exit 1; }
|
||||
git add registry/registry.json registry/catalog-artifact/local-vectors.json registry/catalog-artifact/local-vectors.bin
|
||||
typecheck:
|
||||
glob: "*.{ts,tsx}"
|
||||
run: cd packages/core && bunx tsc --noEmit && cd ../studio && bunx tsc --noEmit && cd ../.. && bunx tsc --noEmit -p scripts/tsconfig.json
|
||||
|
||||
+3
-2
@@ -34,7 +34,8 @@
|
||||
"check:tracked-artifacts": "node scripts/check-tracked-artifacts.mjs",
|
||||
"check:docs-snippet-motion": "node scripts/check-docs-snippet-motion.mjs",
|
||||
"check:docs-catalog": "node scripts/check-docs-catalog.mjs",
|
||||
"check:catalog-drift": "node scripts/check-registry-set-delta.mjs origin/main && tsx scripts/catalog-drift.ts",
|
||||
"generate:catalog": "bun scripts/generate-catalog.ts",
|
||||
"check:catalog-drift": "tsx scripts/catalog-drift.ts",
|
||||
"verify:catalog-payloads": "tsx scripts/verify-catalog-payloads.ts",
|
||||
"check:workspace-contracts": "node scripts/check-workspace-contracts.mjs",
|
||||
"check:package-cycles": "node scripts/check-package-cycles.mjs",
|
||||
@@ -51,7 +52,7 @@
|
||||
"player:perf": "bun run --filter @hyperframes/player perf",
|
||||
"format:check": "oxfmt --check .",
|
||||
"knip": "knip",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
|
||||
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
|
||||
"test:skills": "node --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'",
|
||||
"generate:previews": "tsx scripts/generate-template-previews.ts",
|
||||
|
||||
+36
-34
@@ -1,9 +1,11 @@
|
||||
// Fails when the committed docs/public/catalog, which the docs build serves as-is, differs from generator output.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, readdirSync, readFileSync, mkdtempSync, rmSync } from "node:fs";
|
||||
// Source PRs validate generated output; the publish PR also checks its committed snapshot.
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { cpSync, mkdirSync, readdirSync, readFileSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, relative, resolve, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { generateCatalog } from "./generate-catalog.ts";
|
||||
import { GENERATED_CATALOG_PATHS } from "./catalog-generated-paths.mjs";
|
||||
import { runAsCommand } from "./entrypoint.ts";
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
@@ -34,40 +36,40 @@ export function treeDifferences(generatedRoot: string, committedRoot: string): s
|
||||
];
|
||||
}
|
||||
|
||||
function generateInto(outRoot: string): number {
|
||||
const run = spawnSync("npx", ["tsx", "scripts/generate-catalog-payloads.ts"], {
|
||||
cwd: repoRoot,
|
||||
env: { ...process.env, CATALOG_PAYLOAD_ROOT: outRoot },
|
||||
stdio: ["ignore", "ignore", "inherit"],
|
||||
});
|
||||
return run.status ?? 1;
|
||||
}
|
||||
|
||||
/** The catalog as `git` holds it, so a file the working tree has but a .gitignore rule keeps out cannot hide drift. */
|
||||
function extractCommittedCatalog(into: string): string {
|
||||
const archive = spawnSync("git", ["archive", "HEAD", "docs/public/catalog"], {
|
||||
cwd: repoRoot,
|
||||
maxBuffer: 2 ** 31 - 1,
|
||||
});
|
||||
if (archive.status !== 0) throw new Error("git archive of docs/public/catalog failed.");
|
||||
mkdirSync(into, { recursive: true });
|
||||
const untar = spawnSync("tar", ["-x", "-C", into], { input: archive.stdout });
|
||||
if (untar.status !== 0) throw new Error("could not unpack the committed catalog.");
|
||||
return join(into, "docs/public/catalog");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
await generateCatalog();
|
||||
for (const script of [
|
||||
"scripts/check-docs-catalog.mjs",
|
||||
"scripts/catalog/check-artifact-coverage.ts",
|
||||
])
|
||||
execFileSync("bun", [script], { cwd: repoRoot, stdio: "inherit" });
|
||||
if (process.argv.includes("--committed")) checkCommittedCatalog();
|
||||
else
|
||||
console.log("Catalog sources generate successfully; derived files belong to the publish PR.");
|
||||
}
|
||||
|
||||
function checkCommittedCatalog(): void {
|
||||
const base = mkdtempSync(join(tmpdir(), "catalog-drift-"));
|
||||
const outRoot = join(base, "generated");
|
||||
try {
|
||||
if (generateInto(outRoot) !== 0) throw new Error("The catalog payload generator failed.");
|
||||
const committedRoot = extractCommittedCatalog(join(base, "committed"));
|
||||
const differences = treeDifferences(outRoot, committedRoot);
|
||||
if (differences.length === 0) return console.log("docs/public/catalog matches the generator.");
|
||||
const listed = differences.slice(0, MAX_LISTED).map((line) => ` ${line}`);
|
||||
throw new Error(
|
||||
`docs/public/catalog differs from the generator in ${differences.length} file(s):\n${listed.join("\n")}\nRegenerate with \`tsx scripts/generate-catalog-payloads.ts\` and commit the result.`,
|
||||
);
|
||||
const committed = join(base, "committed");
|
||||
const generated = join(base, "generated");
|
||||
mkdirSync(committed);
|
||||
mkdirSync(generated);
|
||||
const archive = execFileSync("git", ["archive", "HEAD", ...GENERATED_CATALOG_PATHS], {
|
||||
cwd: repoRoot,
|
||||
maxBuffer: 2 ** 31 - 1,
|
||||
});
|
||||
execFileSync("tar", ["-x", "-C", committed], { input: archive });
|
||||
for (const path of GENERATED_CATALOG_PATHS) {
|
||||
mkdirSync(dirname(join(generated, path)), { recursive: true });
|
||||
cpSync(join(repoRoot, path), join(generated, path), { recursive: true });
|
||||
}
|
||||
const differences = treeDifferences(generated, committed);
|
||||
if (differences.length > 0)
|
||||
throw new Error(
|
||||
`Published catalog differs from generation:\n${differences.slice(0, MAX_LISTED).join("\n")}`,
|
||||
);
|
||||
console.log("Committed catalog matches generation.");
|
||||
} finally {
|
||||
rmSync(base, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
export const GENERATED_CATALOG_PATHS = [
|
||||
"registry/registry.json",
|
||||
"registry/catalog-artifact/local-vectors.json",
|
||||
"registry/catalog-artifact/local-vectors.bin",
|
||||
"docs/catalog/blocks",
|
||||
"docs/catalog/components",
|
||||
"docs/public/catalog",
|
||||
"docs/public/catalog-index.json",
|
||||
"docs/snippets/catalog-gallery-data.mdx",
|
||||
"docs/docs.json",
|
||||
];
|
||||
|
||||
export function isGeneratedCatalogPath(path) {
|
||||
return GENERATED_CATALOG_PATHS.some((root) => path === root || path.startsWith(`${root}/`));
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { catalogChanges, commitBatches } from "./publish-catalog.mjs";
|
||||
import { GENERATED_CATALOG_PATHS } from "./catalog-generated-paths.mjs";
|
||||
import { committedCatalogOutputs } from "./check-catalog-source-pr.mjs";
|
||||
|
||||
test("catalog source edits reject generated commits while a publication alone is accepted", () => {
|
||||
assert.deepEqual(
|
||||
committedCatalogOutputs(["registry/components/new/index.html", "docs/docs.json"]),
|
||||
["docs/docs.json"],
|
||||
);
|
||||
assert.deepEqual(committedCatalogOutputs(["docs/docs.json"]), []);
|
||||
assert.deepEqual(committedCatalogOutputs(["registry/components/new/registry-item.json"]), []);
|
||||
});
|
||||
|
||||
test("publication includes changed, new and deleted generated files and excludes source changes", (t) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-publish-test-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const git = (...args) => execFileSync("git", args, { cwd: root });
|
||||
const write = (path, content) => {
|
||||
mkdirSync(dirname(join(root, path)), { recursive: true });
|
||||
writeFileSync(join(root, path), content);
|
||||
};
|
||||
git("init", "-q");
|
||||
write("registry/registry.json", "old");
|
||||
write("docs/public/catalog/blocks/gone.json", "old");
|
||||
write("registry/blocks/source/index.html", "source");
|
||||
git("add", ".");
|
||||
git(
|
||||
"-c",
|
||||
"user.name=Test",
|
||||
"-c",
|
||||
"user.email=test@example.invalid",
|
||||
"-c",
|
||||
"commit.gpgsign=false",
|
||||
"commit",
|
||||
"-qm",
|
||||
"fixture",
|
||||
);
|
||||
write("registry/registry.json", "new");
|
||||
write("registry/blocks/source/index.html", "changed source");
|
||||
write("docs/public/catalog/blocks/new.json", Buffer.from([0, 255, 4]));
|
||||
rmSync(join(root, "docs/public/catalog/blocks/gone.json"));
|
||||
assert.deepEqual(catalogChanges(root, "HEAD"), [
|
||||
{ path: "docs/public/catalog/blocks/gone.json", kind: "delete" },
|
||||
{ path: "docs/public/catalog/blocks/new.json", kind: "write", contents: "AP8E" },
|
||||
{ path: "registry/registry.json", kind: "write", contents: "bmV3" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("signed API batches preserve deletions and limit each request to 100 changes", () => {
|
||||
const changes = Array.from({ length: 101 }, (_, index) => ({
|
||||
kind: "delete",
|
||||
path: `docs/public/catalog/${index}.json`,
|
||||
}));
|
||||
const batches = commitBatches(changes);
|
||||
assert.deepEqual(
|
||||
batches.map((batch) => batch.deletions.length),
|
||||
[100, 1],
|
||||
);
|
||||
assert.deepEqual(batches[1], {
|
||||
additions: [],
|
||||
deletions: [{ path: "docs/public/catalog/100.json" }],
|
||||
});
|
||||
});
|
||||
|
||||
test("oversized API files fail before any publication", () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
commitBatches([
|
||||
{
|
||||
kind: "write",
|
||||
path: "docs/public/catalog/large.json",
|
||||
contents: "x".repeat(10 * 1024 * 1024),
|
||||
},
|
||||
]),
|
||||
/exceeds API batch budget/,
|
||||
);
|
||||
});
|
||||
|
||||
function publicationFixture(t, changed) {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-publication-api-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
|
||||
git("init", "-q");
|
||||
for (const path of GENERATED_CATALOG_PATHS) {
|
||||
const target = path.includes(".") ? path : `${path}/fixture.json`;
|
||||
mkdirSync(dirname(join(root, target)), { recursive: true });
|
||||
writeFileSync(join(root, target), "published");
|
||||
}
|
||||
git("add", ".");
|
||||
git(
|
||||
"-c",
|
||||
"user.name=Test",
|
||||
"-c",
|
||||
"user.email=test@example.invalid",
|
||||
"-c",
|
||||
"commit.gpgsign=false",
|
||||
"commit",
|
||||
"-qm",
|
||||
"fixture",
|
||||
);
|
||||
const base = git("rev-parse", "HEAD");
|
||||
if (changed) writeFileSync(join(root, "registry/registry.json"), "new publication");
|
||||
const executable = join(root, "gh");
|
||||
writeFileSync(
|
||||
executable,
|
||||
`#!/usr/bin/env node
|
||||
const fs = require("node:fs");
|
||||
const args = process.argv.slice(2);
|
||||
const endpoint = args[1];
|
||||
const method = args[args.indexOf("--method") + 1];
|
||||
const body = args.includes("--input") ? JSON.parse(fs.readFileSync(0, "utf8")) : undefined;
|
||||
fs.appendFileSync(process.env.API_CALLS, JSON.stringify({ endpoint, method, body }) + "\\n");
|
||||
if (endpoint === "graphql" || method === "DELETE") {
|
||||
console.error(endpoint === "graphql" ? "commit rejected" : "cleanup rejected");
|
||||
process.exit(1);
|
||||
}
|
||||
if (endpoint.endsWith("/ref/heads/main")) console.log(process.env.BASE);
|
||||
else if (endpoint.includes("/matching-refs/")) console.log("refs/heads/bot/catalog-publish");
|
||||
else if (endpoint.includes("/ref/heads/bot/catalog-publish")) console.log("a".repeat(40));
|
||||
else if (endpoint.includes("/commits/")) console.log("b".repeat(40));
|
||||
else if (endpoint.includes("/pulls?")) console.log("42");
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
const calls = join(root, "calls.jsonl");
|
||||
const run = () =>
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
`
|
||||
import { publish } from ${JSON.stringify(new URL("./publish-catalog.mjs", import.meta.url).href)};
|
||||
try { publish(process.env.FIXTURE_ROOT); }
|
||||
catch (error) {
|
||||
console.log(JSON.stringify({ message: error.message, causes: error.errors?.map(cause => cause.message) }));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
`,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${root}:${process.env.PATH}`,
|
||||
BASE: base,
|
||||
API_CALLS: calls,
|
||||
FIXTURE_ROOT: root,
|
||||
GITHUB_REPOSITORY: "test/catalog",
|
||||
GITHUB_RUN_ID: "123",
|
||||
GITHUB_RUN_ATTEMPT: "1",
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
return {
|
||||
root,
|
||||
base,
|
||||
run,
|
||||
calls: () =>
|
||||
readFileSync(calls, "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line)),
|
||||
};
|
||||
}
|
||||
|
||||
test("reverting unpublished sources closes the obsolete PR and resets its branch to main", (t) => {
|
||||
const fixture = publicationFixture(t, false);
|
||||
fixture.run();
|
||||
assert.deepEqual(
|
||||
fixture.calls().filter((call) => call.method !== "GET"),
|
||||
[
|
||||
{ endpoint: "repos/test/catalog/pulls/42", method: "PATCH", body: { state: "closed" } },
|
||||
{
|
||||
endpoint: "repos/test/catalog/git/refs/heads/bot/catalog-publish",
|
||||
method: "PATCH",
|
||||
body: { sha: fixture.base, force: true },
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("a failed staging cleanup preserves the publication failure as well", (t) => {
|
||||
const fixture = publicationFixture(t, true);
|
||||
assert.throws(fixture.run, (error) => {
|
||||
const result = JSON.parse(error.stdout.trim().split("\n").at(-1));
|
||||
assert.equal(result.message, "Publication and staging cleanup failed.");
|
||||
assert.equal(result.causes.length, 2);
|
||||
assert.match(result.causes[0], /commit rejected/);
|
||||
assert.match(result.causes[1], /cleanup rejected/);
|
||||
return true;
|
||||
});
|
||||
assert.equal(fixture.calls().at(-1).method, "DELETE");
|
||||
assert.equal(
|
||||
fixture.calls().some((call) => call.method === "PATCH"),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("publication refuses symlinks instead of reading outside the generated artifact", (t) => {
|
||||
const fixture = publicationFixture(t, false);
|
||||
const artifact = join(fixture.root, "registry/registry.json");
|
||||
const unrelated = join(fixture.root, "private.txt");
|
||||
writeFileSync(unrelated, "must not be published");
|
||||
rmSync(artifact);
|
||||
symlinkSync(unrelated, artifact);
|
||||
assert.throws(() => catalogChanges(fixture.root, "HEAD"), /ELOOP|symbolic link/);
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { isGeneratedCatalogPath } from "./catalog-generated-paths.mjs";
|
||||
|
||||
const ITEM_FILE = /^registry\/(blocks|components|examples)\/[^/]+\//;
|
||||
const ITEM_MANIFEST = /^registry\/[^/]+\/[^/]+\/registry-item\.json$/;
|
||||
const INDEX = "registry/registry.json";
|
||||
|
||||
export function committedCatalogOutputs(paths, removalOnlyIndex = false) {
|
||||
if (!paths.some((path) => ITEM_FILE.test(path))) return [];
|
||||
return paths.filter(isGeneratedCatalogPath).filter((path) => path !== INDEX || !removalOnlyIndex);
|
||||
}
|
||||
|
||||
function git(...args) {
|
||||
return execFileSync("git", args, { encoding: "utf8" });
|
||||
}
|
||||
|
||||
function sourceFiles(ref) {
|
||||
return git("ls-tree", "-r", "--name-only", "-z", ref, "--", "registry")
|
||||
.split("\0")
|
||||
.filter((path) => ITEM_FILE.test(path));
|
||||
}
|
||||
|
||||
function removedItems(base) {
|
||||
const before = sourceFiles(base);
|
||||
const after = new Set(sourceFiles("HEAD"));
|
||||
const remainingDirectories = new Set([...after].map((path) => ITEM_FILE.exec(path)[0]));
|
||||
return before
|
||||
.filter((path) => ITEM_MANIFEST.test(path) && !after.has(path))
|
||||
.map((path) => {
|
||||
const directory = ITEM_FILE.exec(path)[0];
|
||||
if (remainingDirectories.has(directory))
|
||||
throw new Error(
|
||||
`Remove the complete item directory when deleting its manifest: ${directory}`,
|
||||
);
|
||||
return JSON.parse(git("show", `${base}:${path}`));
|
||||
});
|
||||
}
|
||||
|
||||
function checkIndexRemovals(base, indexChanged) {
|
||||
const removed = new Set(removedItems(base).map((item) => `${item.type}/${item.name}`));
|
||||
const before = JSON.parse(git("show", `${base}:${INDEX}`));
|
||||
const after = JSON.parse(git("show", `HEAD:${INDEX}`));
|
||||
const items = before.items.filter((item) => !removed.has(`${item.type}/${item.name}`));
|
||||
const hasRemovals = items.length < before.items.length;
|
||||
if (!isDeepStrictEqual(after, { ...before, items }))
|
||||
throw new Error(
|
||||
"Source PRs must remove deleted items from registry/registry.json in the same commit. " +
|
||||
"Do not add entries or change index metadata; automation publishes those changes.",
|
||||
);
|
||||
if (indexChanged && !hasRemovals)
|
||||
throw new Error(
|
||||
"Only removal of entries whose item directories are deleted may edit registry/registry.json.",
|
||||
);
|
||||
return hasRemovals;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const requestedBase = process.argv[2] ?? "origin/main";
|
||||
const base = git("merge-base", requestedBase, "HEAD").trim();
|
||||
const paths = git("diff", "--name-only", "-z", base, "HEAD").split("\0").filter(Boolean);
|
||||
if (!paths.some((path) => ITEM_FILE.test(path))) return;
|
||||
const removalOnly = checkIndexRemovals(base, paths.includes(INDEX));
|
||||
const outputs = committedCatalogOutputs(paths, removalOnly);
|
||||
if (outputs.length > 0)
|
||||
throw new Error(
|
||||
`Catalog PRs commit item sources only, with a removal-only index exception for deleted items. Restore these generated files:\n${outputs.join("\n")}`,
|
||||
);
|
||||
console.log("Catalog source PR contains only item sources and any required index removals.");
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main();
|
||||
@@ -0,0 +1,142 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { test } from "node:test";
|
||||
|
||||
const guard = fileURLToPath(new URL("./check-catalog-source-pr.mjs", import.meta.url));
|
||||
const old = { name: "old", type: "hyperframes:component" };
|
||||
const kept = { name: "kept", type: "hyperframes:block" };
|
||||
|
||||
function fixture(t) {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-source-pr-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
|
||||
const write = (path, contents) => {
|
||||
mkdirSync(dirname(join(root, path)), { recursive: true });
|
||||
writeFileSync(join(root, path), contents);
|
||||
};
|
||||
const item = (directory, entry) => {
|
||||
write(`registry/${directory}/${entry.name}/registry-item.json`, JSON.stringify(entry));
|
||||
write(`registry/${directory}/${entry.name}/index.html`, "<div>item</div>");
|
||||
};
|
||||
const index = (items, revision = "published") =>
|
||||
write(
|
||||
"registry/registry.json",
|
||||
JSON.stringify({ name: "catalog", items, vectorRevision: revision }),
|
||||
);
|
||||
const commit = () => {
|
||||
git("add", "-A");
|
||||
git(
|
||||
"-c",
|
||||
"user.name=Test",
|
||||
"-c",
|
||||
"user.email=test@example.invalid",
|
||||
"-c",
|
||||
"commit.gpgsign=false",
|
||||
"-c",
|
||||
"core.hooksPath=/dev/null",
|
||||
"commit",
|
||||
"-qm",
|
||||
"fixture",
|
||||
);
|
||||
};
|
||||
git("init", "-q");
|
||||
item("components", old);
|
||||
write("registry/components/old/assets/registry-item.json", "asset metadata");
|
||||
item("blocks", kept);
|
||||
index([old, kept]);
|
||||
commit();
|
||||
const base = git("rev-parse", "HEAD");
|
||||
return {
|
||||
root,
|
||||
item,
|
||||
index,
|
||||
removeOld: () => rmSync(join(root, "registry/components/old"), { recursive: true }),
|
||||
check: () => {
|
||||
commit();
|
||||
return spawnSync(process.execPath, [guard, base], { cwd: root, encoding: "utf8" });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("a deletion removes its index entry in the same source PR", (t) => {
|
||||
const repo = fixture(t);
|
||||
repo.removeOld();
|
||||
repo.index([kept]);
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
});
|
||||
|
||||
test("a rename unpublishes the old name and leaves the new name for automation", (t) => {
|
||||
const repo = fixture(t);
|
||||
repo.removeOld();
|
||||
repo.item("components", { ...old, name: "renamed" });
|
||||
repo.index([kept]);
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
});
|
||||
|
||||
test("an add-plus-delete PR carries only the old entry removal", (t) => {
|
||||
const repo = fixture(t);
|
||||
repo.removeOld();
|
||||
repo.item("blocks", { name: "new", type: "hyperframes:block" });
|
||||
repo.index([kept]);
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
});
|
||||
|
||||
test("a deletion cannot leave a dangling index entry", (t) => {
|
||||
const repo = fixture(t);
|
||||
repo.removeOld();
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr, /must remove deleted items/);
|
||||
});
|
||||
|
||||
test("the removal exception cannot publish an added entry", (t) => {
|
||||
const repo = fixture(t);
|
||||
repo.removeOld();
|
||||
const added = { name: "new", type: "hyperframes:block" };
|
||||
repo.item("blocks", added);
|
||||
repo.index([kept, added]);
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr, /Do not add entries/);
|
||||
});
|
||||
|
||||
test("the removal exception cannot unpublish a directory that still exists", (t) => {
|
||||
const repo = fixture(t);
|
||||
repo.item("blocks", { name: "new", type: "hyperframes:block" });
|
||||
repo.index([kept]);
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr, /must remove deleted items/);
|
||||
});
|
||||
|
||||
test("the removal exception preserves index metadata", (t) => {
|
||||
const repo = fixture(t);
|
||||
repo.removeOld();
|
||||
repo.index([kept], "changed");
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr, /change index metadata/);
|
||||
});
|
||||
|
||||
test("deleting only the manifest does not qualify as deleting an item directory", (t) => {
|
||||
const repo = fixture(t);
|
||||
rmSync(join(repo.root, "registry/components/old/registry-item.json"));
|
||||
repo.index([kept]);
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr, /complete item directory/);
|
||||
});
|
||||
|
||||
test("deleting nested asset metadata does not unpublish its item", (t) => {
|
||||
const repo = fixture(t);
|
||||
rmSync(join(repo.root, "registry/components/old/assets/registry-item.json"));
|
||||
const result = repo.check();
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { ensureLocalModel } from "../packages/cli/src/registry/localModel.ts";
|
||||
import { generateRegistryManifest } from "./generate-registry-items.ts";
|
||||
import { runAsCommand } from "./entrypoint.ts";
|
||||
|
||||
export async function generateCatalog(): Promise<void> {
|
||||
const root = fileURLToPath(new URL("..", import.meta.url));
|
||||
generateRegistryManifest(root);
|
||||
if (!(await ensureLocalModel()))
|
||||
throw new Error("Could not download the pinned catalog embedding model.");
|
||||
const run = (script: string) => execFileSync("bun", [script], { cwd: root, stdio: "inherit" });
|
||||
run("scripts/catalog/build-local-vectors.ts");
|
||||
// Rebuild the entire payload tree so removed items and orphaned assets disappear.
|
||||
rmSync(join(root, "docs/public/catalog"), { recursive: true, force: true });
|
||||
run("scripts/generate-catalog-payloads.ts");
|
||||
run("scripts/generate-catalog-pages.ts");
|
||||
run("scripts/sync-docs-catalog.mjs");
|
||||
}
|
||||
|
||||
runAsCommand(import.meta.url, generateCatalog);
|
||||
@@ -0,0 +1,31 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { generateRegistryManifest } from "./generate-registry-items.ts";
|
||||
|
||||
test("indexing preserves authored example manifests and indexes source-only additions", (t) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "registry-index-test-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
for (const dir of ["blocks", "components", "examples"])
|
||||
mkdirSync(join(root, "registry", dir), { recursive: true });
|
||||
const example = '{"name":"starter","type":"hyperframes:example","custom":"preserve me"}\n';
|
||||
mkdirSync(join(root, "registry/examples/starter"));
|
||||
writeFileSync(join(root, "registry/examples/starter/registry-item.json"), example);
|
||||
mkdirSync(join(root, "registry/components/new"));
|
||||
writeFileSync(
|
||||
join(root, "registry/components/new/registry-item.json"),
|
||||
'{"name":"new","type":"hyperframes:component"}',
|
||||
);
|
||||
generateRegistryManifest(root);
|
||||
assert.equal(
|
||||
readFileSync(join(root, "registry/examples/starter/registry-item.json"), "utf8"),
|
||||
example,
|
||||
);
|
||||
const manifest = JSON.parse(readFileSync(join(root, "registry/registry.json"), "utf8"));
|
||||
assert.deepEqual(manifest.items, [
|
||||
{ name: "starter", type: "hyperframes:example" },
|
||||
{ name: "new", type: "hyperframes:component" },
|
||||
]);
|
||||
});
|
||||
@@ -1,245 +1,56 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Generate registry-item.json manifests for every example in registry/examples/,
|
||||
* plus the top-level registry/registry.json manifest.
|
||||
*
|
||||
* Reads the legacy registry/examples/templates.json (label + hint) and probes
|
||||
* each example's index.html for dimensions / duration data attributes.
|
||||
* Placeholder `__VIDEO_DURATION__` falls back to 10 (the init-time default).
|
||||
*
|
||||
* Idempotent — safe to re-run, but will overwrite any hand-edits. Intended as
|
||||
* one-shot scaffolding for PR 3.
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/generate-registry-items.ts
|
||||
* bun run scripts/generate-registry-items.ts --only warm-grain
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, readdirSync, statSync } from "node:fs";
|
||||
import { join, relative, resolve, dirname } from "node:path";
|
||||
import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
ITEM_TYPE_DIRS,
|
||||
type FileTarget,
|
||||
type FileType,
|
||||
type RegistryItem,
|
||||
type RegistryManifest,
|
||||
// Import from source, like every other script here: bun workspace linking
|
||||
// does not resolve for scripts outside packages/, so the package name
|
||||
// typechecks on a machine with a warm node_modules and fails in CI.
|
||||
} from "../packages/core/src/index.js";
|
||||
import { ITEM_TYPE_DIRS } from "../packages/core/src/registry/types.ts";
|
||||
import { runAsCommand } from "./entrypoint.ts";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = resolve(scriptDir, "..");
|
||||
const examplesDir = resolve(repoRoot, "registry", ITEM_TYPE_DIRS["hyperframes:example"]);
|
||||
const registryManifestPath = resolve(repoRoot, "registry/registry.json");
|
||||
const legacyManifestPath = resolve(examplesDir, "templates.json");
|
||||
|
||||
const DEFAULT_DURATION_SECONDS = 10;
|
||||
const PLACEHOLDER_DURATION = "__VIDEO_DURATION__";
|
||||
|
||||
interface LegacyTemplateEntry {
|
||||
id: string;
|
||||
label: string;
|
||||
hint: string;
|
||||
bundled: boolean;
|
||||
function readIdentity(manifest: string): { name: unknown; type: unknown } {
|
||||
const item: unknown = JSON.parse(readFileSync(manifest, "utf8"));
|
||||
if (typeof item !== "object" || item === null)
|
||||
throw new Error(`Registry identity must be an object: ${manifest}`);
|
||||
return { name: Reflect.get(item, "name"), type: Reflect.get(item, "type") };
|
||||
}
|
||||
|
||||
interface LegacyManifest {
|
||||
templates: LegacyTemplateEntry[];
|
||||
function directoryItems(root: string, type: string, directory: string) {
|
||||
const path = join(root, "registry", directory);
|
||||
return readdirSync(path, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.flatMap((entry) => {
|
||||
const manifest = join(path, entry.name, "registry-item.json");
|
||||
if (!existsSync(manifest)) return [];
|
||||
const item = readIdentity(manifest);
|
||||
if (item.name !== entry.name || item.type !== type)
|
||||
throw new Error(`Registry identity does not match its directory: ${manifest}`);
|
||||
return [{ name: entry.name, type }];
|
||||
});
|
||||
}
|
||||
|
||||
function readLegacyManifest(): LegacyTemplateEntry[] {
|
||||
try {
|
||||
const raw = readFileSync(legacyManifestPath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as LegacyManifest;
|
||||
return parsed.templates;
|
||||
} catch {
|
||||
// templates.json was the bootstrap source and has been deleted. Fall back
|
||||
// to scanning existing registry-item.json files and reconstructing entries.
|
||||
return scanExistingItems();
|
||||
export function generateRegistryManifest(root: string): void {
|
||||
const items = Object.entries(ITEM_TYPE_DIRS).flatMap(([type, directory]) =>
|
||||
directoryItems(root, type, directory),
|
||||
);
|
||||
const names = new Set<string>();
|
||||
for (const item of items) {
|
||||
if (names.has(item.name)) throw new Error(`Duplicate registry name: ${item.name}`);
|
||||
names.add(item.name);
|
||||
}
|
||||
writeFileSync(
|
||||
join(root, "registry/registry.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
$schema: "https://hyperframes.heygen.com/schema/registry.json",
|
||||
name: "hyperframes",
|
||||
homepage: "https://hyperframes.heygen.com",
|
||||
items,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n",
|
||||
);
|
||||
console.log(`Indexed ${items.length} registry items without changing their sources.`);
|
||||
}
|
||||
|
||||
function scanExistingItems(): LegacyTemplateEntry[] {
|
||||
const entries: LegacyTemplateEntry[] = [];
|
||||
for (const dir of readdirSync(examplesDir, { withFileTypes: true })) {
|
||||
if (!dir.isDirectory()) continue;
|
||||
const itemPath = join(examplesDir, dir.name, "registry-item.json");
|
||||
try {
|
||||
const item = JSON.parse(readFileSync(itemPath, "utf-8")) as RegistryItem;
|
||||
entries.push({ id: item.name, label: item.title, hint: item.description, bundled: false });
|
||||
} catch {
|
||||
// No manifest — skip.
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function extractAttr(html: string, attr: string): string | undefined {
|
||||
const match = new RegExp(`data-${attr}="([^"]*)"`).exec(html);
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
interface CanvasMeta {
|
||||
width: number;
|
||||
height: number;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
function probeCanvas(exampleDir: string): CanvasMeta {
|
||||
const html = readFileSync(join(exampleDir, "index.html"), "utf-8");
|
||||
const width = Number(extractAttr(html, "width") ?? 1920);
|
||||
const height = Number(extractAttr(html, "height") ?? 1080);
|
||||
const rawDuration = extractAttr(html, "duration");
|
||||
const duration =
|
||||
rawDuration === undefined || rawDuration === PLACEHOLDER_DURATION
|
||||
? DEFAULT_DURATION_SECONDS
|
||||
: Number(rawDuration);
|
||||
return { width, height, duration };
|
||||
}
|
||||
|
||||
function fileTypeFor(path: string): FileType {
|
||||
if (path.endsWith(".html")) return "hyperframes:composition";
|
||||
return "hyperframes:asset";
|
||||
}
|
||||
|
||||
/** Walk the example dir and collect every tracked file (HTML + assets). */
|
||||
function collectFiles(exampleDir: string): FileTarget[] {
|
||||
const files: FileTarget[] = [];
|
||||
const walk = (dir: string): void => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(full);
|
||||
} else if (entry.isFile()) {
|
||||
// Skip the registry-item.json itself if it already exists from a
|
||||
// prior run; we're regenerating it.
|
||||
if (entry.name === "registry-item.json") continue;
|
||||
const rel = relative(exampleDir, full);
|
||||
files.push({ path: rel, target: rel, type: fileTypeFor(rel) });
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(exampleDir);
|
||||
files.sort((a, b) => a.path.localeCompare(b.path));
|
||||
return files;
|
||||
}
|
||||
|
||||
function buildItem(entry: LegacyTemplateEntry): RegistryItem {
|
||||
// The `blank` template is bundled inside the CLI package; don't generate a
|
||||
// manifest in registry/examples/ for it.
|
||||
const exampleDir = join(examplesDir, entry.id);
|
||||
const canvas = probeCanvas(exampleDir);
|
||||
const files = collectFiles(exampleDir);
|
||||
|
||||
return {
|
||||
$schema: "https://hyperframes.heygen.com/schema/registry-item.json",
|
||||
name: entry.id,
|
||||
type: "hyperframes:example",
|
||||
title: entry.label,
|
||||
description: entry.hint,
|
||||
dimensions: { width: canvas.width, height: canvas.height },
|
||||
duration: canvas.duration,
|
||||
files,
|
||||
};
|
||||
}
|
||||
|
||||
function writeItem(item: RegistryItem): void {
|
||||
if (item.type !== "hyperframes:example") return;
|
||||
const out = join(examplesDir, item.name, "registry-item.json");
|
||||
writeFileSync(out, JSON.stringify(item, null, 2) + "\n", "utf-8");
|
||||
console.log(`wrote ${relative(repoRoot, out)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* The manifest lists every item on disk, not just the examples this script
|
||||
* scaffolds. Blocks and components carry hand-authored registry-item.json
|
||||
* files; scanning the tree keeps them from being dropped on regeneration
|
||||
* (previously this rewrote 300+ entries down to the 8 examples).
|
||||
*/
|
||||
function writeRegistryManifest(): void {
|
||||
const items: Array<{ name: string; type: RegistryItem["type"] }> = [];
|
||||
for (const [type, dir] of Object.entries(ITEM_TYPE_DIRS) as Array<
|
||||
[RegistryItem["type"], string]
|
||||
>) {
|
||||
const typeDir = resolve(repoRoot, "registry", dir);
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = readdirSync(typeDir);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const name of entries.sort()) {
|
||||
try {
|
||||
statSync(join(typeDir, name, "registry-item.json"));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
items.push({ name, type });
|
||||
}
|
||||
}
|
||||
// `catalogArtifact.revision` belongs to build-local-vectors.ts, which stamps
|
||||
// it so the CLI and the coverage gate can tell whether the published vectors
|
||||
// still describe this registry. This script owns the item list and nothing
|
||||
// else, so it carries that field through rather than rewriting the file
|
||||
// without it: dropping it makes the gate fail and the staleness check answer
|
||||
// "missing" until someone rebuilds the index.
|
||||
const existing = ((): { catalogArtifact?: unknown } => {
|
||||
try {
|
||||
return JSON.parse(readFileSync(registryManifestPath, "utf-8")) as {
|
||||
catalogArtifact?: unknown;
|
||||
};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
})();
|
||||
const manifest: RegistryManifest = {
|
||||
$schema: "https://hyperframes.heygen.com/schema/registry.json",
|
||||
name: "hyperframes",
|
||||
homepage: "https://hyperframes.heygen.com",
|
||||
items,
|
||||
...(existing.catalogArtifact ? { catalogArtifact: existing.catalogArtifact } : {}),
|
||||
} as RegistryManifest;
|
||||
writeFileSync(registryManifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
|
||||
console.log(`wrote ${relative(repoRoot, registryManifestPath)} (${items.length} items)`);
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const args = process.argv.slice(2);
|
||||
const onlyIdx = args.indexOf("--only");
|
||||
const only = onlyIdx >= 0 ? args[onlyIdx + 1] : undefined;
|
||||
|
||||
const legacy = readLegacyManifest();
|
||||
// Skip bundled templates (e.g. `blank`) — they live inside the CLI package,
|
||||
// not under registry/examples/.
|
||||
const onDisk = legacy.filter((t) => !t.bundled);
|
||||
const filtered = only ? onDisk.filter((t) => t.id === only) : onDisk;
|
||||
|
||||
if (filtered.length === 0) {
|
||||
console.error(
|
||||
only
|
||||
? `No example matches --only ${only}. Available: ${onDisk.map((t) => t.id).join(", ")}`
|
||||
: "No examples found in registry/examples/templates.json",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
for (const entry of filtered) {
|
||||
const exampleDir = join(examplesDir, entry.id);
|
||||
try {
|
||||
statSync(exampleDir);
|
||||
} catch {
|
||||
console.warn(`skip ${entry.id}: directory not found at ${relative(repoRoot, exampleDir)}`);
|
||||
continue;
|
||||
}
|
||||
writeItem(buildItem(entry));
|
||||
}
|
||||
|
||||
// Only rewrite the top-level manifest on a full-run (not --only).
|
||||
if (!only) {
|
||||
writeRegistryManifest();
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
runAsCommand(import.meta.url, async () =>
|
||||
generateRegistryManifest(resolve(fileURLToPath(new URL("..", import.meta.url)))),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import {
|
||||
closeSync,
|
||||
constants,
|
||||
fstatSync,
|
||||
mkdtempSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
} from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { GENERATED_CATALOG_PATHS } from "./catalog-generated-paths.mjs";
|
||||
|
||||
const BRANCH = "bot/catalog-publish";
|
||||
const TITLE = "chore(catalog): publish generated catalog";
|
||||
const MAX_BATCH_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
function git(root, args) {
|
||||
return execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
|
||||
}
|
||||
|
||||
export function catalogChanges(root, base) {
|
||||
const tracked = git(root, [
|
||||
"diff",
|
||||
"--name-only",
|
||||
"--no-renames",
|
||||
"-z",
|
||||
base,
|
||||
"--",
|
||||
...GENERATED_CATALOG_PATHS,
|
||||
]);
|
||||
const untracked = git(root, [
|
||||
"ls-files",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"-z",
|
||||
"--",
|
||||
...GENERATED_CATALOG_PATHS,
|
||||
]);
|
||||
const removed = new Set(
|
||||
git(root, [
|
||||
"diff",
|
||||
"--name-only",
|
||||
"--diff-filter=D",
|
||||
"--no-renames",
|
||||
"-z",
|
||||
base,
|
||||
"--",
|
||||
...GENERATED_CATALOG_PATHS,
|
||||
]).split("\0"),
|
||||
);
|
||||
return [...new Set([...tracked.split("\0"), ...untracked.split("\0")])]
|
||||
.filter(Boolean)
|
||||
.sort()
|
||||
.map((path) => {
|
||||
if (removed.has(path)) return { path, kind: "delete" };
|
||||
return { path, kind: "write", contents: artifactContents(resolve(root, path)) };
|
||||
});
|
||||
}
|
||||
|
||||
function artifactContents(path) {
|
||||
const descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
if (!fstatSync(descriptor).isFile())
|
||||
throw new Error(`Generated artifact must be a regular file: ${path}`);
|
||||
return readFileSync(descriptor).toString("base64");
|
||||
} finally {
|
||||
closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function batchEntry(change) {
|
||||
const size = Buffer.byteLength(JSON.stringify(change));
|
||||
if (size > MAX_BATCH_BYTES)
|
||||
throw new Error(`Generated artifact exceeds API batch budget: ${change.path}`);
|
||||
if (change.kind === "delete") return { size, field: "deletions", file: { path: change.path } };
|
||||
return { size, field: "additions", file: { path: change.path, contents: change.contents } };
|
||||
}
|
||||
|
||||
export function commitBatches(changes) {
|
||||
let batch = { additions: [], deletions: [] };
|
||||
const batches = [batch];
|
||||
let bytes = 0;
|
||||
for (const change of changes) {
|
||||
const { size, field, file } = batchEntry(change);
|
||||
if (bytes + size > MAX_BATCH_BYTES || batch.additions.length + batch.deletions.length === 100) {
|
||||
batch = { additions: [], deletions: [] };
|
||||
batches.push(batch);
|
||||
bytes = 0;
|
||||
}
|
||||
batch[field].push(file);
|
||||
bytes += size;
|
||||
}
|
||||
return batches.filter((entry) => entry.additions.length + entry.deletions.length > 0);
|
||||
}
|
||||
|
||||
function api(endpoint, method = "GET", body, jq) {
|
||||
const args = ["api", endpoint, "--method", method];
|
||||
if (body !== undefined) args.push("--input", "-");
|
||||
if (jq) args.push("--jq", jq);
|
||||
return execFileSync("gh", args, {
|
||||
input: body === undefined ? undefined : JSON.stringify(body),
|
||||
encoding: "utf8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
timeout: 60_000,
|
||||
}).trim();
|
||||
}
|
||||
|
||||
function commitOid(value) {
|
||||
if (!/^[a-f0-9]{40}$/.test(value)) throw new Error("GitHub returned an invalid commit id.");
|
||||
return value;
|
||||
}
|
||||
|
||||
function catalogTree(root) {
|
||||
const temp = mkdtempSync(join(tmpdir(), "catalog-index-"));
|
||||
const run = (args) =>
|
||||
execFileSync("git", args, {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, GIT_INDEX_FILE: join(temp, "index") },
|
||||
}).trim();
|
||||
try {
|
||||
run(["read-tree", "HEAD"]);
|
||||
run(["add", "-A", "--", ...GENERATED_CATALOG_PATHS]);
|
||||
return run(["write-tree"]);
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function signedCommit(repository, branch, head, fileChanges) {
|
||||
const query = `mutation($input: CreateCommitOnBranchInput!) {
|
||||
createCommitOnBranch(input: $input) { commit { oid signature { isValid } } }
|
||||
}`;
|
||||
const result = api(
|
||||
"graphql",
|
||||
"POST",
|
||||
{
|
||||
query,
|
||||
variables: {
|
||||
input: {
|
||||
branch: { repositoryNameWithOwner: repository, branchName: branch },
|
||||
expectedHeadOid: head,
|
||||
message: { headline: TITLE },
|
||||
fileChanges,
|
||||
},
|
||||
},
|
||||
},
|
||||
'.data.createCommitOnBranch.commit | if .signature.isValid then .oid else error("Commit is not signed") end',
|
||||
);
|
||||
return commitOid(result);
|
||||
}
|
||||
|
||||
function openPublishPrNumber(repository) {
|
||||
const owner = repository.split("/")[0];
|
||||
const endpoint = `repos/${repository}/pulls`;
|
||||
const numbers = api(
|
||||
`${endpoint}?state=open&head=${owner}:${BRANCH}&base=main`,
|
||||
"GET",
|
||||
undefined,
|
||||
".[].number",
|
||||
)
|
||||
.split("\n")
|
||||
.filter(Boolean);
|
||||
if (numbers.length > 1) throw new Error("More than one catalog publish PR is open.");
|
||||
const number = numbers[0];
|
||||
if (number !== undefined && !/^\d+$/.test(number))
|
||||
throw new Error("GitHub returned an invalid PR number.");
|
||||
return number;
|
||||
}
|
||||
|
||||
function openPublishPr(repository, base) {
|
||||
const endpoint = `repos/${repository}/pulls`;
|
||||
const number = openPublishPrNumber(repository);
|
||||
const body =
|
||||
`Generated catalog snapshot from ${base}.\n\n` +
|
||||
"Item sources are reviewed in their own PRs. This PR publishes the registry index, search vectors, docs pages, payloads and navigation together.\n\n" +
|
||||
"Approve any GitHub Actions runs awaiting approval, then review and merge this PR after checks pass. Publication uses GITHUB_TOKEN and GitHub-signed API commits; branch protection remains in effect.";
|
||||
if (number !== undefined) {
|
||||
if (api(`${endpoint}/${number}`, "GET", undefined, ".body") !== body)
|
||||
api(`${endpoint}/${number}`, "PATCH", { title: TITLE, body });
|
||||
} else {
|
||||
api(endpoint, "POST", { title: TITLE, body, head: BRANCH, base: "main" });
|
||||
}
|
||||
}
|
||||
|
||||
function actionEnvironment() {
|
||||
const { GITHUB_REPOSITORY: repository = "", GITHUB_RUN_ID: run = "" } = process.env;
|
||||
if (!/^[\w.-]+\/[\w.-]+$/.test(repository) || !/^\d+$/.test(run))
|
||||
throw new Error("Run publication in GitHub Actions.");
|
||||
return { repository, run, endpoint: `repos/${repository}/git` };
|
||||
}
|
||||
|
||||
function assertCurrentMain(endpoint, base) {
|
||||
const current = commitOid(api(`${endpoint}/ref/heads/main`, "GET", undefined, ".object.sha"));
|
||||
if (current !== base)
|
||||
throw new Error("Main advanced during publication; the next push run will regenerate it.");
|
||||
}
|
||||
|
||||
function updateStandingBranch(endpoint, exists, head) {
|
||||
if (exists) api(`${endpoint}/refs/heads/${BRANCH}`, "PATCH", { sha: head, force: true });
|
||||
else api(`${endpoint}/refs`, "POST", { ref: `refs/heads/${BRANCH}`, sha: head });
|
||||
}
|
||||
|
||||
function clearObsoletePublication(repository, base, exists) {
|
||||
const endpoint = `repos/${repository}/git`;
|
||||
const number = openPublishPrNumber(repository);
|
||||
assertCurrentMain(endpoint, base);
|
||||
if (number !== undefined)
|
||||
api(`repos/${repository}/pulls/${number}`, "PATCH", { state: "closed" });
|
||||
if (exists) updateStandingBranch(endpoint, true, base);
|
||||
console.log("No unpublished catalog changes; obsolete publication cleared.");
|
||||
}
|
||||
|
||||
function snapshotMatches(root, endpoint, exists) {
|
||||
if (!exists) return false;
|
||||
const previous = commitOid(
|
||||
api(`${endpoint}/ref/heads/${BRANCH}`, "GET", undefined, ".object.sha"),
|
||||
);
|
||||
const tree = commitOid(api(`${endpoint}/commits/${previous}`, "GET", undefined, ".tree.sha"));
|
||||
return tree === catalogTree(root);
|
||||
}
|
||||
|
||||
function cleanStagingBranch(endpoint, staging, errors) {
|
||||
try {
|
||||
api(`${endpoint}/refs/heads/${staging}`, "DELETE");
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
if (errors.length === 1) throw errors[0];
|
||||
if (errors.length > 1)
|
||||
throw new AggregateError(errors, "Publication and staging cleanup failed.");
|
||||
}
|
||||
|
||||
function publishSnapshot(context, base, exists, batches) {
|
||||
const { repository, run, endpoint } = context;
|
||||
const staging = `${BRANCH}-build-${run}-${process.env.GITHUB_RUN_ATTEMPT ?? "1"}`;
|
||||
api(`${endpoint}/refs`, "POST", { ref: `refs/heads/${staging}`, sha: base });
|
||||
const errors = [];
|
||||
try {
|
||||
let head = base;
|
||||
for (const batch of batches) head = signedCommit(repository, staging, head, batch);
|
||||
assertCurrentMain(endpoint, base);
|
||||
updateStandingBranch(endpoint, exists, head);
|
||||
openPublishPr(repository, base);
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
cleanStagingBranch(endpoint, staging, errors);
|
||||
}
|
||||
|
||||
export function publish(root) {
|
||||
const base = commitOid(git(root, ["rev-parse", "HEAD"]));
|
||||
const changes = catalogChanges(root, base);
|
||||
const batches = commitBatches(changes);
|
||||
console.log(
|
||||
`Catalog publication: ${changes.length} files, ${batches.length} signed commit batches.`,
|
||||
);
|
||||
if (process.argv.includes("--dry-run")) return;
|
||||
const context = actionEnvironment();
|
||||
const { repository, endpoint } = context;
|
||||
assertCurrentMain(endpoint, base);
|
||||
const refs = api(`${endpoint}/matching-refs/heads/${BRANCH}`, "GET", undefined, ".[].ref").split(
|
||||
"\n",
|
||||
);
|
||||
const exists = refs.includes(`refs/heads/${BRANCH}`);
|
||||
if (batches.length === 0) return clearObsoletePublication(repository, base, exists);
|
||||
if (snapshotMatches(root, endpoint, exists)) {
|
||||
openPublishPr(repository, base);
|
||||
console.log("Standing catalog PR already contains this snapshot.");
|
||||
return;
|
||||
}
|
||||
publishSnapshot(context, base, exists, batches);
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
publish(fileURLToPath(new URL("..", import.meta.url)));
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env tsx
|
||||
// Explicit scaffolding overwrites authored example manifests; indexing never runs it.
|
||||
// Usage: bun scripts/scaffold-example-manifests.ts [--only warm-grain]
|
||||
|
||||
import { readFileSync, writeFileSync, readdirSync, statSync } from "node:fs";
|
||||
import { join, relative, resolve, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
ITEM_TYPE_DIRS,
|
||||
type FileTarget,
|
||||
type FileType,
|
||||
type RegistryItem,
|
||||
// Import from source, like every other script here: bun workspace linking
|
||||
// does not resolve for scripts outside packages/, so the package name
|
||||
// typechecks on a machine with a warm node_modules and fails in CI.
|
||||
} from "../packages/core/src/index.js";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = resolve(scriptDir, "..");
|
||||
const examplesDir = resolve(repoRoot, "registry", ITEM_TYPE_DIRS["hyperframes:example"]);
|
||||
const legacyManifestPath = resolve(examplesDir, "templates.json");
|
||||
|
||||
const DEFAULT_DURATION_SECONDS = 10;
|
||||
const PLACEHOLDER_DURATION = "__VIDEO_DURATION__";
|
||||
|
||||
interface LegacyTemplateEntry {
|
||||
id: string;
|
||||
label: string;
|
||||
hint: string;
|
||||
bundled: boolean;
|
||||
}
|
||||
|
||||
interface LegacyManifest {
|
||||
templates: LegacyTemplateEntry[];
|
||||
}
|
||||
|
||||
function readLegacyManifest(): LegacyTemplateEntry[] {
|
||||
try {
|
||||
const raw = readFileSync(legacyManifestPath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as LegacyManifest;
|
||||
return parsed.templates;
|
||||
} catch {
|
||||
// templates.json was the bootstrap source and has been deleted. Fall back
|
||||
// to scanning existing registry-item.json files and reconstructing entries.
|
||||
return scanExistingItems();
|
||||
}
|
||||
}
|
||||
|
||||
function scanExistingItems(): LegacyTemplateEntry[] {
|
||||
const entries: LegacyTemplateEntry[] = [];
|
||||
for (const dir of readdirSync(examplesDir, { withFileTypes: true })) {
|
||||
if (!dir.isDirectory()) continue;
|
||||
const itemPath = join(examplesDir, dir.name, "registry-item.json");
|
||||
try {
|
||||
const item = JSON.parse(readFileSync(itemPath, "utf-8")) as RegistryItem;
|
||||
entries.push({ id: item.name, label: item.title, hint: item.description, bundled: false });
|
||||
} catch {
|
||||
// No manifest — skip.
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function extractAttr(html: string, attr: string): string | undefined {
|
||||
const match = new RegExp(`data-${attr}="([^"]*)"`).exec(html);
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
interface CanvasMeta {
|
||||
width: number;
|
||||
height: number;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
function durationSeconds(rawDuration: string | undefined): number {
|
||||
return rawDuration === undefined || rawDuration === PLACEHOLDER_DURATION
|
||||
? DEFAULT_DURATION_SECONDS
|
||||
: Number(rawDuration);
|
||||
}
|
||||
|
||||
function probeCanvas(exampleDir: string): CanvasMeta {
|
||||
const html = readFileSync(join(exampleDir, "index.html"), "utf-8");
|
||||
const width = Number(extractAttr(html, "width") ?? 1920);
|
||||
const height = Number(extractAttr(html, "height") ?? 1080);
|
||||
const rawDuration = extractAttr(html, "duration");
|
||||
const duration = durationSeconds(rawDuration);
|
||||
return { width, height, duration };
|
||||
}
|
||||
|
||||
function fileTypeFor(path: string): FileType {
|
||||
if (path.endsWith(".html")) return "hyperframes:composition";
|
||||
return "hyperframes:asset";
|
||||
}
|
||||
|
||||
/** Walk the example dir and collect every tracked file (HTML + assets). */
|
||||
function collectFiles(exampleDir: string): FileTarget[] {
|
||||
return readdirSync(exampleDir, { recursive: true, withFileTypes: true })
|
||||
.filter((entry) => entry.isFile() && entry.name !== "registry-item.json")
|
||||
.map((entry) => {
|
||||
const path = relative(exampleDir, join(entry.parentPath, entry.name));
|
||||
return { path, target: path, type: fileTypeFor(path) };
|
||||
})
|
||||
.sort((a, b) => a.path.localeCompare(b.path));
|
||||
}
|
||||
|
||||
function buildItem(entry: LegacyTemplateEntry): RegistryItem {
|
||||
// The `blank` template is bundled inside the CLI package; don't generate a
|
||||
// manifest in registry/examples/ for it.
|
||||
const exampleDir = join(examplesDir, entry.id);
|
||||
const canvas = probeCanvas(exampleDir);
|
||||
const files = collectFiles(exampleDir);
|
||||
|
||||
return {
|
||||
$schema: "https://hyperframes.heygen.com/schema/registry-item.json",
|
||||
name: entry.id,
|
||||
type: "hyperframes:example",
|
||||
title: entry.label,
|
||||
description: entry.hint,
|
||||
dimensions: { width: canvas.width, height: canvas.height },
|
||||
duration: canvas.duration,
|
||||
files,
|
||||
};
|
||||
}
|
||||
|
||||
function writeItem(item: RegistryItem): void {
|
||||
if (item.type !== "hyperframes:example") return;
|
||||
const out = join(examplesDir, item.name, "registry-item.json");
|
||||
writeFileSync(out, JSON.stringify(item, null, 2) + "\n", "utf-8");
|
||||
console.log(`wrote ${relative(repoRoot, out)}`);
|
||||
}
|
||||
|
||||
function selectedExamples(): LegacyTemplateEntry[] {
|
||||
const args = process.argv.slice(2);
|
||||
const onlyIdx = args.indexOf("--only");
|
||||
const only = onlyIdx >= 0 ? args[onlyIdx + 1] : undefined;
|
||||
const onDisk = readLegacyManifest().filter((entry) => !entry.bundled);
|
||||
if (!only) return onDisk;
|
||||
const filtered = onDisk.filter((entry) => entry.id === only);
|
||||
if (filtered.length === 0)
|
||||
throw new Error(
|
||||
`No example matches --only ${only}. Available: ${onDisk.map((entry) => entry.id).join(", ")}`,
|
||||
);
|
||||
return filtered;
|
||||
}
|
||||
|
||||
function scaffoldExample(entry: LegacyTemplateEntry): void {
|
||||
const exampleDir = join(examplesDir, entry.id);
|
||||
try {
|
||||
statSync(exampleDir);
|
||||
} catch {
|
||||
console.warn(`skip ${entry.id}: directory not found at ${relative(repoRoot, exampleDir)}`);
|
||||
return;
|
||||
}
|
||||
writeItem(buildItem(entry));
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const entries = selectedExamples();
|
||||
if (entries.length === 0)
|
||||
throw new Error("No examples found in registry/examples/templates.json");
|
||||
for (const entry of entries) scaffoldExample(entry);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -1,6 +1,11 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
changedItems,
|
||||
declaresWebgpu,
|
||||
itemsFromDiff,
|
||||
withoutAbortedMedia,
|
||||
@@ -88,3 +93,28 @@ describe("withoutAbortedMedia", () => {
|
||||
assert.deepEqual(withoutAbortedMedia([aborted, ...kept]), kept);
|
||||
});
|
||||
});
|
||||
|
||||
it("browser selection includes regenerated edits and new untracked payloads", (t) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-selection-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const git = (...args: string[]) => execFileSync("git", args, { cwd: root });
|
||||
mkdirSync(join(root, "docs/public/catalog/blocks"), { recursive: true });
|
||||
const old = join(root, "docs/public/catalog/blocks/old.json");
|
||||
writeFileSync(old, "{}");
|
||||
git("init", "-q");
|
||||
git("add", ".");
|
||||
git(
|
||||
"-c",
|
||||
"user.name=Test",
|
||||
"-c",
|
||||
"user.email=test@example.invalid",
|
||||
"-c",
|
||||
"commit.gpgsign=false",
|
||||
"commit",
|
||||
"-qm",
|
||||
"fixture",
|
||||
);
|
||||
writeFileSync(old, '{"html":"updated"}');
|
||||
writeFileSync(join(root, "docs/public/catalog/blocks/new.json"), "{}");
|
||||
assert.deepEqual([...changedItems("HEAD", root)].sort(), ["new", "old"]);
|
||||
});
|
||||
|
||||
@@ -42,13 +42,21 @@ export function itemsFromDiff(diffOutput: string): Set<string> {
|
||||
return items;
|
||||
}
|
||||
|
||||
function changedItems(ref: string): Set<string> {
|
||||
export function changedItems(ref: string, cwd = join(scriptDir, "..")): Set<string> {
|
||||
const out = execFileSync(
|
||||
"git",
|
||||
["diff", "--name-only", "--diff-filter=ACMR", `${ref}...HEAD`, "--", "docs/public/catalog"],
|
||||
{ encoding: "utf-8", cwd: join(scriptDir, "..") },
|
||||
["diff", "--name-only", "--diff-filter=ACMR", ref, "--", "docs/public/catalog"],
|
||||
{ encoding: "utf-8", cwd },
|
||||
);
|
||||
return itemsFromDiff(out);
|
||||
const added = execFileSync(
|
||||
"git",
|
||||
["ls-files", "--others", "--exclude-standard", "--", "docs/public/catalog"],
|
||||
{
|
||||
encoding: "utf-8",
|
||||
cwd,
|
||||
},
|
||||
);
|
||||
return itemsFromDiff(out + "\n" + added);
|
||||
}
|
||||
|
||||
function payloadFiles(
|
||||
|
||||
Reference in New Issue
Block a user