Files
hyperframes/packages/cli/src/utils/publishProject.ts
T
Michael Wang 04202e8a4c fix(cli): uploads leave out outside scripts; .ts counts as video only when it is one (#7818)
* fix(cli): uploads leave out outside scripts; .ts counts as video only when it is one

* chore(studio-server): raise unpacked size budget for the upload file-type rule

* fix(studio-server): transport stream check never waits on a pipe and needs binary bytes
2026-11-15 09:52:49 +00:00

1091 lines
38 KiB
TypeScript

import { envProxyEnabled } from "@hyperframes/core/env-proxy";
import { basename, dirname, isAbsolute, join, posix, relative, resolve } from "node:path";
import {
closeSync,
existsSync,
fstatSync,
openSync,
readdirSync,
readFileSync,
type Stats,
statSync,
} from "node:fs";
import { parseHTML } from "linkedom";
import AdmZip from "adm-zip";
import type { Ignore } from "ignore";
import { projectIgnore } from "@hyperframes/lint";
import { isNonRelativeUrl, isPathInside, rewriteSrcsetUrls } from "@hyperframes/core";
import { cssUrlMatches, replaceMatches } from "@hyperframes/parsers/css-url-scans";
import { decodeUrlPathVariants, encodeUrlPath } from "@hyperframes/parsers";
import { buildAuthHeaders } from "../auth/client.js";
import { tryResolveCredential } from "../auth/index.js";
import { isAuthError } from "../auth/errors.js";
import { envCredentialVar, isTokenExpired, type ResolvedCredential } from "../auth/resolver.js";
import { refreshIfNeeded } from "../cloud/auth.js";
import { writeProjectLink } from "./projectLink.js";
import {
isPrivateCompositionFile,
isPrivateOutsideFile,
linksOutside,
linksToPrivateOutsideFile,
receivedFiles,
} from "./privateProjectFile.js";
import {
leftOutReference,
leftOutWarning,
type LeftOutReason,
type OnLeftOut,
} from "./leftOutReference.js";
import { isUploadedOutsideFileType } from "./outsideFileType.js";
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
const PUBLISH_CONTENT_TYPE = "application/zip";
const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
const PUBLISH_TRANSPORT_ATTEMPTS = 2;
const PUBLISH_RETRY_DELAY_MS = 200;
// Conservative floor — most connections are faster, but this prevents
// premature aborts on slow/unstable networks (hotel wifi, tethering).
const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000;
export interface PublishArchiveResult {
buffer: Buffer;
fileCount: number;
}
export interface PublishedProjectResponse {
projectId: string;
title: string;
fileCount: number;
url: string;
claimToken: string;
/** True when the project is owned by the authenticated publisher (created-and-owned or updated in place). */
claimed: boolean;
}
interface StagedUploadResponse {
uploadUrl: string;
uploadKey: string;
contentType: string;
uploadHeaders: Record<string, string>;
expiresInSeconds: number;
}
type JsonRecord = Record<string, unknown>;
function isRecord(value: unknown): value is JsonRecord {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function dataRecord(payload: unknown): JsonRecord | null {
if (!isRecord(payload) || !isRecord(payload["data"])) return null;
return payload["data"];
}
function stringField(record: JsonRecord, key: string): string | null {
const value = record[key];
return typeof value === "string" ? value : null;
}
function parsePublishedProjectResponse(payload: unknown): PublishedProjectResponse | null {
const data = dataRecord(payload);
if (!data) return null;
const projectId = stringField(data, "project_id");
const title = stringField(data, "title");
const url = stringField(data, "url");
const claimToken = stringField(data, "claim_token") ?? "";
const claimed = data["claimed"] === true;
const fileCount = data["file_count"];
if (!projectId || !title || !url || typeof fileCount !== "number") {
return null;
}
// Anonymous publishes must return a claim token; owned (claimed) ones need none.
if (!claimed && !claimToken) {
return null;
}
return {
projectId,
title,
fileCount,
url,
claimToken,
claimed,
};
}
function parseStagedUploadResponse(
payload: unknown,
archiveByteLength: number,
): StagedUploadResponse | null {
const data = dataRecord(payload);
if (!data) return null;
const uploadUrl = stringField(data, "upload_url");
const uploadKey = stringField(data, "upload_key");
const contentType = stringField(data, "content_type") || PUBLISH_CONTENT_TYPE;
if (!uploadUrl || !uploadKey) return null;
const rawExpires = data["expires_in_seconds"];
const expiresInSeconds = typeof rawExpires === "number" && rawExpires > 0 ? rawExpires : 1800;
return {
uploadUrl,
uploadKey,
contentType,
uploadHeaders: getUploadHeaders(data, uploadUrl, contentType, archiveByteLength),
expiresInSeconds,
};
}
function getUploadHeaders(
data: JsonRecord,
uploadUrl: string,
contentType: string,
archiveByteLength: number,
): Record<string, string> {
const headers: Record<string, string> = {};
const uploadHeaders = data["upload_headers"];
if (isRecord(uploadHeaders)) {
for (const [key, value] of Object.entries(uploadHeaders)) {
if (typeof value === "string" && key.trim()) {
headers[key] = value;
}
}
}
setHeaderIfAbsent(headers, "content-type", contentType);
const signedHeaders =
new URL(uploadUrl).searchParams.get("X-Amz-SignedHeaders")?.split(";") ?? [];
if (signedHeaders.includes("x-amz-server-side-encryption")) {
setHeaderIfAbsent(headers, "x-amz-server-side-encryption", "AES256");
}
if (signedHeaders.includes("content-length")) {
setHeaderIfAbsent(headers, "content-length", String(archiveByteLength));
}
return headers;
}
function setHeaderIfAbsent(headers: Record<string, string>, name: string, value: string): void {
if (!Object.keys(headers).some((key) => key.toLowerCase() === name)) headers[name] = value;
}
async function readJson(response: Response): Promise<unknown> {
return response
.clone()
.json()
.catch(() => null);
}
class CredentialRejectedError extends Error {}
async function metadataRequestError(response: Response, fallback: string): Promise<Error> {
const message = await readErrorMessage(response, fallback);
return response.status === 401 ? new CredentialRejectedError(message) : new Error(message);
}
const LOGIN_EXPIRED = "Your login expired. Run hyperframes auth login, then publish again.";
const LOGIN_REQUIRED = "Publishing needs an account. Run hyperframes auth login to publish.";
const LOGIN_CHANGED = "Your login changed during publish. Run publish again.";
function rejectedCredentialMessage(credential: ResolvedCredential): string {
const envVar = envCredentialVar(credential.source);
return envVar ? `${envVar} was rejected. Fix or unset it, then publish again.` : LOGIN_EXPIRED;
}
/** Resolves the credential, or refreshes `checked` (a credential already resolved) without re-resolving. */
export async function resolvePublishCredential(
checked?: ResolvedCredential | null,
): Promise<ResolvedCredential | null> {
try {
const credential = checked === undefined ? await tryResolveCredential() : checked;
if (!credential) return null;
// A checked login can expire during a long proxy bake; refresh it rather than fall back to an API key.
const expired =
credential.type === "oauth" && isTokenExpired(credential.expires_at, new Date());
if (expired && !credential.refresh_token) throw new Error(LOGIN_EXPIRED);
return await refreshIfNeeded(expired ? { ...credential, refreshable: true } : credential, {
envFallback: checked === undefined,
});
} catch (error) {
if (isAuthError(error) && (error.code === "REFRESH_FAILED" || error.code === "LOGIN_EXPIRED")) {
throw new Error(LOGIN_EXPIRED);
}
if (isAuthError(error) && error.code === "LOGIN_CHANGED") throw new Error(LOGIN_CHANGED);
throw error;
}
}
async function readErrorMessage(response: Response, fallback: string): Promise<string> {
const contentType = response.headers.get("content-type") || "";
if (contentType.includes("application/json")) {
const payload = await readJson(response);
if (isRecord(payload) && typeof payload["message"] === "string") {
return payload["message"];
}
}
if (response.status === 403 && response.headers.get("cf-mitigated") === "challenge") {
return "Publish upload was blocked before reaching HyperFrames. Please retry after staged uploads are available.";
}
const text = await response.text().catch(() => "");
return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
}
function systemErrorMetadata(value: unknown): string[] {
if (!isRecord(value)) return [];
const metadata: string[] = [];
if (typeof value["code"] === "string") metadata.push(value["code"]);
if (typeof value["syscall"] === "string") metadata.push(`syscall=${value["syscall"]}`);
if (typeof value["errno"] === "string" || typeof value["errno"] === "number") {
metadata.push(`errno=${value["errno"]}`);
}
return metadata;
}
function redactUrlQuery(message: string): string {
return message.replace(/(https?:\/\/[^\s?]+)\?[^\s]+/gu, "$1?[redacted]");
}
function proxySupportHint(): string {
const proxyConfigured = ["HTTPS_PROXY", "HTTP_PROXY", "https_proxy", "http_proxy"].some((key) =>
Boolean(process.env[key]?.trim()),
);
if (!proxyConfigured || envProxyEnabled()) return "";
return (
". Proxy variables are set but ignored by Node fetch; if this network requires them, retry with " +
"NODE_USE_ENV_PROXY=1 (Node 22.21+)"
);
}
function describeFetchFailure(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
const cause = error instanceof Error ? error.cause : undefined;
const causeMessage = cause instanceof Error ? cause.message : "";
const metadata = [...systemErrorMetadata(cause), ...systemErrorMetadata(error)].filter(
(value, index, all) => all.indexOf(value) === index,
);
const distinctCauseMessage = causeMessage && causeMessage !== message ? causeMessage : "";
const detail = [metadata.join(", "), distinctCauseMessage].filter(Boolean).join(": ");
return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`;
}
function isRequestTimeout(error: unknown): boolean {
return (
error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError")
);
}
function waitBeforePublishRetry(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, PUBLISH_RETRY_DELAY_MS));
}
async function fetchForPublish(
input: string,
createInit: () => RequestInit,
failureStage: string,
attempts = 1,
): Promise<Response> {
if (attempts < 1) throw new RangeError("Publish fetch attempts must be at least 1");
let lastError: unknown;
let attemptsMade = 0;
for (let attempt = 1; attempt <= attempts; attempt += 1) {
attemptsMade = attempt;
try {
return await fetch(input, createInit());
} catch (error) {
lastError = error;
if (isRequestTimeout(error) || attempt === attempts) break;
await waitBeforePublishRetry();
}
}
const attemptDetail = attemptsMade > 1 ? ` after ${attemptsMade} attempts` : "";
throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(lastError)}`, {
cause: lastError instanceof Error ? lastError : undefined,
});
}
export function uploadTimeoutMs(byteLength: number): number {
return Math.max(
PUBLISH_UPLOAD_MIN_TIMEOUT_MS,
Math.ceil((byteLength / PUBLISH_UPLOAD_BYTES_PER_SECOND) * 1000),
);
}
// Absolute, non-negotiable exclusions only — anything a `.hyperframesignore`
// negation rule should be able to override (including dot-prefixed paths;
// see projectIgnore) must go through the ignore matcher instead of
// short-circuiting here.
function shouldIgnoreSegment(segment: string): boolean {
return IGNORED_DIRS.has(segment) || IGNORED_FILES.has(segment);
}
function collectProjectFiles(
rootDir: string,
currentDir: string,
paths: string[],
matcher: Ignore,
): void {
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
if (shouldIgnoreSegment(entry.name)) continue;
const absolutePath = join(currentDir, entry.name);
const relativePath = relative(rootDir, absolutePath).replaceAll("\\", "/");
if (!relativePath) continue;
if (entry.isDirectory()) {
if (matcher.ignores(`${relativePath}/`)) continue;
collectProjectFiles(rootDir, absolutePath, paths, matcher);
continue;
}
if (!statSync(absolutePath).isFile()) continue;
if (matcher.ignores(relativePath)) continue;
paths.push(relativePath);
}
}
const EXT_ASSETS_PREFIX = "_ext";
/** Files whose text a reference rewrite or scan reads; they are always staged in memory. */
export const TEXT_FILE_EXTS = /\.(html?|css|m?js|cjs|json|svg|xml|txt)$/i;
/** A large non-text project file left on disk; `size` and `mtimeMs` detect an edit after staging. */
export interface DiskFile {
absPath: string;
size: number;
mtimeMs: number;
}
export type StagedFile = Buffer | DiskFile;
export function stagedSize(file: StagedFile): number {
return Buffer.isBuffer(file) ? file.byteLength : file.size;
}
function assertSame(file: DiskFile, now: Stats | undefined): void {
if (now?.size !== file.size || now.mtimeMs !== file.mtimeMs) {
throw new Error(`${file.absPath} changed on disk while it was being prepared. Run it again.`);
}
}
export function assertUnchanged(file: DiskFile): void {
assertSame(file, statSync(file.absPath, { throwIfNoEntry: false }));
}
/** Reads through one descriptor, so `check` (run before and after) sees the stat of the bytes returned. */
export function readChecked(path: string, check: (stat: Stats) => void): Buffer {
const fd = openSync(path, "r");
try {
check(fstatSync(fd));
const bytes = readFileSync(fd);
check(fstatSync(fd));
return bytes;
} finally {
closeSync(fd);
}
}
/** The staged bytes; a file left on disk is read now and must not have changed since staging. */
export function readStaged(file: StagedFile): Buffer {
if (Buffer.isBuffer(file)) return file;
return readChecked(file.absPath, (stat) => assertSame(file, stat));
}
function stageFile(absPath: string, onDiskMinBytes: number): StagedFile {
const fd = openSync(absPath, "r");
try {
const { size, mtimeMs } = fstatSync(fd);
if (size < onDiskMinBytes || TEXT_FILE_EXTS.test(absPath)) return readFileSync(fd);
return { absPath, size, mtimeMs };
} finally {
closeSync(fd);
}
}
interface ExternalAssetContext {
absProjectDir: string;
fileContents: Map<string, StagedFile>;
onDiskMinBytes: number;
externalMap: Map<string, string>;
usedArchivePaths: Set<string>;
leftOut: Set<string>;
onLeftOut?: OnLeftOut;
received: (path: string) => boolean;
}
function addExternalAsset(ctx: ExternalAssetContext, absPath: string): string {
const existing = ctx.externalMap.get(absPath);
if (existing) return existing;
const rel = relative(ctx.absProjectDir, absPath).replaceAll("\\", "/");
const stripped = rel.replace(/^(?:\.\.\/)+/, "");
let archivePath = archiveKey(`${EXT_ASSETS_PREFIX}/${stripped}`);
if (ctx.usedArchivePaths.has(archivePath)) {
const ext = posix.extname(archivePath);
const base = archivePath.slice(0, archivePath.length - ext.length);
let i = 2;
while (ctx.usedArchivePaths.has(`${base}_${i}${ext}`)) i++;
archivePath = `${base}_${i}${ext}`;
}
ctx.fileContents.set(archivePath, stageFile(absPath, ctx.onDiskMinBytes));
ctx.externalMap.set(absPath, archivePath);
ctx.usedArchivePaths.add(archivePath);
return archivePath;
}
function staysOut(ctx: ExternalAssetContext, rel: string, absPath: string): boolean {
if (!rel || ctx.fileContents.has(rel) || rel.split("/").some(shouldIgnoreSegment)) return true;
return (
isPrivateCompositionFile(ctx.absProjectDir, absPath) ||
(existsSync(absPath) && linksToPrivateOutsideFile(ctx.absProjectDir, absPath))
);
}
/** Ships a referenced in-project file the ignore rules skipped, resolved from its folder or the root. */
function includeReferencedProjectFile(
ctx: ExternalAssetContext,
resolvedPath: string,
rawPath: string,
requireOnDisk: boolean,
entryPath: string,
): void {
const fromRoot = resolve(ctx.absProjectDir, rawPath);
for (const candidate of [resolvedPath, fromRoot]) {
const absPath = candidate.replace(/[?#].*$/, "");
if (!isPathInside(absPath, ctx.absProjectDir)) continue;
const rel = archiveKey(relative(ctx.absProjectDir, absPath).replaceAll("\\", "/"));
if (staysOut(ctx, rel, absPath)) return;
if (!existsSync(absPath)) continue;
if (ctx.received(entryPath) && linksOutside(ctx.absProjectDir, absPath)) {
return leaveOut(ctx, entryPath, rawPath, absPath, "received");
}
if (statSync(absPath).isFile()) {
ctx.fileContents.set(rel, stageFile(absPath, ctx.onDiskMinBytes));
}
return;
}
if (!requireOnDisk) return;
throw new Error(`The composition references "${rawPath}", which is not in the project folder.`);
}
/** References with no fallback: a missing file here cannot render. */
const HARD_REFERENCES = new Set(["video[src]", "audio[src]", "img[src]", "video[poster]"]);
const relativeToProject = (ctx: ExternalAssetContext, absPath: string) =>
encodeUrlPath(relative(ctx.absProjectDir, absPath).replaceAll("\\", "/"));
function leaveOut(
ctx: ExternalAssetContext,
entryPath: string,
rawPath: string,
absPath: string,
reason: LeftOutReason = "blocklisted",
): void {
const warning = leftOutWarning(entryPath, rawPath, reason);
if (ctx.leftOut.has(warning)) return;
ctx.leftOut.add(warning);
ctx.onLeftOut?.(
warning,
leftOutReference(ctx.absProjectDir, entryPath, rawPath, absPath, reason),
);
}
function archiveTargetOf(
ctx: ExternalAssetContext,
absPath: string,
rawPath: string,
options: { entryPath: string; requireOnDisk: boolean; filesystemPath: boolean },
): string | null | undefined {
if (isPathInside(absPath, ctx.absProjectDir)) {
includeReferencedProjectFile(ctx, absPath, rawPath, options.requireOnDisk, options.entryPath);
return options.filesystemPath ? relativeToProject(ctx, absPath) : null;
}
try {
if (!existsSync(absPath) || !statSync(absPath).isFile()) return undefined;
if (ctx.received(options.entryPath)) {
leaveOut(ctx, options.entryPath, rawPath, absPath, "received");
return undefined;
}
if (isPrivateOutsideFile(ctx.absProjectDir, absPath)) {
leaveOut(ctx, options.entryPath, rawPath, absPath);
return undefined;
}
if (!isUploadedOutsideFileType(absPath)) {
leaveOut(ctx, options.entryPath, rawPath, absPath, "file_type");
return undefined;
}
} catch {
return undefined;
}
return encodeUrlPath(addExternalAsset(ctx, absPath));
}
function tryResolveExternal(
ctx: ExternalAssetContext,
rawPath: string,
referrerAbsDir: string,
entryPath: string,
requireOnDisk = false,
): string | null {
const filesystemPath = isAbsolute(rawPath) && !rawPath.startsWith("//");
if (isNonRelativeUrl(rawPath) && !filesystemPath) return null;
const cut = filesystemPath ? rawPath.search(/[?#]/) : -1;
const [path, suffix] = cut < 0 ? [rawPath, ""] : [rawPath.slice(0, cut), rawPath.slice(cut)];
for (const variant of decodeUrlPathVariants(path)) {
const target = archiveTargetOf(ctx, resolve(referrerAbsDir, variant), rawPath, {
entryPath,
requireOnDisk,
filesystemPath,
});
if (target !== undefined) return target && target + suffix;
}
return null;
}
/** Resolves a raw attribute or CSS `url()` value to the archive path or absolute
* URL it should point at, or `null` to leave it untouched. */
export type HtmlAttributeResolver = (
rawValue: string,
referrerAbsDir: string,
ref?: { attr: string; tag: string },
) => string | null;
function referenceFrom(entryPath: string, target: string): string {
return isNonRelativeUrl(target) ? target : posix.relative(posix.dirname(entryPath), target);
}
export function rewriteCssUrls(
css: string,
referrerAbsDir: string,
entryPath: string,
resolveTarget: HtmlAttributeResolver,
): { css: string; modified: boolean } {
let modified = false;
const rewritten = replaceMatches(css, cssUrlMatches(css), (full, quote, rawUrl) => {
const target = resolveTarget((rawUrl || "").trim(), referrerAbsDir);
if (!target) return full;
modified = true;
return `url(${quote || ""}${referenceFrom(entryPath, target)}${quote || ""})`;
});
return { css: rewritten, modified };
}
interface RewriteHtmlAttributesOptions {
/** Attributes to inspect (default: src + href, matching the external-asset
* localization use case below). */
attrs?: string[];
/** CSS selector narrowing which elements are inspected (default: derived
* from `attrs`, e.g. `"[src], [href]"`). Callers that only care about one
* tag (e.g. `<video>`) pass something like `"video[src]"`. */
selector?: string;
}
/**
* Walk every element matching `selector` (default: anything with `src`/
* `href`) and rewrite the given `attrs` whose value `resolveTarget` maps to an
* archive path. Shared by `localizeHtmlEntry` below (external-asset
* localization) and `publishProxyBake.ts` (proxy baking only rewrites
* `<video src>`), so the rewrite mechanics (attribute walk + entry-relative
* path rewrite) live in one place while each caller supplies its own
* resolution rule.
*/
export function rewriteHtmlAttributes(
document: Document,
referrerAbsDir: string,
entryPath: string,
resolveTarget: HtmlAttributeResolver,
options: RewriteHtmlAttributesOptions = {},
): boolean {
const attrs = options.attrs ?? ["src", "href"];
const selector = options.selector ?? attrs.map((attr) => `[${attr}]`).join(", ");
let modified = false;
for (const el of document.querySelectorAll(selector)) {
for (const attr of attrs) {
const val = (el.getAttribute(attr) || "").trim();
if (!val) continue;
const target = resolveTarget(val, referrerAbsDir, { attr, tag: el.localName });
if (!target) continue;
el.setAttribute(attr, referenceFrom(entryPath, target));
modified = true;
}
}
return modified;
}
export function rewriteStyleBlocks(
document: Document,
referrerAbsDir: string,
entryPath: string,
resolveTarget: HtmlAttributeResolver,
): boolean {
let modified = false;
for (const styleEl of document.querySelectorAll("style")) {
const css = styleEl.textContent || "";
if (!css.includes("url(")) continue;
const result = rewriteCssUrls(css, referrerAbsDir, entryPath, resolveTarget);
if (result.modified) {
styleEl.textContent = result.css;
modified = true;
}
}
for (const el of document.querySelectorAll("[style]")) {
const style = el.getAttribute("style") || "";
if (!style.includes("url(")) continue;
const result = rewriteCssUrls(style, referrerAbsDir, entryPath, resolveTarget);
if (result.modified) {
el.setAttribute("style", result.css);
modified = true;
}
}
return modified;
}
function rewriteHtmlSrcsets(
document: Document,
referrerAbsDir: string,
entryPath: string,
resolveTarget: HtmlAttributeResolver,
): boolean {
let modified = false;
for (const el of document.querySelectorAll("[srcset]")) {
const srcset = el.getAttribute("srcset");
if (!srcset) continue;
const rewritten = rewriteSrcsetUrls(srcset, (url) => {
const target = resolveTarget(url, referrerAbsDir);
return target ? referenceFrom(entryPath, target) : url;
});
if (rewritten === srcset) continue;
el.setAttribute("srcset", rewritten);
modified = true;
}
return modified;
}
function localizeHtmlEntry(ctx: ExternalAssetContext, entryPath: string, content: Buffer): void {
const referrerAbsDir = resolve(ctx.absProjectDir, dirname(entryPath));
const { document } = parseHTML(content.toString("utf-8"));
const resolveTarget: HtmlAttributeResolver = (val, dir, ref) =>
tryResolveExternal(
ctx,
val,
dir,
entryPath,
ref !== undefined && HARD_REFERENCES.has(`${ref.tag}[${ref.attr}]`),
);
const attrsChanged = rewriteHtmlAttributes(document, referrerAbsDir, entryPath, resolveTarget, {
attrs: ["src", "href", "poster"],
});
const stylesChanged = rewriteStyleBlocks(document, referrerAbsDir, entryPath, resolveTarget);
const srcsetsChanged = rewriteHtmlSrcsets(document, referrerAbsDir, entryPath, resolveTarget);
if (attrsChanged || stylesChanged || srcsetsChanged) {
ctx.fileContents.set(entryPath, Buffer.from(document.toString(), "utf-8"));
}
}
function localizeCssEntry(ctx: ExternalAssetContext, entryPath: string, content: Buffer): void {
const referrerAbsDir = resolve(ctx.absProjectDir, dirname(entryPath));
const css = content.toString("utf-8");
if (!css.includes("url(")) return;
const result = rewriteCssUrls(css, referrerAbsDir, entryPath, (val, dir) =>
tryResolveExternal(ctx, val, dir, entryPath),
);
if (result.modified) {
ctx.fileContents.set(entryPath, Buffer.from(result.css, "utf-8"));
}
}
/**
* Scan HTML and CSS files for asset references that resolve outside the
* project directory. Copy those files into the archive under `_ext/` and
* rewrite the references so the published project is self-contained.
*/
export function localizeExternalAssets(
absProjectDir: string,
fileContents: Map<string, StagedFile>,
onDiskMinBytes = Infinity,
onLeftOut?: OnLeftOut,
): number {
const ctx: ExternalAssetContext = {
absProjectDir,
fileContents,
onDiskMinBytes,
externalMap: new Map(),
usedArchivePaths: new Set(),
leftOut: new Set(),
onLeftOut,
received: receivedFiles(absProjectDir),
};
for (const [entryPath, content] of [...fileContents.entries()]) {
if (entryPath.startsWith(EXT_ASSETS_PREFIX + "/") || !Buffer.isBuffer(content)) continue;
if (entryPath.endsWith(".html") || entryPath.endsWith(".htm")) {
localizeHtmlEntry(ctx, entryPath, content);
} else if (entryPath.endsWith(".css")) {
localizeCssEntry(ctx, entryPath, content);
}
}
return ctx.externalMap.size;
}
/**
* Walk the project dir, read every non-ignored file, and localize external
* (out-of-project) asset references. Returns the in-memory archive file map —
* the seam `publish.ts` hooks a proxy-baking transform into (U6) between this
* and `zipPublishFileMap` below. `cloud render` never sees this seam: it
* keeps calling `createPublishArchive` directly.
*/
export function buildPublishFileMap(
projectDir: string,
onLeftOut?: OnLeftOut,
): Map<string, Buffer> {
return stagePublishFiles(projectDir, Infinity, onLeftOut) as Map<string, Buffer>;
}
export const archiveKey = (path: string): string => path.normalize("NFC");
function composeNamesInText(fileContents: Map<string, StagedFile>, filePaths: string[]): void {
const names = new Map<string, string>();
for (const segment of filePaths.flatMap((path) => path.split("/"))) {
const composed = archiveKey(segment);
if (composed === segment) continue;
names.set(segment, composed);
names.set(encodeURIComponent(segment), encodeURIComponent(composed));
}
if (names.size === 0) return;
for (const [key, file] of fileContents) {
if (!Buffer.isBuffer(file) || !TEXT_FILE_EXTS.test(key)) continue;
let text = file.toString("utf-8");
const before = text;
for (const [from, to] of names) text = text.replaceAll(from, to);
if (text !== before) fileContents.set(key, Buffer.from(text, "utf-8"));
}
}
type WalkedLeftOutReason = Exclude<LeftOutReason, "file_type">;
const WALKED_LEFT_OUT: Record<WalkedLeftOutReason, string> = {
private: "holds secrets or tool data (an env file, Git data or Studio records)",
blocklisted: "links to a file outside the project in a private location",
received: "links to a file outside the project and came in with an imported project",
};
function walkedFileLeftOut(
absProjectDir: string,
filePath: string,
absPath: string,
received: (path: string) => boolean,
): WalkedLeftOutReason | null {
if (isPrivateCompositionFile(absProjectDir, absPath)) return "private";
if (linksToPrivateOutsideFile(absProjectDir, absPath)) return "blocklisted";
if (received(filePath) && linksOutside(absProjectDir, absPath)) return "received";
return null;
}
/** {@link buildPublishFileMap}, but non-text files of `onDiskMinBytes` or more stay on disk. */
export function stagePublishFiles(
projectDir: string,
onDiskMinBytes: number,
onLeftOut?: OnLeftOut,
): Map<string, StagedFile> {
const absProjectDir = resolve(projectDir);
const filePaths: string[] = [];
collectProjectFiles(absProjectDir, absProjectDir, filePaths, projectIgnore(absProjectDir));
if (!filePaths.includes("index.html")) {
throw new Error(
"Project archive must include index.html at the root. Check that .hyperframesignore does not exclude it.",
);
}
const fileContents = new Map<string, StagedFile>();
const diskPathOf = new Map<string, string>();
const received = receivedFiles(absProjectDir);
for (const filePath of filePaths) {
const absPath = join(absProjectDir, filePath);
const reason = walkedFileLeftOut(absProjectDir, filePath, absPath, received);
if (reason) {
onLeftOut?.(
`${filePath} ${WALKED_LEFT_OUT[reason]}; it was left out of the upload.`,
leftOutReference(absProjectDir, filePath, filePath, absPath, reason),
);
continue;
}
const key = archiveKey(filePath);
const clash = diskPathOf.get(key);
if (clash !== undefined) {
throw new Error(
`"${clash}" and "${filePath}" have the same name written in two Unicode forms and would overwrite each other in the archive. Rename one of them.`,
);
}
diskPathOf.set(key, filePath);
fileContents.set(key, stageFile(absPath, onDiskMinBytes));
}
composeNamesInText(fileContents, filePaths);
localizeExternalAssets(absProjectDir, fileContents, onDiskMinBytes, onLeftOut);
return fileContents;
}
/** Zip an in-memory archive file map (from `buildPublishFileMap`, optionally
* transformed in between, e.g. by proxy baking) into the final archive buffer. */
/**
* Fixed entry timestamp, so the same files always zip to the same bytes.
*
* adm-zip stamps every entry with `new Date()` as it is constructed, and a ZIP
* timestamp has two-second granularity — so archiving identical content twice
* produced different bytes whenever the two runs landed either side of a
* two-second boundary. That makes the archive's digest a function of the clock
* rather than of its contents, which is the opposite of what a
* content-addressed artifact needs.
*
* Built from local components on purpose: adm-zip's DOS date encoding reads `getFullYear`,
* `getMonth`, `getHours` and friends, so a fixed *instant* would still encode
* differently in different timezones. Fixing the wall-clock reading is what
* makes the bytes match across machines. 1980-01-01 is the earliest a DOS
* timestamp can represent.
*/
const ARCHIVE_ENTRY_TIME = new Date(1980, 0, 1, 0, 0, 0, 0);
export function zipPublishFileMap(fileContents: Map<string, Buffer>): PublishArchiveResult {
const archive = new AdmZip();
for (const [filePath, content] of fileContents) {
archive.addFile(filePath, content);
}
for (const entry of archive.getEntries()) {
entry.header.time = ARCHIVE_ENTRY_TIME;
}
return {
buffer: archive.toBuffer(),
fileCount: fileContents.size,
};
}
/**
* Thin composition of `buildPublishFileMap` + `zipPublishFileMap` — signature
* and behavior UNCHANGED from before the U6 split. `cloud render` composes the
* same two functions without an intermediate transform and must stay
* byte-identical (never see baked proxies); only `publish.ts` inserts a baking
* transform between them.
*/
export function createPublishArchive(
projectDir: string,
onLeftOut?: OnLeftOut,
): PublishArchiveResult {
return zipPublishFileMap(buildPublishFileMap(projectDir, onLeftOut));
}
export function getPublishApiBaseUrl(): string {
return (
process.env["HYPERFRAMES_PUBLISHED_PROJECTS_API_URL"] ||
process.env["HEYGEN_API_URL"] ||
"https://api2.heygen.com"
).replace(/\/$/, "");
}
function archiveArrayBuffer(archive: PublishArchiveResult): ArrayBuffer {
const arrayBuffer = new ArrayBuffer(archive.buffer.byteLength);
new Uint8Array(arrayBuffer).set(archive.buffer);
return arrayBuffer;
}
async function publishProjectArchiveDirect(
apiBaseUrl: string,
title: string,
archive: PublishArchiveResult,
isPublic: boolean,
authHeaders: Record<string, string>,
projectId: string | undefined,
): Promise<PublishedProjectResponse> {
const body = new FormData();
body.set("title", title);
if (isPublic) body.set("is_public", "true");
if (projectId) body.set("project_id", projectId);
body.set(
"file",
new File([archiveArrayBuffer(archive)], `${title}.zip`, { type: PUBLISH_CONTENT_TYPE }),
);
const headers: Record<string, string> = { ...authHeaders };
const response = await fetchForPublish(
`${apiBaseUrl}/v1/hyperframes/projects/publish`,
() => ({
method: "POST",
body,
headers,
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
}),
"Failed to publish project",
);
const payload = await readJson(response);
const publishedProject = parsePublishedProjectResponse(payload);
if (!response.ok || !publishedProject) {
throw await metadataRequestError(response, "Failed to publish project");
}
return publishedProject;
}
async function uploadArchiveToPresignedUrl(
stagedUpload: StagedUploadResponse,
archive: PublishArchiveResult,
): Promise<void> {
const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS;
const s3Response = await fetchForPublish(
stagedUpload.uploadUrl,
() => ({
method: "PUT",
body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
headers: stagedUpload.uploadHeaders,
signal: AbortSignal.timeout(
Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
),
}),
"Failed to upload project archive",
PUBLISH_TRANSPORT_ATTEMPTS,
);
if (!s3Response.ok) {
throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive"));
}
}
async function publishProjectArchiveStaged(
apiBaseUrl: string,
title: string,
archive: PublishArchiveResult,
isPublic: boolean,
authHeaders: Record<string, string>,
projectId: string | undefined,
): Promise<PublishedProjectResponse | null> {
const fileName = `${title}.zip`;
const uploadResponse = await fetchForPublish(
`${apiBaseUrl}/v1/hyperframes/projects/publish/upload`,
() => ({
method: "POST",
body: JSON.stringify({
file_name: fileName,
content_type: PUBLISH_CONTENT_TYPE,
content_length: archive.buffer.byteLength,
}),
headers: {
...authHeaders,
"content-type": "application/json",
},
signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
}),
"Failed to prepare project upload",
PUBLISH_TRANSPORT_ATTEMPTS,
);
if (uploadResponse.status === 404 || uploadResponse.status === 405) {
return null;
}
const uploadPayload = await readJson(uploadResponse);
const stagedUpload = parseStagedUploadResponse(uploadPayload, archive.buffer.byteLength);
if (!uploadResponse.ok || !stagedUpload) {
throw await metadataRequestError(uploadResponse, "Failed to prepare project upload");
}
await uploadArchiveToPresignedUrl(stagedUpload, archive);
const completeResponse = await fetchForPublish(
`${apiBaseUrl}/v1/hyperframes/projects/publish/complete`,
() => ({
method: "POST",
body: JSON.stringify({
upload_key: stagedUpload.uploadKey,
file_name: fileName,
title,
...(isPublic ? { is_public: true } : {}),
...(projectId ? { project_id: projectId } : {}),
}),
headers: {
...authHeaders,
"content-type": "application/json",
},
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
}),
"Failed to finalize project publish",
);
const completePayload = await readJson(completeResponse);
const publishedProject = parsePublishedProjectResponse(completePayload);
if (!completeResponse.ok || !publishedProject) {
throw await metadataRequestError(completeResponse, "Failed to publish project");
}
return publishedProject;
}
export interface PublishOptions {
public?: boolean;
/** Stable project id to update in place. Only sent when authenticated. */
projectId?: string;
/** Shared team space id, sent as X-Space-Id so team members converge. Only when authenticated. */
spaceId?: string;
/**
* Pre-built archive to upload instead of building one fresh from
* `projectDir` via `createPublishArchive`. `publish.ts` passes this so it
* can bake proxies into the file map between `buildPublishFileMap` and
* `zipPublishFileMap` (U6); callers that omit it (e.g. `feedback`'s
* minimal-repro publish) keep today's behavior unchanged.
*/
archive?: PublishArchiveResult;
/** The credential the caller already checked; `null` publishes anonymously. Resolved here when omitted. */
credential?: ResolvedCredential | null;
}
export async function publishProjectArchive(
projectDir: string,
opts: PublishOptions = {},
): Promise<PublishedProjectResponse> {
const isPublic = opts.public === true;
const title = basename(projectDir);
const archive = opts.archive ?? createPublishArchive(projectDir);
const apiBaseUrl = getPublishApiBaseUrl();
const credential = await resolvePublishCredential(opts.credential);
const authHeaders = credential ? buildAuthHeaders(credential) : {};
// A stable id / team space only mean something to an authenticated owner — the server
// ignores them otherwise, and anonymous publishes always mint a fresh project.
const projectId = credential ? opts.projectId : undefined;
const spaceId = credential ? opts.spaceId : undefined;
// X-Space-Id rides with the auth headers on the metadata requests only (never the
// presigned S3 PUT), so the server resolves the shared team space instead of the personal one.
const metadataHeaders = spaceId ? { ...authHeaders, "x-space-id": spaceId } : authHeaders;
let result: PublishedProjectResponse;
try {
result =
(await publishProjectArchiveStaged(
apiBaseUrl,
title,
archive,
isPublic,
metadataHeaders,
projectId,
)) ??
(await publishProjectArchiveDirect(
apiBaseUrl,
title,
archive,
isPublic,
metadataHeaders,
projectId,
));
// Only a bearer login owns a publish; the server drops one it cannot verify.
if (credential?.type === "oauth" && !result.claimed) {
throw new CredentialRejectedError("unclaimed publish");
}
} catch (error) {
if (error instanceof CredentialRejectedError) {
throw new Error(credential ? rejectedCredentialMessage(credential) : LOGIN_REQUIRED);
}
throw error;
}
// Remember the server's id + url so the next publish of this directory updates in place.
if (result.claimed) {
writeProjectLink(projectDir, { projectId: result.projectId, url: result.url });
}
return result;
}