diff --git a/packages/core/src/compiler/htmlBundler.test.ts b/packages/core/src/compiler/htmlBundler.test.ts index 74ea21f1a1..2d0005ad58 100644 --- a/packages/core/src/compiler/htmlBundler.test.ts +++ b/packages/core/src/compiler/htmlBundler.test.ts @@ -201,6 +201,45 @@ describe("bundleToSingleHtml", () => { expect(reads).not.toContain(join(dir, "notes.md")); }); + it.each([ + [ + "UTF-16LE", + (text: string) => Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")]), + ], + [ + "UTF-16BE", + (text: string) => + Buffer.concat([Buffer.from([0xfe, 0xff]), Buffer.from(text, "utf16le").swap16()]), + ], + [ + "UTF-8 with a BOM", + (text: string) => Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(text)]), + ], + ])( + "reads a page saved as %s by its byte order mark, as a browser does", + async (_name, encode) => { + const dir = makeTempProject({}); + writeFileSync( + join(dir, "index.html"), + encode(` +
+
+
`), + ); + writeFileSync( + join(dir, "scene.html"), + encode(``), + ); + + const bundled = await bundleToSingleHtml(dir); + + expect(bundled).not.toContain("\u0000"); + expect(bundled).not.toContain("\uFEFF"); + expect(bundled).toContain('data-composition-id="root"'); + expect(bundled).toContain("Café scene"); + }, + ); + it("bundles a direct composition entry with paths relative to its file", async () => { const dir = makeTempProject({ "index.html": "wrong entry", diff --git a/packages/core/src/compiler/htmlBundler.ts b/packages/core/src/compiler/htmlBundler.ts index 22327d4a15..4f6e87b962 100644 --- a/packages/core/src/compiler/htmlBundler.ts +++ b/packages/core/src/compiler/htmlBundler.ts @@ -72,6 +72,7 @@ import { } from "./inlineSubCompositions"; import { isSafePath, resolveWithinProject } from "../safePath.js"; import { ensureHfIds } from "@hyperframes/parsers/hf-ids"; +import { readTextFile } from "@hyperframes/parsers/asset-resolution"; import { HF_COLOR_GRADING_ATTR } from "../colorGrading"; import { assignMediaRenderIds } from "./mediaRenderIds"; @@ -120,7 +121,7 @@ function safeReadFile(filePath: string): string | null { noteRead(filePath); if (!existsSync(filePath)) return null; try { - return readFileSync(filePath, "utf-8"); + return readTextFile(filePath); } catch { return null; } @@ -1069,7 +1070,7 @@ async function bundleProject(projectDir: string, options?: BundleOptions): Promi throw new Error(`${entryFile} not found in project directory`); } noteRead(indexPath); - const entryHtml = readFileSync(indexPath, "utf-8"); + const entryHtml = readTextFile(indexPath); const mountedShell = readMountedSceneShell(projectDir, entryFile, entryHtml); const sourceDir = mountedShell ? projectDir : dirname(indexPath); // Every mount of a sub-composition asks again for each of its url()s; isSafePath realpaths the whole chain. diff --git a/packages/lint/src/project.test.ts b/packages/lint/src/project.test.ts index ac56524e08..505edc6d36 100644 --- a/packages/lint/src/project.test.ts +++ b/packages/lint/src/project.test.ts @@ -1449,3 +1449,23 @@ describe("a media file whose own name holds # or ?", () => { expect(finding?.fixHint).toContain("Add the file"); }); }); + +describe("a page saved as UTF-16", () => { + it("lints as the same page saved as UTF-8", async () => { + const page = ` +
+

Café

+
+ +`; + const utf8 = makeProject(page); + const utf16 = makeProject(page); + writeFileSync( + join(utf16, "index.html"), + Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(page, "utf16le")]), + ); + const codes = async (dir: string) => + (await lintProject(dir)).results.flatMap((r) => r.result.findings.map((f) => f.code)).sort(); + expect(await codes(utf16)).toEqual(await codes(utf8)); + }); +}); diff --git a/packages/lint/src/project.ts b/packages/lint/src/project.ts index 33a1839db5..908f68f92a 100644 --- a/packages/lint/src/project.ts +++ b/packages/lint/src/project.ts @@ -1,6 +1,6 @@ export { shouldBlockRender } from "./shouldBlockRender.js"; import { createHash } from "node:crypto"; -import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { existsSync, readdirSync } from "node:fs"; import { dirname, extname, join, relative, resolve } from "node:path"; import { rewriteAssetPath } from "@hyperframes/parsers/asset-paths"; import { spansOverlap } from "@hyperframes/parsers/time-spans"; @@ -32,6 +32,7 @@ import { resolveExistingLocalAsset, resolveLocalAssetCandidates, resolveProjectRelativeSrc, + readTextFile, } from "@hyperframes/parsers/asset-resolution"; import { encodeUrlPath } from "@hyperframes/parsers"; import { @@ -293,7 +294,7 @@ export async function lintProject( let totalWarnings = 0; let totalInfos = 0; - const rootHtml = readFileSync(indexPath, "utf-8"); + const rootHtml = readTextFile(indexPath); const projectFontFamilies = declaredProjectFontFamilies( projectDir, mountedSources(projectDir, rootHtml, rootCompSrcPath), @@ -333,7 +334,7 @@ export async function lintProject( const files = collectHtmlFiles(compositionsDir, "").sort(); for (const file of files) { const filePath = join(compositionsDir, file); - const html = readFileSync(filePath, "utf-8"); + const html = readTextFile(filePath); const compSrcPath = `compositions/${file}`; allHtmlSources.push({ html, compSrcPath }); // Mountable fragments (figma component imports, registry snippets) are @@ -740,7 +741,7 @@ function lintMultipleRootCompositions(projectDir: string): HyperframeLintFinding const rootCompositions: string[] = []; for (const file of rootHtmlFiles) { if (PAGES_OPENED_BESIDE_INDEX.has(file)) continue; - const content = readFileSync(join(projectDir, file), "utf-8"); + const content = readTextFile(join(projectDir, file)); if (/data-composition-id/i.test(content)) { rootCompositions.push(file); } diff --git a/packages/parsers/src/assetResolution.test.ts b/packages/parsers/src/assetResolution.test.ts index 8499bb0cb5..0f1f8a8f3d 100644 --- a/packages/parsers/src/assetResolution.test.ts +++ b/packages/parsers/src/assetResolution.test.ts @@ -9,6 +9,7 @@ import { isWithinProjectRoot, maskNonScannableRanges, readProjectFile, + readTextFile, resolveProjectRelativeSrc, } from "./assetResolution.js"; @@ -317,3 +318,21 @@ describe("readProjectFile", () => { expect(readProjectFile(join(dir, "pipe.html"))).toEqual({ kind: "folder" }); }); }); + +describe("readTextFile and readProjectFile read a byte order mark as browsers do", () => { + const text = "

Café

"; + const saved: Array<[string, Buffer]> = [ + ["UTF-16LE", Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")])], + ["UTF-16BE", Buffer.concat([Buffer.from([0xfe, 0xff]), Buffer.from(text, "utf16le").swap16()])], + ["UTF-8 with a mark", Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(text)])], + ["UTF-8", Buffer.from(text)], + ]; + it.each(saved)("%s", (_name, bytes) => { + const dir = mkdtempSync(join(tmpdir(), "hf-text-")); + const path = join(dir, "page.html"); + writeFileSync(path, bytes); + expect(readTextFile(path)).toBe(text); + expect(readProjectFile(path)).toEqual({ kind: "file", text }); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/packages/parsers/src/assetResolution.ts b/packages/parsers/src/assetResolution.ts index 0a6bc25774..144af5041b 100644 --- a/packages/parsers/src/assetResolution.ts +++ b/packages/parsers/src/assetResolution.ts @@ -223,6 +223,18 @@ export type ProjectFileRead = | { kind: "folder" } | { kind: "missing" }; +// A byte order mark names the encoding, as browsers read it: PowerShell 5.1 saves files as UTF-16LE with one. +export function decodeTextHonoringBom(bytes: Buffer): string { + if (bytes[0] === 0xff && bytes[1] === 0xfe) return bytes.subarray(2).toString("utf16le"); + if (bytes[0] === 0xfe && bytes[1] === 0xff) + return Buffer.from(bytes.subarray(2, bytes.length - (bytes.length % 2))) + .swap16() + .toString("utf16le"); + return bytes.toString("utf-8").replace(/^\uFEFF/, ""); +} + +export const readTextFile = (path: string): string => decodeTextHonoringBom(readFileSync(path)); + /** Reads through one descriptor, so the file-type check and the read see the same file. */ export function readProjectFile(path: string): ProjectFileRead { let fd: number; @@ -237,7 +249,7 @@ export function readProjectFile(path: string): ProjectFileRead { } try { if (!fstatSync(fd).isFile()) return { kind: "folder" }; - return { kind: "file", text: readFileSync(fd, "utf-8") }; + return { kind: "file", text: decodeTextHonoringBom(readFileSync(fd)) }; } finally { closeSync(fd); } diff --git a/packages/producer/src/services/htmlCompiler.encoding.test.ts b/packages/producer/src/services/htmlCompiler.encoding.test.ts new file mode 100644 index 0000000000..6c3a41dbce --- /dev/null +++ b/packages/producer/src/services/htmlCompiler.encoding.test.ts @@ -0,0 +1,37 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { compileForRender } from "./htmlCompiler.js"; + +const utf16le = (text: string) => + Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")]); + +describe("compileForRender a page saved as UTF-16LE", () => { + it("compiles its text, not NUL bytes, with its sub-composition", async () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-compiler-utf16-")); + writeFileSync( + join(projectDir, "index.html"), + utf16le(` + +
+
+ +
+`), + ); + writeFileSync( + join(projectDir, "scene.html"), + utf16le(``), + ); + + const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir); + + expect(compiled.html).not.toContain("\u0000"); + expect(compiled.html).toContain('data-composition-id="root"'); + expect(compiled.html).toContain("Café scene"); + }); +}); diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index 54a32d482d..e6991ad708 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -12,7 +12,7 @@ import { decodedUrlPath } from "@hyperframes/parsers/asset-paths"; */ import { createHash } from "node:crypto"; -import { createReadStream, existsSync, mkdirSync, readFileSync } from "fs"; +import { createReadStream, existsSync, mkdirSync } from "fs"; import { join, dirname, resolve, basename, relative } from "path"; import { parseHTML } from "linkedom"; import { @@ -71,6 +71,7 @@ import { import { isUnresolvedAssetPlaceholder, readProjectFile, + readTextFile, } from "@hyperframes/parsers/asset-resolution"; import { extractMediaMetadata, extractAudioMetadata } from "../utils/ffprobe.js"; import { isPathInside, toExternalAssetKey } from "../utils/paths.js"; @@ -2038,11 +2039,7 @@ export async function compileForRender( downloadDir: string, options: CompileForRenderOptions = {}, ): Promise { - const entryHtml = rebaseDirectEntryAssetPaths( - readFileSync(htmlPath, "utf-8"), - projectDir, - htmlPath, - ); + const entryHtml = rebaseDirectEntryAssetPaths(readTextFile(htmlPath), projectDir, htmlPath); const rawHtml = rewriteUnresolvableGsapToCdn(entryHtml, projectDir); // Pre-flight: every data-composition-src reference must resolve to a diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index b4bee80f17..cdd6da9c11 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -35,7 +35,6 @@ import { audioFailureDiagnostics } from "@hyperframes/engine"; import { existsSync, mkdirSync, - readFileSync, readdirSync, rmSync, statSync, @@ -60,6 +59,7 @@ import { import { HF_AUDIO_GROUP_TAG } from "@hyperframes/core/audio-groups"; import { extractStandaloneEntryFromIndex, isTemplateEntry } from "@hyperframes/core/compiler"; import { HTML_BODY_CSS_HEIGHT_FIRST_RE, HTML_BODY_CSS_WIDTH_FIRST_RE } from "@hyperframes/parsers"; +import { readTextFile } from "@hyperframes/parsers/asset-resolution"; import { type EngineConfig, extractMediaMetadata, @@ -3213,7 +3213,7 @@ async function executeRenderPipeline(input: { // If entryFile is a sub-composition (