fix(core): a page saved as UTF-16 bundles as its text, not NUL bytes (#6580)

* fix(core): a page saved as UTF-16 bundles as its text, not NUL bytes

* fix(core): render, lint and the asset reader decode UTF-16 pages too

packages/core/src/compiler/htmlBundler.test.ts

@@ -201,6 +201,45 @@ describe("bundleToSingleHtml", () => {
     expect(reads).not.toContain(join(dir, "notes.md"));
   });
 
+  it.each([
+    [
+      "UTF-16LE",
+      (text: string) => Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")]),
+    ],
+    [
+      "UTF-16BE",
+      (text: string) =>
+        Buffer.concat([Buffer.from([0xfe, 0xff]), Buffer.from(text, "utf16le").swap16()]),
+    ],
+    [
+      "UTF-8 with a BOM",
+      (text: string) => Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(text)]),
+    ],
+  ])(
+    "reads a page saved as %s by its byte order mark, as a browser does",
+    async (_name, encode) => {
+      const dir = makeTempProject({});
+      writeFileSync(
+        join(dir, "index.html"),
+        encode(`<!doctype html><html><body>
+        <div data-composition-id="root" data-width="320" data-height="180" data-duration="2">
+          <div data-composition-id="scene" data-composition-src="scene.html" data-start="0" data-duration="2"></div>
+        </div></body></html>`),
+      );
+      writeFileSync(
+        join(dir, "scene.html"),
+        encode(`<template><div data-composition-id="scene"><h1>Café scene</h1></div></template>`),
+      );
+
+      const bundled = await bundleToSingleHtml(dir);
+
+      expect(bundled).not.toContain("\u0000");
+      expect(bundled).not.toContain("\uFEFF");
+      expect(bundled).toContain('data-composition-id="root"');
+      expect(bundled).toContain("Café scene");
+    },
+  );
+
   it("bundles a direct composition entry with paths relative to its file", async () => {
     const dir = makeTempProject({
       "index.html": "<html><body>wrong entry</body></html>",

packages/core/src/compiler/htmlBundler.ts

@@ -72,6 +72,7 @@ import {
 } from "./inlineSubCompositions";
 import { isSafePath, resolveWithinProject } from "../safePath.js";
 import { ensureHfIds } from "@hyperframes/parsers/hf-ids";
+import { readTextFile } from "@hyperframes/parsers/asset-resolution";
 import { HF_COLOR_GRADING_ATTR } from "../colorGrading";
 import { assignMediaRenderIds } from "./mediaRenderIds";
 
@@ -120,7 +121,7 @@ function safeReadFile(filePath: string): string | null {
   noteRead(filePath);
   if (!existsSync(filePath)) return null;
   try {
-    return readFileSync(filePath, "utf-8");
+    return readTextFile(filePath);
   } catch {
     return null;
   }
@@ -1069,7 +1070,7 @@ async function bundleProject(projectDir: string, options?: BundleOptions): Promi
     throw new Error(`${entryFile} not found in project directory`);
   }
   noteRead(indexPath);
-  const entryHtml = readFileSync(indexPath, "utf-8");
+  const entryHtml = readTextFile(indexPath);
   const mountedShell = readMountedSceneShell(projectDir, entryFile, entryHtml);
   const sourceDir = mountedShell ? projectDir : dirname(indexPath);
   // Every mount of a sub-composition asks again for each of its url()s; isSafePath realpaths the whole chain.

packages/lint/src/project.test.ts

@@ -1449,3 +1449,23 @@ describe("a media file whose own name holds # or ?", () => {
     expect(finding?.fixHint).toContain("Add the file");
   });
 });
+
+describe("a page saved as UTF-16", () => {
+  it("lints as the same page saved as UTF-8", async () => {
+    const page = `<!doctype html><html><body>
+  <div data-composition-id="root" data-width="1920" data-height="1080" data-start="0" data-duration="5">
+    <h1>Café</h1>
+  </div>
+  <script>window.__timelines["root"] = gsap.timeline({ paused: true });</script>
+</body></html>`;
+    const utf8 = makeProject(page);
+    const utf16 = makeProject(page);
+    writeFileSync(
+      join(utf16, "index.html"),
+      Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(page, "utf16le")]),
+    );
+    const codes = async (dir: string) =>
+      (await lintProject(dir)).results.flatMap((r) => r.result.findings.map((f) => f.code)).sort();
+    expect(await codes(utf16)).toEqual(await codes(utf8));
+  });
+});

packages/lint/src/project.ts

@@ -1,6 +1,6 @@
 export { shouldBlockRender } from "./shouldBlockRender.js";
 import { createHash } from "node:crypto";
-import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { existsSync, readdirSync } from "node:fs";
 import { dirname, extname, join, relative, resolve } from "node:path";
 import { rewriteAssetPath } from "@hyperframes/parsers/asset-paths";
 import { spansOverlap } from "@hyperframes/parsers/time-spans";
@@ -32,6 +32,7 @@ import {
   resolveExistingLocalAsset,
   resolveLocalAssetCandidates,
   resolveProjectRelativeSrc,
+  readTextFile,
 } from "@hyperframes/parsers/asset-resolution";
 import { encodeUrlPath } from "@hyperframes/parsers";
 import {
@@ -293,7 +294,7 @@ export async function lintProject(
   let totalWarnings = 0;
   let totalInfos = 0;
 
-  const rootHtml = readFileSync(indexPath, "utf-8");
+  const rootHtml = readTextFile(indexPath);
   const projectFontFamilies = declaredProjectFontFamilies(
     projectDir,
     mountedSources(projectDir, rootHtml, rootCompSrcPath),
@@ -333,7 +334,7 @@ export async function lintProject(
     const files = collectHtmlFiles(compositionsDir, "").sort();
     for (const file of files) {
       const filePath = join(compositionsDir, file);
-      const html = readFileSync(filePath, "utf-8");
+      const html = readTextFile(filePath);
       const compSrcPath = `compositions/${file}`;
       allHtmlSources.push({ html, compSrcPath });
       // Mountable fragments (figma component imports, registry snippets) are
@@ -740,7 +741,7 @@ function lintMultipleRootCompositions(projectDir: string): HyperframeLintFinding
     const rootCompositions: string[] = [];
     for (const file of rootHtmlFiles) {
       if (PAGES_OPENED_BESIDE_INDEX.has(file)) continue;
-      const content = readFileSync(join(projectDir, file), "utf-8");
+      const content = readTextFile(join(projectDir, file));
       if (/data-composition-id/i.test(content)) {
         rootCompositions.push(file);
       }

packages/parsers/src/assetResolution.test.ts

@@ -9,6 +9,7 @@ import {
   isWithinProjectRoot,
   maskNonScannableRanges,
   readProjectFile,
+  readTextFile,
   resolveProjectRelativeSrc,
 } from "./assetResolution.js";
 
@@ -317,3 +318,21 @@ describe("readProjectFile", () => {
     expect(readProjectFile(join(dir, "pipe.html"))).toEqual({ kind: "folder" });
   });
 });
+
+describe("readTextFile and readProjectFile read a byte order mark as browsers do", () => {
+  const text = "<h1>Café</h1>";
+  const saved: Array<[string, Buffer]> = [
+    ["UTF-16LE", Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")])],
+    ["UTF-16BE", Buffer.concat([Buffer.from([0xfe, 0xff]), Buffer.from(text, "utf16le").swap16()])],
+    ["UTF-8 with a mark", Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(text)])],
+    ["UTF-8", Buffer.from(text)],
+  ];
+  it.each(saved)("%s", (_name, bytes) => {
+    const dir = mkdtempSync(join(tmpdir(), "hf-text-"));
+    const path = join(dir, "page.html");
+    writeFileSync(path, bytes);
+    expect(readTextFile(path)).toBe(text);
+    expect(readProjectFile(path)).toEqual({ kind: "file", text });
+    rmSync(dir, { recursive: true, force: true });
+  });
+});

packages/parsers/src/assetResolution.ts

@@ -223,6 +223,18 @@ export type ProjectFileRead =
   | { kind: "folder" }
   | { kind: "missing" };
 
+// A byte order mark names the encoding, as browsers read it: PowerShell 5.1 saves files as UTF-16LE with one.
+export function decodeTextHonoringBom(bytes: Buffer): string {
+  if (bytes[0] === 0xff && bytes[1] === 0xfe) return bytes.subarray(2).toString("utf16le");
+  if (bytes[0] === 0xfe && bytes[1] === 0xff)
+    return Buffer.from(bytes.subarray(2, bytes.length - (bytes.length % 2)))
+      .swap16()
+      .toString("utf16le");
+  return bytes.toString("utf-8").replace(/^\uFEFF/, "");
+}
+
+export const readTextFile = (path: string): string => decodeTextHonoringBom(readFileSync(path));
+
 /** Reads through one descriptor, so the file-type check and the read see the same file. */
 export function readProjectFile(path: string): ProjectFileRead {
   let fd: number;
@@ -237,7 +249,7 @@ export function readProjectFile(path: string): ProjectFileRead {
   }
   try {
     if (!fstatSync(fd).isFile()) return { kind: "folder" };
-    return { kind: "file", text: readFileSync(fd, "utf-8") };
+    return { kind: "file", text: decodeTextHonoringBom(readFileSync(fd)) };
   } finally {
     closeSync(fd);
   }

packages/producer/src/services/htmlCompiler.encoding.test.ts

@@ -0,0 +1,37 @@
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { compileForRender } from "./htmlCompiler.js";
+
+const utf16le = (text: string) =>
+  Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")]);
+
+describe("compileForRender a page saved as UTF-16LE", () => {
+  it("compiles its text, not NUL bytes, with its sub-composition", async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), "hf-compiler-utf16-"));
+    writeFileSync(
+      join(projectDir, "index.html"),
+      utf16le(`<!doctype html>
+<html><body>
+  <div data-composition-id="root" data-width="1920" data-height="1080" data-duration="2">
+    <div data-composition-id="scene" data-composition-src="scene.html" data-start="0" data-duration="2"></div>
+    <script>
+      window.__timelines = window.__timelines || {};
+      window.__timelines.root = { duration: function() { return 2; } };
+    </script>
+  </div>
+</body></html>`),
+    );
+    writeFileSync(
+      join(projectDir, "scene.html"),
+      utf16le(`<template><div data-composition-id="scene"><h1>Café scene</h1></div></template>`),
+    );
+
+    const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
+
+    expect(compiled.html).not.toContain("\u0000");
+    expect(compiled.html).toContain('data-composition-id="root"');
+    expect(compiled.html).toContain("Café scene");
+  });
+});

packages/producer/src/services/htmlCompiler.ts

@@ -12,7 +12,7 @@ import { decodedUrlPath } from "@hyperframes/parsers/asset-paths";
  */
 
 import { createHash } from "node:crypto";
-import { createReadStream, existsSync, mkdirSync, readFileSync } from "fs";
+import { createReadStream, existsSync, mkdirSync } from "fs";
 import { join, dirname, resolve, basename, relative } from "path";
 import { parseHTML } from "linkedom";
 import {
@@ -71,6 +71,7 @@ import {
 import {
   isUnresolvedAssetPlaceholder,
   readProjectFile,
+  readTextFile,
 } from "@hyperframes/parsers/asset-resolution";
 import { extractMediaMetadata, extractAudioMetadata } from "../utils/ffprobe.js";
 import { isPathInside, toExternalAssetKey } from "../utils/paths.js";
@@ -2038,11 +2039,7 @@ export async function compileForRender(
   downloadDir: string,
   options: CompileForRenderOptions = {},
 ): Promise<CompiledComposition> {
-  const entryHtml = rebaseDirectEntryAssetPaths(
-    readFileSync(htmlPath, "utf-8"),
-    projectDir,
-    htmlPath,
-  );
+  const entryHtml = rebaseDirectEntryAssetPaths(readTextFile(htmlPath), projectDir, htmlPath);
   const rawHtml = rewriteUnresolvableGsapToCdn(entryHtml, projectDir);
 
   // Pre-flight: every data-composition-src reference must resolve to a

packages/producer/src/services/renderOrchestrator.ts

@@ -35,7 +35,6 @@ import { audioFailureDiagnostics } from "@hyperframes/engine";
 import {
   existsSync,
   mkdirSync,
-  readFileSync,
   readdirSync,
   rmSync,
   statSync,
@@ -60,6 +59,7 @@ import {
 import { HF_AUDIO_GROUP_TAG } from "@hyperframes/core/audio-groups";
 import { extractStandaloneEntryFromIndex, isTemplateEntry } from "@hyperframes/core/compiler";
 import { HTML_BODY_CSS_HEIGHT_FIRST_RE, HTML_BODY_CSS_WIDTH_FIRST_RE } from "@hyperframes/parsers";
+import { readTextFile } from "@hyperframes/parsers/asset-resolution";
 import {
   type EngineConfig,
   extractMediaMetadata,
@@ -3213,7 +3213,7 @@ async function executeRenderPipeline(input: {
     // If entryFile is a sub-composition (<template> wrapper), reuse the real
     // index.html shell and isolate the matching host instead of fabricating
     // a new standalone document.
-    const rawEntry = readFileSync(htmlPath, "utf-8");
+    const rawEntry = readTextFile(htmlPath);
     if (entryFile !== "index.html" && isTemplateEntry(rawEntry)) {
       const wrapperPath = join(workDir, "standalone-entry.html");
       const projectIndexPath = join(projectDir, "index.html");
@@ -3223,7 +3223,7 @@ async function executeRenderPipeline(input: {
         );
       }
       const standaloneHtml = extractStandaloneEntryFromIndex(
-        readFileSync(projectIndexPath, "utf-8"),
+        readTextFile(projectIndexPath),
         entryFile,
         rawEntry,
       );