diff --git a/skills-manifest.json b/skills-manifest.json index 424d0fd46..8dda3e5a5 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -2,7 +2,7 @@ "source": "heygen-com/hyperframes", "skills": { "embedded-captions": { - "hash": "14e79d53c24e3945", + "hash": "1c02f679a329bce7", "files": 138 }, "faceless-explainer": { @@ -38,7 +38,7 @@ "files": 19 }, "hyperframes-creative": { - "hash": "bacb5205ea5eb3d8", + "hash": "0068574ecd376e60", "files": 78 }, "hyperframes-keyframes": { @@ -50,7 +50,7 @@ "files": 12 }, "media-use": { - "hash": "7e329ade41b1c1ba", + "hash": "1b0ce647f5c7df95", "files": 152 }, "motion-graphics": { @@ -58,7 +58,7 @@ "files": 23 }, "music-to-video": { - "hash": "55a2b5fdcd6f892c", + "hash": "0c04f6ed00e90077", "files": 132 }, "pr-to-video": { @@ -70,7 +70,7 @@ "files": 28 }, "remotion-to-hyperframes": { - "hash": "3ecc684432b298dd", + "hash": "bf184a65059b95e8", "files": 70 }, "slideshow": { diff --git a/skills/embedded-captions/scripts/gen-stroke-path.py b/skills/embedded-captions/scripts/gen-stroke-path.py index 45dc1d862..b53d6ddf2 100644 --- a/skills/embedded-captions/scripts/gen-stroke-path.py +++ b/skills/embedded-captions/scripts/gen-stroke-path.py @@ -10,10 +10,21 @@ Prints: the path `d` string + layout info on stderr. """ import re, sys +# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on +# every platform; say so rather than depending on the console's code page. Carry +# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives +# stderr "backslashreplace" so the diagnostic path can never itself raise. +for _stream in (sys.stdout, sys.stderr): + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors=_stream.errors) + font_path, text, target_w, baseline_y, x0 = ( sys.argv[1], sys.argv[2], float(sys.argv[3]), float(sys.argv[4]), float(sys.argv[5])) -svg = open(font_path).read() +# SVG fonts are UTF-8 and each glyph's `unicode="…"` attribute IS a literal character. +# Decoded with the platform default, a non-ASCII glyph key silently becomes the wrong +# character (or raises) on Windows, so that glyph never matches the requested text. +svg = open(font_path, encoding="utf-8").read() glyphs = {} for m in re.finditer(r']*?horiz-adv-x="([\d.]+)"(?:[^>]*?d="([^"]*)")?', svg): ch, adv, d = m.group(1), float(m.group(2)), m.group(3) or "" diff --git a/skills/hyperframes-creative/scripts/extract-audio-data.py b/skills/hyperframes-creative/scripts/extract-audio-data.py index b4efba788..b76ad9959 100644 --- a/skills/hyperframes-creative/scripts/extract-audio-data.py +++ b/skills/hyperframes-creative/scripts/extract-audio-data.py @@ -22,6 +22,14 @@ import sys import numpy as np +# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on +# every platform; say so rather than depending on the console's code page. Carry +# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives +# stderr "backslashreplace" so the diagnostic path can never itself raise. +for _stream in (sys.stdout, sys.stderr): + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors=_stream.errors) + # --------------------------------------------------------------------------- # FFT parameters # @@ -51,7 +59,10 @@ def decode_audio(path: str) -> np.ndarray: ] result = subprocess.run(cmd, capture_output=True) if result.returncode != 0: - print(f"ffmpeg error: {result.stderr.decode()}", file=sys.stderr) + # Decoding ffmpeg's diagnostics strictly makes the reporter the thing that + # crashes: a Windows ffmpeg emits cp1252 bytes, and UnicodeDecodeError here + # would bury the actual failure it was trying to report. + print(f"ffmpeg error: {result.stderr.decode('utf-8', errors='replace')}", file=sys.stderr) sys.exit(1) return np.frombuffer(result.stdout, dtype=np.int16).astype(np.float32) / 32768.0 @@ -178,7 +189,7 @@ def main(): data = extract(args.input, args.fps, args.bands) - with open(args.output, "w") as f: + with open(args.output, "w", encoding="utf-8") as f: json.dump(data, f) print(f"Wrote {args.output} ({data['totalFrames']} frames, {data['bands']} bands)", file=sys.stderr) diff --git a/skills/media-use/audio/scripts/lyria-recipe.py b/skills/media-use/audio/scripts/lyria-recipe.py index c28cba8f7..5fdf7d3df 100644 --- a/skills/media-use/audio/scripts/lyria-recipe.py +++ b/skills/media-use/audio/scripts/lyria-recipe.py @@ -20,6 +20,14 @@ import sys import wave from pathlib import Path +# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on +# every platform; say so rather than depending on the console's code page. Carry +# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives +# stderr "backslashreplace" so the diagnostic path can never itself raise. +for _stream in (sys.stdout, sys.stderr): + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors=_stream.errors) + DEFAULT_PROMPT = "Uplifting corporate tech, bright and modern, gentle piano with synth pads" SAMPLE_RATE = 48000 CHANNELS = 2 diff --git a/skills/music-to-video/scripts/analyze-beatgrid.py b/skills/music-to-video/scripts/analyze-beatgrid.py index 676e250f1..9f389409e 100644 --- a/skills/music-to-video/scripts/analyze-beatgrid.py +++ b/skills/music-to-video/scripts/analyze-beatgrid.py @@ -35,6 +35,15 @@ import librosa import numpy as np import soundfile as sf +# Windows sizes stdio to the ANSI code page (cp1252), which cannot encode the glyphs +# the brief prints (Δ, →) — every `--print` run died with UnicodeEncodeError. These +# scripts emit UTF-8 on every platform; say so instead of trading the glyphs away. +# Carry `errors` across: reconfigure() resets it to "strict", and CPython deliberately +# gives stderr "backslashreplace" so the diagnostic path can never itself raise. +for _stream in (sys.stdout, sys.stderr): + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors=_stream.errors) + SR = 22050 HOP = 512 # ~23 ms frames AUDIOMAP_VERSION = 2 @@ -517,7 +526,9 @@ def main() -> None: a = ap.parse_args() d = analyze(a.audio, phrase_bars=a.phrase_bars) if a.out: - Path(a.out).write_text(json.dumps(d, ensure_ascii=False, indent=2)) + # ensure_ascii=False means the payload can carry non-ASCII, so the file + # encoding cannot be left to the platform default (cp1252 on Windows). + Path(a.out).write_text(json.dumps(d, ensure_ascii=False, indent=2), encoding="utf-8") dens = " ".join(f"{s.get('level', '?')}:{s.get('density', '?')}" for s in d.get("energy_phases", [])) print( f"[analyze-beatgrid] wrote audiomap {a.out} · {len(d.get('energy_phases', []))} phases · density [{dens}]", diff --git a/skills/python-encoding.test.mjs b/skills/python-encoding.test.mjs new file mode 100644 index 000000000..5564fba8d --- /dev/null +++ b/skills/python-encoding.test.mjs @@ -0,0 +1,109 @@ +// Skill Python scripts run on the user's own machine, Windows included. There, Python +// sizes stdio and text-mode file IO to the ANSI code page (cp1252) rather than UTF-8: +// +// * printing a glyph cp1252 has no slot for (Δ, →) raises UnicodeEncodeError, which +// is how `analyze-beatgrid.py --print` died on every Windows run; +// * reading a UTF-8 source raises UnicodeDecodeError, or worse, decodes each byte to +// the wrong character and the script silently keys off it. +// +// So every skill Python script pins UTF-8 explicitly. This test is the guard: the class +// of bug returns the moment one file IO call drops `encoding=` or a new script ships +// without the stdio block. Repo-internal dev scripts (packages/**) are out of scope — +// they only ever run on CI and maintainer machines. +import assert from "node:assert/strict"; +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, it } from "node:test"; + +const SKILLS_DIR = dirname(fileURLToPath(import.meta.url)); + +function pythonScripts(dir) { + const found = []; + for (const entry of readdirSync(dir)) { + const abs = join(dir, entry); + if (statSync(abs).isDirectory()) found.push(...pythonScripts(abs)); + else if (entry.endsWith(".py")) found.push(abs); + } + return found; +} + +const scripts = pythonScripts(SKILLS_DIR).map((abs) => ({ + rel: relative(SKILLS_DIR, abs), + source: readFileSync(abs, "utf8"), +})); + +describe("skill Python scripts pin UTF-8", () => { + it("finds the scripts (guards against a layout change silently emptying this suite)", () => { + assert.ok(scripts.length >= 5, `expected >=5 skill Python scripts, found ${scripts.length}`); + }); + + for (const { rel, source } of scripts) { + it(`${rel} reconfigures stdio to UTF-8, preserving the errors handler`, () => { + // `errors=` is not optional. reconfigure() resets the handler to "strict", and + // CPython gives stderr "backslashreplace" on purpose so the diagnostic path can + // never itself raise — dropping it moves the crash onto error reporting. + assert.match( + source, + /reconfigure\(encoding="utf-8", errors=_stream\.errors\)/, + 'add the `for _stream in (sys.stdout, sys.stderr): ... reconfigure(encoding="utf-8", errors=_stream.errors)` block', + ); + }); + + it(`${rel} passes encoding= to every text-mode file IO call`, () => { + const offenders = fileIoCalls(source) + .filter((call) => !isBinary(call)) + .filter((call) => !/\bencoding\s*=/.test(call.text)) + .map((call) => call.text); + assert.deepEqual(offenders, [], `text IO without encoding= in ${rel}`); + }); + } +}); + +/** + * Every `open(...)` / `read_text(...)` / `write_text(...)` call in the source, paired with + * its argument text. Scanning to the balanced close paren beats a regex here: a regex has + * to cap nesting depth, and a call it fails to match is a call it silently exempts — the + * opposite of what a guard is for. + */ +function fileIoCalls(source) { + const calls = []; + const opener = /\b(open|read_text|write_text)\(/g; + for (let m = opener.exec(source); m; m = opener.exec(source)) { + const end = closingParen(source, m.index + m[0].length - 1); + if (end < 0) continue; // unbalanced source; nothing to assert + calls.push({ + name: m[1], + args: source.slice(m.index + m[0].length, end), + text: source.slice(m.index, end + 1), + }); + } + return calls; +} + +/** + * Index of the `)` closing the `(` at `start`, or -1 if the source is unbalanced. + * Counting with arithmetic rather than branches keeps this at the size it deserves: + * `start` is the opening paren, so depth returns to 0 exactly at its partner. + */ +function closingParen(source, start) { + let depth = 0; + for (let i = start; i < source.length; i++) { + depth += Number(source[i] === "(") - Number(source[i] === ")"); + if (depth === 0) return i; + } + return -1; +} + +// A binary mode is a WHOLE argument (comma-delimited) made only of mode characters, one of +// which is `b`. Both halves matter. Testing the whole call text for any quoted "b" let a +// payload key spell the check away — `write_text(json.dumps({"bpm": 120}))` exempted +// itself, and "bpm"/"bars" are literally analyze-beatgrid's own keys. Restricting to mode +// characters rules that out without having to split arguments: "bpm" holds `p` and `m`, +// which no mode does. +const BINARY_MODE = /(^|,)\s*(["'])[rwxa+t]*b[rwxa+t]*\2\s*(,|$)/; + +/** True when no encoding applies. `Path.read_text`/`write_text` have no mode: always text. */ +function isBinary(call) { + return call.name === "open" && BINARY_MODE.test(call.args); +} diff --git a/skills/remotion-to-hyperframes/scripts/lint_source.py b/skills/remotion-to-hyperframes/scripts/lint_source.py index e3a8ecd5c..5b72eec0c 100755 --- a/skills/remotion-to-hyperframes/scripts/lint_source.py +++ b/skills/remotion-to-hyperframes/scripts/lint_source.py @@ -46,6 +46,16 @@ from dataclasses import dataclass, asdict from pathlib import Path from typing import Callable +# Windows sizes stdio to the ANSI code page (cp1252), which cannot encode every glyph +# a finding message carries. These scripts emit UTF-8 on every platform; say so. Carry +# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives +# stderr "backslashreplace" so the diagnostic path can never itself raise — this script +# prints scanned filenames, and an unpaired surrogate in one would otherwise crash the +# reporter instead of being escaped. +for _stream in (sys.stdout, sys.stderr): + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors=_stream.errors) + BLOCKER = "blocker" WARNING = "warning" INFO = "info" @@ -272,7 +282,9 @@ def _find_matching_paren(src: str, open_idx: int) -> int | None: def lint_file(path: Path) -> list[Finding]: - src = path.read_text() + # Remotion sources are UTF-8. Left to the platform default, a source carrying an + # em dash or a curly quote raises UnicodeDecodeError on Windows before any rule runs. + src = path.read_text(encoding="utf-8") findings: list[Finding] = [] def loc(offset: int) -> tuple[int, int]: