From d36c1785b92989cec8e0b1277a09c50c6ba628ca Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 2 Apr 2026 00:47:43 -0700 Subject: [PATCH] feat(captions): energy-based technique selection and mandatory quality checks (#176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Rewrite script-to-style mapping as an energy detection table (high → low) with mandatory animation requirements: karaoke baseline, 2+ highlight techniques, kinetic exits - Replace `tl.call()` per-frame audio-reactive pattern with group-level GSAP tweens — read peak bass/treble for each group's time range and modulate entrance intensity at build time, no per-frame callbacks needed - Add transcript quality check with automatic retry rules (>20% music tokens = retry with larger model) - Add caption word structure lint rule (`.caption-group` + ``) for studio editor compatibility - Add multilingual model guidance and decision tree for model selection ## Test plan - [ ] Skill files render correctly as markdown - [ ] Cross-references between SKILL.md, dynamic-techniques.md, and transcript-guide.md resolve correctly - [ ] `dynamic-techniques.md` audio-reactive section uses `tl.to()`/`tl.set()` only, no `tl.call()` loops 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- CLAUDE.md | 14 +- lefthook.yml | 3 + packages/cli/src/cli.ts | 1 + packages/cli/src/commands/validate.ts | 247 ++++++++++ packages/cli/src/whisper/normalize.ts | 19 +- packages/core/src/runtime/captionOverrides.ts | 132 +++++ packages/core/src/runtime/init.ts | 5 + packages/studio/src/App.tsx | 221 ++++++++- .../components/CaptionAnimationPanel.tsx | 300 ++++++++++++ .../captions/components/CaptionOverlay.tsx | 462 ++++++++++++++++++ .../components/CaptionPropertyPanel.tsx | 294 +++++++++++ .../captions/components/CaptionTimeline.tsx | 187 +++++++ .../studio/src/captions/generator.test.ts | 279 +++++++++++ packages/studio/src/captions/generator.ts | 372 ++++++++++++++ .../src/captions/hooks/useCaptionSync.ts | 163 ++++++ packages/studio/src/captions/index.ts | 10 + packages/studio/src/captions/parser.test.ts | 377 ++++++++++++++ packages/studio/src/captions/parser.ts | 312 ++++++++++++ packages/studio/src/captions/store.ts | 270 ++++++++++ packages/studio/src/captions/types.ts | 207 ++++++++ packages/studio/vite.config.ts | 18 +- skills/hyperframes-compose/SKILL.md | 2 + 22 files changed, 3881 insertions(+), 14 deletions(-) create mode 100644 packages/cli/src/commands/validate.ts create mode 100644 packages/core/src/runtime/captionOverrides.ts create mode 100644 packages/studio/src/captions/components/CaptionAnimationPanel.tsx create mode 100644 packages/studio/src/captions/components/CaptionOverlay.tsx create mode 100644 packages/studio/src/captions/components/CaptionPropertyPanel.tsx create mode 100644 packages/studio/src/captions/components/CaptionTimeline.tsx create mode 100644 packages/studio/src/captions/generator.test.ts create mode 100644 packages/studio/src/captions/generator.ts create mode 100644 packages/studio/src/captions/hooks/useCaptionSync.ts create mode 100644 packages/studio/src/captions/index.ts create mode 100644 packages/studio/src/captions/parser.test.ts create mode 100644 packages/studio/src/captions/parser.ts create mode 100644 packages/studio/src/captions/store.ts create mode 100644 packages/studio/src/captions/types.ts diff --git a/CLAUDE.md b/CLAUDE.md index f5f607efe..6ddf7412c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,7 +34,7 @@ The skills encode HyperFrames-specific patterns (e.g., required `class="clip"` o - When creating a video from audio (music video, lyric video, audio visualizer with text) → invoke BOTH `/hyperframes-compose` AND `/hyperframes-captions` - When writing GSAP animations → invoke `/gsap-core` and `/gsap-timeline` BEFORE writing any code - When optimizing animation performance → invoke `/gsap-performance` BEFORE making changes -- After creating or editing any `.html` composition → run `npx hyperframes lint` and fix all errors before considering the task complete +- After creating or editing any `.html` composition → run `npx hyperframes lint` and `npx hyperframes validate` in parallel, fix all errors before opening the studio or considering the task complete. `lint` checks the HTML structure statically; `validate` loads the composition in headless Chrome and catches runtime JS errors, missing assets, and failed network requests. Always validate before `npx hyperframes preview`. ### Installing skills @@ -67,6 +67,18 @@ pnpm build # Build all packages pnpm test # Run tests ``` +### Linting & Formatting + +This project uses **oxlint** and **oxfmt** (not biome, not eslint, not prettier). + +```bash +bunx oxlint # Lint +bunx oxfmt # Format (write) +bunx oxfmt --check # Format (check only, used by pre-commit hook) +``` + +Always run both on changed files before committing. The lefthook pre-commit hook runs `bunx oxlint` and `bunx oxfmt --check` automatically. + ## Key Concepts - **Compositions** are HTML files with `data-*` attributes defining timeline, tracks, and media diff --git a/lefthook.yml b/lefthook.yml index a413bfff3..9807101e0 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -7,6 +7,9 @@ pre-commit: format: glob: "*.{js,jsx,ts,tsx,json,css,md,yaml,yml}" run: bunx oxfmt --check {staged_files} + typecheck: + glob: "*.{ts,tsx}" + run: cd packages/core && bunx tsc --noEmit && cd ../studio && bunx tsc --noEmit commit-msg: commands: diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 11a9b761b..3ea75e883 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -37,6 +37,7 @@ const subCommands = { doctor: () => import("./commands/doctor.js").then((m) => m.default), upgrade: () => import("./commands/upgrade.js").then((m) => m.default), telemetry: () => import("./commands/telemetry.js").then((m) => m.default), + validate: () => import("./commands/validate.js").then((m) => m.default), }; const main = defineCommand({ diff --git a/packages/cli/src/commands/validate.ts b/packages/cli/src/commands/validate.ts new file mode 100644 index 000000000..089c58fcf --- /dev/null +++ b/packages/cli/src/commands/validate.ts @@ -0,0 +1,247 @@ +import { defineCommand } from "citty"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve, join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { resolveProject } from "../utils/project.js"; +import { c } from "../ui/colors.js"; +import { withMeta } from "../utils/updateCheck.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +interface ConsoleEntry { + level: "error" | "warning"; + text: string; + url?: string; + line?: number; +} + +/** + * Bundle the project HTML with the runtime injected, serve it via a minimal + * static server, open headless Chrome, and collect console errors. + */ +async function validateInBrowser( + projectDir: string, + opts: { timeout?: number }, +): Promise<{ errors: ConsoleEntry[]; warnings: ConsoleEntry[] }> { + const { bundleToSingleHtml } = await import("@hyperframes/core/compiler"); + const { ensureBrowser } = await import("../browser/manager.js"); + + // 1. Bundle + let html = await bundleToSingleHtml(projectDir); + + // Inject local runtime if available + const runtimePath = resolve( + __dirname, + "..", + "..", + "..", + "core", + "dist", + "hyperframe.runtime.iife.js", + ); + if (existsSync(runtimePath)) { + const runtimeSource = readFileSync(runtimePath, "utf-8"); + html = html.replace( + /]*data-hyperframes-preview-runtime[^>]*src="[^"]*"[^>]*><\/script>/, + ``, + ); + } + + // 2. Start minimal file server for project assets (audio, images, fonts, json) + const { createServer } = await import("node:http"); + const { getMimeType } = await import("@hyperframes/core/studio-api"); + + const server = createServer((req, res) => { + const url = req.url ?? "/"; + if (url === "/" || url === "/index.html") { + res.writeHead(200, { "Content-Type": "text/html" }); + res.end(html); + return; + } + // Serve project files + const filePath = join(projectDir, decodeURIComponent(url)); + if (existsSync(filePath)) { + res.writeHead(200, { "Content-Type": getMimeType(filePath) }); + res.end(readFileSync(filePath)); + return; + } + res.writeHead(404); + res.end(); + }); + + const port = await new Promise((resolvePort) => { + server.listen(0, () => { + const addr = server.address(); + resolvePort(typeof addr === "object" && addr ? addr.port : 0); + }); + }); + + const errors: ConsoleEntry[] = []; + const warnings: ConsoleEntry[] = []; + + try { + // 3. Launch headless Chrome + const browser = await ensureBrowser(); + const puppeteer = await import("puppeteer-core"); + const chromeBrowser = await puppeteer.default.launch({ + headless: true, + executablePath: browser.executablePath, + args: ["--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage"], + }); + + const page = await chromeBrowser.newPage(); + await page.setViewport({ width: 1920, height: 1080 }); + + // 4. Capture console messages + page.on("console", (msg) => { + const type = msg.type(); + const loc = msg.location(); + const text = msg.text(); + if (type === "error") { + // Network errors show as console errors but with no useful location. + // We capture those separately via response/requestfailed events. + if (text.startsWith("Failed to load resource")) return; + errors.push({ level: "error", text, url: loc.url, line: loc.lineNumber }); + } else if (type === "warn") { + warnings.push({ level: "warning", text, url: loc.url, line: loc.lineNumber }); + } + }); + + // Capture uncaught exceptions + page.on("pageerror", (err) => { + const message = err instanceof Error ? err.message : String(err); + errors.push({ level: "error", text: message }); + }); + + // Capture failed network requests for project assets (skip favicon, data: URIs) + page.on("requestfailed", (req) => { + const url = req.url(); + if (url.includes("favicon")) return; + if (url.startsWith("data:")) return; + // Extract the path relative to the server + const urlObj = new URL(url); + const path = decodeURIComponent(urlObj.pathname).replace(/^\//, ""); + const failure = req.failure()?.errorText ?? "net::ERR_FAILED"; + errors.push({ level: "error", text: `Failed to load ${path}: ${failure}`, url }); + }); + + // Capture HTTP errors (404, 500, etc.) for project assets + page.on("response", (res) => { + const status = res.status(); + if (status >= 400) { + const url = res.url(); + if (url.includes("favicon")) return; + const urlObj = new URL(url); + const path = decodeURIComponent(urlObj.pathname).replace(/^\//, ""); + errors.push({ level: "error", text: `${status} loading ${path}`, url }); + } + }); + + // 5. Navigate and wait + const timeoutMs = opts.timeout ?? 3000; + await page.goto(`http://127.0.0.1:${port}/`, { + waitUntil: "domcontentloaded", + timeout: 10000, + }); + + // Wait for scripts to settle + await new Promise((r) => setTimeout(r, timeoutMs)); + + await chromeBrowser.close(); + } finally { + server.close(); + } + + return { errors, warnings }; +} + +export default defineCommand({ + meta: { + name: "validate", + description: `Load a composition in headless Chrome and report console errors + +Examples: + hyperframes validate + hyperframes validate ./my-project + hyperframes validate --json + hyperframes validate --timeout 5000`, + }, + args: { + dir: { + type: "positional", + description: "Project directory", + required: false, + }, + json: { + type: "boolean", + description: "Output as JSON", + default: false, + }, + timeout: { + type: "string", + description: "Ms to wait for scripts to settle (default: 3000)", + default: "3000", + }, + }, + async run({ args }) { + const project = resolveProject(args.dir); + const timeout = parseInt(args.timeout as string, 10) || 3000; + + if (!args.json) { + console.log(`${c.accent("◆")} Validating ${c.accent(project.name)} in headless Chrome`); + } + + try { + const { errors, warnings } = await validateInBrowser(project.dir, { timeout }); + + if (args.json) { + console.log( + JSON.stringify( + withMeta({ + ok: errors.length === 0, + errors, + warnings, + }), + null, + 2, + ), + ); + process.exit(errors.length > 0 ? 1 : 0); + } + + if (errors.length === 0 && warnings.length === 0) { + console.log(`${c.success("◇")} No console errors`); + return; + } + + console.log(); + for (const e of errors) { + const loc = e.line ? ` (line ${e.line})` : ""; + console.log(` ${c.error("✗")} ${e.text}${c.dim(loc)}`); + } + for (const w of warnings) { + const loc = w.line ? ` (line ${w.line})` : ""; + console.log(` ${c.warn("⚠")} ${w.text}${c.dim(loc)}`); + } + console.log(); + console.log(`${c.accent("◇")} ${errors.length} error(s), ${warnings.length} warning(s)`); + + process.exit(errors.length > 0 ? 1 : 0); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + if (args.json) { + console.log( + JSON.stringify( + withMeta({ ok: false, error: message, errors: [], warnings: [] }), + null, + 2, + ), + ); + process.exit(1); + } + console.error(`${c.error("✗")} ${message}`); + process.exit(1); + } + }, +}); diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.ts index 7b2d4fce9..fbc083f50 100644 --- a/packages/cli/src/whisper/normalize.ts +++ b/packages/cli/src/whisper/normalize.ts @@ -2,6 +2,10 @@ import { readFileSync, readdirSync, writeFileSync } from "node:fs"; import { extname, join } from "node:path"; export interface Word { + /** Stable identifier for referencing this word in overrides and compositions. + * Assigned during normalization as `w{index}`. Optional for backwards compat + * with existing transcript.json files that predate this field. */ + id?: string; text: string; start: number; end: number; @@ -150,13 +154,13 @@ function parseWhisperCpp(data: Record): Word[] { } function parseOpenAI(data: Record): Word[] { - const rawWords = (data.words ?? []) as Array<{ + const words = (data.words ?? []) as Array<{ word?: string; text?: string; start?: number; end?: number; }>; - return rawWords + return words .map((w) => ({ text: (w.word ?? w.text ?? "").trim(), start: round3(w.start ?? 0), @@ -280,8 +284,14 @@ export function loadTranscript(filePath: string): { words: Word[]; format: Trans const ext = extname(filePath).toLowerCase(); const content = readFileSync(filePath, "utf-8"); - if (ext === ".srt") return { words: parseSrt(content), format: "srt" }; - if (ext === ".vtt") return { words: parseVtt(content), format: "vtt" }; + if (ext === ".srt") { + const words = parseSrt(content).map((w, i) => ({ ...w, id: w.id ?? `w${i}` })); + return { words, format: "srt" }; + } + if (ext === ".vtt") { + const words = parseVtt(content).map((w, i) => ({ ...w, id: w.id ?? `w${i}` })); + return { words, format: "vtt" }; + } // JSON formats — parse once, detect, then extract words const parsed = JSON.parse(content); @@ -293,6 +303,7 @@ export function loadTranscript(filePath: string): { words: Word[]; format: Trans : format === "openai" ? parseOpenAI(parsed) : (parsed as Word[]).map((w) => ({ + id: w.id ?? "", text: w.text.trim(), start: round3(w.start), end: round3(w.end), diff --git a/packages/core/src/runtime/captionOverrides.ts b/packages/core/src/runtime/captionOverrides.ts new file mode 100644 index 000000000..47699474e --- /dev/null +++ b/packages/core/src/runtime/captionOverrides.ts @@ -0,0 +1,132 @@ +/** + * Caption Overrides — applies per-word style overrides from a JSON data file. + * + * Strategy: wrap each overridden word span in an inline-block wrapper span, + * then apply transforms to the wrapper. The inner span keeps all its original + * GSAP animations (entrance, karaoke, exit) untouched. No tweens are killed. + * + * Matching (in priority order): + * 1. `wordId` — matches by element ID (document.getElementById) + * 2. `wordIndex` — fallback, DOM traversal order across .caption-group > span + */ + +export interface CaptionOverride { + wordId?: string; + wordIndex?: number; + x?: number; + y?: number; + scale?: number; + rotation?: number; + /** Color when the word is being spoken (karaoke active state) */ + activeColor?: string; + /** Color before and after the word is spoken (dim/inactive state) */ + dimColor?: string; + opacity?: number; + fontSize?: number; + fontWeight?: number; + fontFamily?: string; +} + +interface GsapTween { + vars: Record; + startTime(): number; +} + +interface GsapStatic { + set: (target: Element, vars: Record) => void; + killTweensOf: (target: Element, props: string) => void; + getTweensOf: (target: Element) => GsapTween[]; +} + +export function applyCaptionOverrides(): void { + const gsap = (window as unknown as { gsap?: GsapStatic }).gsap; + if (!gsap) return; + + fetch("caption-overrides.json") + .then((r) => { + if (!r.ok) return null; + return r.json(); + }) + .then((data: CaptionOverride[] | null) => { + if (!data || !Array.isArray(data) || data.length === 0) return; + + // Build word element index for wordIndex fallback + const wordEls: Element[] = []; + const groups = document.querySelectorAll(".caption-group"); + for (const group of groups) { + const spans = group.querySelectorAll(":scope > span"); + for (const span of spans) { + wordEls.push(span); + } + } + + for (const override of data) { + let el: Element | null = null; + if (override.wordId) { + el = document.getElementById(override.wordId); + } + if (!el && override.wordIndex !== undefined) { + el = wordEls[override.wordIndex] ?? null; + } + if (!el || !(el instanceof HTMLElement)) continue; + + // Split into transform props (wrapper) and style props (word span) + const transformProps: Record = {}; + const styleProps: Record = {}; + + if (override.x !== undefined) transformProps.x = override.x; + if (override.y !== undefined) transformProps.y = override.y; + if (override.scale !== undefined) transformProps.scale = override.scale; + if (override.rotation !== undefined) transformProps.rotation = override.rotation; + if (override.opacity !== undefined) styleProps.opacity = override.opacity; + if (override.fontSize !== undefined) styleProps.fontSize = `${override.fontSize}px`; + if (override.fontWeight !== undefined) styleProps.fontWeight = override.fontWeight; + if (override.fontFamily !== undefined) styleProps.fontFamily = override.fontFamily; + + // Replace color values in existing GSAP tweens by timeline order. + // For any word, color tweens follow: dim (setup) → active (spoken) → after. + // Sort by startTime and assign by position, not by content heuristics. + if (override.activeColor || override.dimColor) { + const allTweens = gsap.getTweensOf(el); + const colorTweens = allTweens + .filter((tw) => tw.vars.color !== undefined) + .sort((a, b) => a.startTime() - b.startTime()); + + for (let i = 0; i < colorTweens.length; i++) { + if (i === 0 && override.dimColor) { + // First color tween = dim setup + colorTweens[i].vars.color = override.dimColor; + } else if (i === 1 && override.activeColor) { + // Second color tween = active/spoken + colorTweens[i].vars.color = override.activeColor; + } else if (i >= 2 && override.dimColor) { + // Third+ = after/deactivate (use dim color) + colorTweens[i].vars.color = override.dimColor; + } + } + + // Set current visible color (words start in dim state) + if (override.dimColor) { + gsap.set(el, { color: override.dimColor }); + } + } + + // Apply non-color style props + if (Object.keys(styleProps).length > 0) { + gsap.set(el, styleProps); + } + + // Wrap the word in an inline-block span and apply transforms to the wrapper. + // This preserves all GSAP entrance/exit/karaoke animations on the inner span. + if (Object.keys(transformProps).length > 0) { + const wrapper = document.createElement("span"); + wrapper.style.display = "inline-block"; + wrapper.dataset.captionWrapper = "true"; + el.parentNode?.insertBefore(wrapper, el); + wrapper.appendChild(el); + gsap.set(wrapper, transformProps); + } + } + }) + .catch(() => {}); +} diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index a73739f37..25ac6c16f 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -12,6 +12,7 @@ import { createRuntimeState } from "./state"; import { collectRuntimeTimelinePayload } from "./timeline"; import { createRuntimeStartTimeResolver } from "./startResolver"; import { loadExternalCompositions, loadInlineTemplateCompositions } from "./compositionLoader"; +import { applyCaptionOverrides } from "./captionOverrides"; import type { RuntimeDeterministicAdapter, RuntimeJson, RuntimeTimelineLike } from "./types"; import type { PlayerAPI } from "../core.types"; @@ -1316,9 +1317,13 @@ export function initSandboxRuntimeModular(): void { runAdapters("discover", state.currentTime); bindMediaMetadataListeners(); installAssetFailureDiagnostics(); + applyCaptionOverrides(); postTimeline(); postState(true); }); + } else { + // No external/inline compositions to load — apply caption overrides immediately + applyCaptionOverrides(); } const picker = createPickerModule({ diff --git a/packages/studio/src/App.tsx b/packages/studio/src/App.tsx index 6823b6c13..3595613e7 100644 --- a/packages/studio/src/App.tsx +++ b/packages/studio/src/App.tsx @@ -12,6 +12,12 @@ import { LintModal } from "./components/LintModal"; import type { LintFinding } from "./components/LintModal"; import { MediaPreview } from "./components/MediaPreview"; import { isMediaFile } from "./utils/mediaTypes"; +import { CaptionOverlay } from "./captions/components/CaptionOverlay"; +import { CaptionPropertyPanel } from "./captions/components/CaptionPropertyPanel"; +import { CaptionTimeline } from "./captions/components/CaptionTimeline"; +import { useCaptionStore } from "./captions/store"; +import { useCaptionSync } from "./captions/hooks/useCaptionSync"; +import { parseCaptionComposition } from "./captions/parser"; interface EditingFile { path: string; @@ -50,12 +56,134 @@ export function StudioApp() { const [fileTree, setFileTree] = useState([]); const [compIdToSrc, setCompIdToSrc] = useState>(new Map()); const renderQueue = useRenderQueue(projectId); + const captionEditMode = useCaptionStore((s) => s.isEditMode); + const captionHasSelection = useCaptionStore((s) => s.selectedSegmentIds.size > 0); + const captionSync = useCaptionSync(projectId); // Resizable and collapsible panel widths const [leftWidth, setLeftWidth] = useState(240); const [rightWidth, setRightWidth] = useState(400); const [leftCollapsed, setLeftCollapsed] = useState(false); const [rightCollapsed, setRightCollapsed] = useState(true); + // Auto-enter caption edit mode when viewing a captions composition + // Auto-enter caption edit mode when the iframe contains .caption-group elements. + // Listens for the runtime's postMessage events (state/timeline) which fire after + // all compositions are loaded, then checks for caption groups. + // eslint-disable-next-line no-restricted-syntax + useEffect(() => { + if (!projectId) return; + + let pollId: ReturnType | null = null; + let activating = false; + + const tryActivateCaptions = () => { + if (useCaptionStore.getState().isEditMode || activating) { + if (pollId) { clearInterval(pollId); pollId = null; } + return; + } + + const iframe = previewIframeRef.current; + let doc: Document | null = null; + let win: Window | null = null; + try { + doc = iframe?.contentDocument ?? null; + win = iframe?.contentWindow ?? null; + } catch { return; } + if (!doc || !win) return; + + const groups = doc.querySelectorAll(".caption-group"); + if (groups.length === 0) return; + + // Find the captions composition source path. + // The runtime strips data-composition-src after loading, so also check + // data-composition-file (set by the bundler) and the compIdToSrc map. + let captionSrcPath: string | null = null; + + // Strategy 1: data-composition-src or data-composition-file attributes + const compHosts = doc.querySelectorAll("[data-composition-src], [data-composition-file]"); + for (const host of compHosts) { + const src = host.getAttribute("data-composition-src") || host.getAttribute("data-composition-file"); + if (src && src.includes("captions")) { + captionSrcPath = src; + break; + } + } + + // Strategy 2: compIdToSrc map (built from raw index.html before runtime strips attrs) + if (!captionSrcPath) { + for (const [id, src] of compIdToSrc) { + if (id.includes("caption") || src.includes("caption")) { + captionSrcPath = src; + break; + } + } + } + + // Strategy 3: activeCompPath if viewing captions directly + if (!captionSrcPath && activeCompPath?.includes("captions")) { + captionSrcPath = activeCompPath; + } + + // Strategy 4: find composition element with "caption" in its ID + if (!captionSrcPath) { + const captionComp = doc.querySelector('[data-composition-id*="caption"]'); + if (captionComp) { + const compId = captionComp.getAttribute("data-composition-id") || ""; + captionSrcPath = compIdToSrc.get(compId) || null; + } + } + + if (!captionSrcPath) return; + + activating = true; + const srcPath = captionSrcPath; + fetch(`/api/projects/${projectId}/files/${encodeURIComponent(srcPath)}`) + .then((r) => r.json()) + .then((data: { content?: string }) => { + if (!data.content || !doc || !win || useCaptionStore.getState().isEditMode) return; + const root = doc.querySelector("[data-composition-id]"); + const w = parseInt(root?.getAttribute("data-width") ?? "1920", 10); + const h = parseInt(root?.getAttribute("data-height") ?? "1080", 10); + const dur = parseFloat(root?.getAttribute("data-duration") ?? "0"); + const model = parseCaptionComposition(doc, win, data.content, w, h, dur); + if (!model) return; + const store = useCaptionStore.getState(); + store.setModel(model); + store.setSourceFilePath(srcPath); + store.setEditMode(true); + captionSync.loadOverrides(); + }) + .catch(() => {}) + .finally(() => { activating = false; }); + }; + + // Listen for runtime messages that signal composition loading is complete + const handleMessage = (e: MessageEvent) => { + const data = e.data; + if (data?.source === "hf-preview" && (data?.type === "state" || data?.type === "timeline")) { + tryActivateCaptions(); + } + }; + + window.addEventListener("message", handleMessage); + // Try immediately in case compositions are already loaded + tryActivateCaptions(); + // Poll until captions are detected — sub-composition scripts run async + pollId = setInterval(tryActivateCaptions, 200); + + return () => { + window.removeEventListener("message", handleMessage); + if (pollId) clearInterval(pollId); + }; + }, [activeCompPath, projectId, compIdToSrc]); + + // Auto-expand right panel when a caption word is selected + // eslint-disable-next-line no-restricted-syntax + useEffect(() => { + if (captionEditMode) { + setRightCollapsed(!captionHasSelection); + } + }, [captionHasSelection, captionEditMode]); const [globalDragOver, setGlobalDragOver] = useState(false); const [uploadToast, setUploadToast] = useState(null); const [timelineVisible, setTimelineVisible] = useState(false); @@ -159,12 +287,15 @@ export function StudioApp() { [compIdToSrc, activePreviewUrl], ); const [lintModal, setLintModal] = useState(null); + const [consoleErrors, setConsoleErrors] = useState(null); const [linting, setLinting] = useState(false); const [refreshKey, setRefreshKey] = useState(0); const refreshTimerRef = useRef | null>(null); const saveTimerRef = useRef | null>(null); const projectIdRef = useRef(projectId); const previewIframeRef = useRef(null); + const consoleErrorsRef = useRef([]); + // Listen for external file changes (user editing HTML outside the editor). // In dev: use Vite HMR. In embedded/production: use SSE from /api/events. @@ -673,7 +804,68 @@ export function StudioApp() { }} onIframeRef={(iframe) => { previewIframeRef.current = iframe; + consoleErrorsRef.current = []; + setConsoleErrors(null); + if (!iframe) return; + + // Attach error capture after each iframe load (content resets on navigation) + const attachErrorCapture = () => { + try { + const win = iframe.contentWindow as (Window & typeof globalThis) | null; + if (!win) return; + // Guard against double-patching + if ((win as unknown as Record).__hfErrorCapture) return; + (win as unknown as Record).__hfErrorCapture = true; + const origError = win.console.error.bind(win.console); + win.console.error = function (...args: unknown[]) { + origError(...args); + const text = args + .map((a) => (a instanceof Error ? a.message : String(a))) + .join(" "); + if (text.includes("favicon")) return; + consoleErrorsRef.current = [ + ...consoleErrorsRef.current, + { severity: "error", message: text }, + ]; + setConsoleErrors([...consoleErrorsRef.current]); + }; + win.addEventListener("error", (e: ErrorEvent) => { + const text = e.message || String(e); + consoleErrorsRef.current = [ + ...consoleErrorsRef.current, + { severity: "error", message: text }, + ]; + setConsoleErrors([...consoleErrorsRef.current]); + }); + } catch { + // cross-origin — can't attach + } + }; + // Attach now (iframe may already be loaded) and on future loads + attachErrorCapture(); + iframe.addEventListener("load", () => { + consoleErrorsRef.current = []; + setConsoleErrors(null); + attachErrorCapture(); + }); }} + previewOverlay={ + captionEditMode ? ( + + ) : undefined + } + timelineFooter={ + captionEditMode ? ( +
+
+ + Captions + +
+ +
+ ) : undefined + } timelineVisible={timelineVisible} onToggleTimeline={() => setTimelineVisible((v) => !v)} /> @@ -693,14 +885,18 @@ export function StudioApp() { className="flex flex-col border-l border-neutral-800 bg-neutral-900 flex-shrink-0" style={{ width: rightWidth }} > - renderQueue.startRender(30, "standard", format)} - isRendering={renderQueue.isRendering} - /> + {captionEditMode ? ( + + ) : ( + renderQueue.startRender(30, "standard", format)} + isRendering={renderQueue.isRendering} + /> + )} )} @@ -711,6 +907,15 @@ export function StudioApp() { setLintModal(null)} /> )} + {/* Console errors modal — auto-shows when composition has runtime errors */} + {consoleErrors !== null && consoleErrors.length > 0 && projectId && ( + setConsoleErrors(null)} + /> + )} + {/* Global drag-drop overlay */} {globalDragOver && (
diff --git a/packages/studio/src/captions/components/CaptionAnimationPanel.tsx b/packages/studio/src/captions/components/CaptionAnimationPanel.tsx new file mode 100644 index 000000000..62ebe1349 --- /dev/null +++ b/packages/studio/src/captions/components/CaptionAnimationPanel.tsx @@ -0,0 +1,300 @@ +import { memo, useCallback } from "react"; +import { useCaptionStore } from "../store"; +import type { CaptionAnimation } from "../types"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const ENTRANCE_PRESETS = [ + "none", + "fade", + "slide-up", + "slide-down", + "slide-left", + "slide-right", + "pop", + "slam", + "bounce", + "typewriter", + "blur-in", + "flip", + "drop", +]; + +const HIGHLIGHT_PRESETS = [ + "none", + "color-change", + "scale-pop", + "glow-pulse", + "underline-sweep", + "background-fill", + "bounce", +]; + +const EXIT_PRESETS = [ + "none", + "fade", + "slide-up", + "slide-down", + "slide-left", + "slide-right", + "scatter", + "drop", + "collapse", + "blur-out", + "shrink", +]; + +const EASE_PRESETS = [ + "power1.out", + "power2.out", + "power3.out", + "power4.out", + "power1.in", + "power2.in", + "power3.in", + "power1.inOut", + "power2.inOut", + "back.out(1.7)", + "elastic.out(1,0.3)", + "bounce.out", +]; + +// --------------------------------------------------------------------------- +// Shared input class (matches CaptionPropertyPanel) +// --------------------------------------------------------------------------- + +const inputCls = + "w-full bg-neutral-900 border border-neutral-800 rounded px-1.5 py-0.5 text-2xs text-neutral-200 font-mono outline-none focus:border-neutral-600"; + +// --------------------------------------------------------------------------- +// Helper Components +// --------------------------------------------------------------------------- + +function Section({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+
+ + {label} + +
+
{children}
+
+ ); +} + +function Row({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ {label} +
{children}
+
+ ); +} + +// --------------------------------------------------------------------------- +// Animation phase controls +// --------------------------------------------------------------------------- + +interface AnimationPhaseProps { + label: string; + presets: string[]; + animation: CaptionAnimation | null; + showIntensity?: boolean; + onChange: (update: Partial) => void; +} + +function AnimationPhase({ + label, + presets, + animation, + showIntensity, + onChange, +}: AnimationPhaseProps) { + const preset = animation?.preset ?? "none"; + const duration = animation?.duration ?? 0.2; + const ease = animation?.ease ?? "power2.out"; + const stagger = animation?.stagger ?? 0; + const intensity = animation?.intensity ?? 1; + + return ( +
+ + + + + + onChange({ duration: Number(e.target.value) })} + className={inputCls} + /> + + + + + + + + onChange({ stagger: Number(e.target.value) })} + className={inputCls} + /> + + + {showIntensity && ( + +
+ onChange({ intensity: Number(e.target.value) })} + className="flex-1 accent-studio-accent" + /> + + {intensity.toFixed(2)} + +
+ + )} +
+ ); +} + +// --------------------------------------------------------------------------- +// Main component +// --------------------------------------------------------------------------- + +export const CaptionAnimationPanel = memo(function CaptionAnimationPanel() { + const model = useCaptionStore((s) => s.model); + const selectedGroupId = useCaptionStore((s) => s.selectedGroupId); + const selectedSegmentIds = useCaptionStore((s) => s.selectedSegmentIds); + const updateGroupAnimation = useCaptionStore((s) => s.updateGroupAnimation); + const applyAnimationToAll = useCaptionStore((s) => s.applyAnimationToAll); + + // Resolve which group to edit + let resolvedGroupId: string | null = selectedGroupId; + if (!resolvedGroupId && model && selectedSegmentIds.size > 0) { + const firstSegmentId = [...selectedSegmentIds][0]; + if (firstSegmentId) { + for (const [gid, group] of model.groups) { + if (group.segmentIds.includes(firstSegmentId)) { + resolvedGroupId = gid; + break; + } + } + } + } + + const group = resolvedGroupId ? model?.groups.get(resolvedGroupId) : undefined; + const animation = group?.animation; + + // All hooks must be called before any early return + const handleEntranceChange = useCallback( + (update: Partial) => { + if (resolvedGroupId) updateGroupAnimation(resolvedGroupId, "entrance", update); + }, + [resolvedGroupId, updateGroupAnimation], + ); + + const handleHighlightChange = useCallback( + (update: Partial) => { + if (resolvedGroupId) updateGroupAnimation(resolvedGroupId, "highlight", update); + }, + [resolvedGroupId, updateGroupAnimation], + ); + + const handleExitChange = useCallback( + (update: Partial) => { + if (resolvedGroupId) updateGroupAnimation(resolvedGroupId, "exit", update); + }, + [resolvedGroupId, updateGroupAnimation], + ); + + const handleApplyToAll = useCallback(() => { + if (animation) applyAnimationToAll(animation); + }, [animation, applyAnimationToAll]); + + // Empty state — after all hooks + if (!group || !resolvedGroupId || !animation) { + return ( +
+

Select a caption group to edit animations

+
+ ); + } + + return ( +
+ {/* Scrollable content */} +
+ + + + + +
+ + {/* Footer */} +
+ +
+
+ ); +}); diff --git a/packages/studio/src/captions/components/CaptionOverlay.tsx b/packages/studio/src/captions/components/CaptionOverlay.tsx new file mode 100644 index 000000000..ec557c5f1 --- /dev/null +++ b/packages/studio/src/captions/components/CaptionOverlay.tsx @@ -0,0 +1,462 @@ +import { memo, useState, useCallback, useRef } from "react"; +import { useCaptionStore } from "../store"; +import { useMountEffect } from "../../hooks/useMountEffect"; + +interface CaptionOverlayProps { + iframeRef: React.RefObject; +} + +interface WordBox { + segmentId: string; + groupId: string; + groupIndex: number; + wordIndex: number; + x: number; + y: number; + width: number; + height: number; +} + +function readWordBoxes( + iframe: HTMLIFrameElement, + model: { + groupOrder: string[]; + groups: Map; + }, + overlayEl: HTMLElement, +): WordBox[] { + let doc: Document | null = null; + let win: Window | null = null; + try { + doc = iframe.contentDocument; + win = iframe.contentWindow; + } catch { + return []; + } + if (!doc || !win) return []; + + const iframeDisplayRect = iframe.getBoundingClientRect(); + const overlayRect = overlayEl.getBoundingClientRect(); + const nativeW = parseFloat(iframe.style.width) || iframeDisplayRect.width; + const cssScale = iframeDisplayRect.width / nativeW; + const offsetX = iframeDisplayRect.left - overlayRect.left; + const offsetY = iframeDisplayRect.top - overlayRect.top; + + const groupEls = doc.querySelectorAll(".caption-group"); + const boxes: WordBox[] = []; + + for (let gi = 0; gi < model.groupOrder.length; gi++) { + const groupId = model.groupOrder[gi]; + const group = model.groups.get(groupId); + if (!group) continue; + const groupEl = groupEls[gi] as HTMLElement | undefined; + if (!groupEl) continue; + const computed = win.getComputedStyle(groupEl); + if (parseFloat(computed.opacity) <= 0.01 || computed.visibility === "hidden") continue; + // Find word spans — may be direct children or inside wrappers + const resolvedWordEls: HTMLElement[] = []; + for (const child of groupEl.children) { + const c = child as HTMLElement; + if (c.dataset.captionWrapper === "true") { + const inner = c.querySelector(":scope > span"); + if (inner) resolvedWordEls.push(inner); + } else if (c.tagName === "SPAN") { + resolvedWordEls.push(c); + } + } + for (let wi = 0; wi < group.segmentIds.length; wi++) { + const segId = group.segmentIds[wi]; + const wordEl = resolvedWordEls[wi] as HTMLElement | undefined; + if (!wordEl) continue; + const rect = wordEl.getBoundingClientRect(); + boxes.push({ + segmentId: segId, groupId, groupIndex: gi, wordIndex: wi, + x: rect.left * cssScale + offsetX, + y: rect.top * cssScale + offsetY, + width: rect.width * cssScale, + height: rect.height * cssScale, + }); + } + } + return boxes; +} + +function getWordEl(iframe: HTMLIFrameElement, groupIndex: number, wordIndex: number): HTMLElement | null { + let doc: Document | null = null; + try { doc = iframe.contentDocument; } catch { return null; } + if (!doc) return null; + const groupEl = doc.querySelectorAll(".caption-group")[groupIndex]; + if (!groupEl) return null; + // Find word spans — they may be direct children or inside wrapper spans. + // Word spans have class "word" or an id starting with "w". + // Wrappers have data-caption-wrapper="true". + const wordEls: HTMLElement[] = []; + for (const child of groupEl.children) { + const el = child as HTMLElement; + if (el.dataset.captionWrapper === "true") { + // Wrapped word — get the inner span + const inner = el.querySelector(":scope > span"); + if (inner) wordEls.push(inner); + } else if (el.tagName === "SPAN") { + wordEls.push(el); + } + } + return wordEls[wordIndex] ?? null; +} + +/** + * Read GSAP's internal transform state for an element. + * GSAP stores transforms in its own cache, not in el.style.transform. + */ +function readGsapTransform(el: HTMLElement, iframeWin: Window): { x: number; y: number; scale: number; rotation: number } { + const gsap = (iframeWin as unknown as { gsap?: { getProperty?: (el: HTMLElement, prop: string) => number } }).gsap; + if (gsap && gsap.getProperty) { + return { + x: gsap.getProperty(el, "x") || 0, + y: gsap.getProperty(el, "y") || 0, + scale: gsap.getProperty(el, "scale") || 1, + rotation: gsap.getProperty(el, "rotation") || 0, + }; + } + // Fallback: parse from style + const t = el.style.transform || ""; + const scaleMatch = t.match(/scale\(([^)]+)\)/); + const rotMatch = t.match(/rotate\(([^)]+)deg\)/); + const txyMatch = t.match(/translate\(([^,]+)px,\s*([^)]+)px\)/); + return { + x: txyMatch ? parseFloat(txyMatch[1]) : 0, + y: txyMatch ? parseFloat(txyMatch[2]) : 0, + scale: scaleMatch ? parseFloat(scaleMatch[1]) : 1, + rotation: rotMatch ? parseFloat(rotMatch[1]) : 0, + }; +} + +/** + * Get or create an inline-block wrapper span around a word element. + * Transforms are applied to the wrapper so the word's GSAP animations are preserved. + */ +function getOrCreateWrapper(el: HTMLElement): HTMLElement { + // If el IS a wrapper, return it + if (el.dataset.captionWrapper === "true") return el; + // If el's parent is a wrapper, return the parent + const parent = el.parentElement; + if (parent && parent.dataset.captionWrapper === "true") return parent; + // Create new wrapper + const doc = el.ownerDocument; + const wrapper = doc.createElement("span"); + wrapper.style.display = "inline-block"; + wrapper.dataset.captionWrapper = "true"; + el.parentNode?.insertBefore(wrapper, el); + wrapper.appendChild(el); + return wrapper; +} + +/** + * Write transform values to a wrapper span around the word element. + * The word keeps its GSAP animations; the wrapper handles editor transforms. + */ +function writeTransform(el: HTMLElement, iframeWin: Window, x: number, y: number, scale: number, rotation: number) { + const wrapper = getOrCreateWrapper(el); + const gsap = (iframeWin as unknown as { gsap?: { set?: (el: HTMLElement, props: Record) => void } }).gsap; + if (gsap && gsap.set) { + gsap.set(wrapper, { x, y, scale, rotation }); + } else { + wrapper.style.transform = `translate(${x.toFixed(1)}px, ${y.toFixed(1)}px) rotate(${rotation.toFixed(1)}deg) scale(${scale.toFixed(3)})`; + } +} + +/** Sync canvas state back to the Zustand store so the property panel reflects it. + * Only writes non-default values to avoid creating spurious overrides. */ +function syncToStore(segmentId: string, el: HTMLElement, iframeWin: Window) { + const wrapper = getOrCreateWrapper(el); + const { x, y, scale, rotation } = readGsapTransform(wrapper, iframeWin); + const style: Record = {}; + if (Math.abs(x) > 0.5) style.x = x; + if (Math.abs(y) > 0.5) style.y = y; + if (Math.abs(scale - 1) > 0.001) { style.scaleX = scale; style.scaleY = scale; } + if (Math.abs(rotation) > 0.1) style.rotation = rotation; + if (Object.keys(style).length > 0) { + useCaptionStore.getState().updateSegmentStyle(segmentId, style); + } +} + +const HANDLE = 8; +const ROTATION_OFFSET = 20; // px above the selection box + +export const CaptionOverlay = memo(function CaptionOverlay({ + iframeRef, +}: CaptionOverlayProps) { + const isEditMode = useCaptionStore((s) => s.isEditMode); + const model = useCaptionStore((s) => s.model); + const selectedSegmentIds = useCaptionStore((s) => s.selectedSegmentIds); + const selectSegment = useCaptionStore((s) => s.selectSegment); + const clearSelection = useCaptionStore((s) => s.clearSelection); + + const [wordBoxes, setWordBoxes] = useState([]); + const overlayRef = useRef(null); + const modelRef = useRef(model); + modelRef.current = model; + + // Interaction mode — only one active at a time + const interactionRef = useRef< + | { type: "move"; wordEl: HTMLElement; segmentId: string; startMX: number; startMY: number; origTX: number; origTY: number; origScale: number; origRotation: number } + | { type: "scale"; wordEl: HTMLElement; segmentId: string; startMX: number; startWidth: number; origTX: number; origTY: number; origScale: number; origRotation: number } + | { type: "rotate"; wordEl: HTMLElement; segmentId: string; centerX: number; centerY: number; startAngle: number; origTX: number; origTY: number; origRotation: number; origScale: number } + | null + >(null); + + useMountEffect(() => { + if (!isEditMode) return; + let prevBoxes: WordBox[] = []; + const tick = () => { + const iframe = iframeRef.current; + const m = modelRef.current; + const overlay = overlayRef.current; + if (!iframe || !m || !overlay) return; + const next = readWordBoxes(iframe, m, overlay); + // Skip state update if nothing changed (avoids re-render every 66ms) + if (next.length === prevBoxes.length && + next.every((b, i) => Math.abs(b.x - prevBoxes[i].x) < 0.5 && Math.abs(b.y - prevBoxes[i].y) < 0.5)) return; + prevBoxes = next; + setWordBoxes(next); + }; + const id = setInterval(tick, 66); + tick(); + + // Arrow key nudge for selected words + const handleKeyDown = (e: KeyboardEvent) => { + const { selectedSegmentIds: sel, model: m } = useCaptionStore.getState(); + if (sel.size === 0 || !m) return; + const arrow = e.key; + if (!["ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight"].includes(arrow)) return; + + e.preventDefault(); + const step = e.shiftKey ? 10 : 1; + const dx = arrow === "ArrowLeft" ? -step : arrow === "ArrowRight" ? step : 0; + const dy = arrow === "ArrowUp" ? -step : arrow === "ArrowDown" ? step : 0; + + const iframe = iframeRef.current; + const win = iframe?.contentWindow; + if (!iframe || !win) return; + + for (const segId of sel) { + // Find group/word index for this segment + for (let gi = 0; gi < m.groupOrder.length; gi++) { + const group = m.groups.get(m.groupOrder[gi]); + if (!group) continue; + const wi = group.segmentIds.indexOf(segId); + if (wi < 0) continue; + const wordEl = getWordEl(iframe, gi, wi); + if (!wordEl) continue; + const wrapper = getOrCreateWrapper(wordEl); + const state = readGsapTransform(wrapper, win); + writeTransform(wordEl, win, state.x + dx, state.y + dy, state.scale, state.rotation); + syncToStore(segId, wordEl, win); + break; + } + } + }; + + window.addEventListener("keydown", handleKeyDown); + return () => { + clearInterval(id); + window.removeEventListener("keydown", handleKeyDown); + }; + }); + + const getCssScale = useCallback(() => { + const iframe = iframeRef.current; + if (!iframe) return 1; + const rect = iframe.getBoundingClientRect(); + const nativeW = parseFloat(iframe.style.width) || rect.width; + return rect.width / nativeW; + }, [iframeRef]); + + // --- Move --- + const startMove = useCallback((groupIndex: number, wordIndex: number, segmentId: string, e: React.PointerEvent) => { + e.stopPropagation(); + (e.currentTarget as HTMLElement).setPointerCapture(e.pointerId); + const iframe = iframeRef.current; + if (!iframe) return; + const wordEl = getWordEl(iframe, groupIndex, wordIndex); + const win = iframe.contentWindow; + if (!wordEl || !win) return; + const state = readGsapTransform(getOrCreateWrapper(wordEl), win); + interactionRef.current = { + type: "move", wordEl, segmentId, + startMX: e.clientX, startMY: e.clientY, + origTX: state.x, origTY: state.y, + origScale: state.scale, origRotation: state.rotation, + }; + }, [iframeRef]); + + // --- Scale --- + const startScale = useCallback((groupIndex: number, wordIndex: number, segmentId: string, e: React.PointerEvent) => { + e.stopPropagation(); + e.preventDefault(); + (e.currentTarget as HTMLElement).setPointerCapture(e.pointerId); + const iframe = iframeRef.current; + if (!iframe) return; + const wordEl = getWordEl(iframe, groupIndex, wordIndex); + const win = iframe.contentWindow; + if (!wordEl || !win) return; + const rect = wordEl.getBoundingClientRect(); + const state = readGsapTransform(getOrCreateWrapper(wordEl), win); + interactionRef.current = { + type: "scale", wordEl, segmentId, + startMX: e.clientX, startWidth: rect.width, + origTX: state.x, origTY: state.y, + origScale: state.scale, origRotation: state.rotation, + }; + }, [iframeRef]); + + // --- Rotate --- + const startRotate = useCallback((box: WordBox, e: React.PointerEvent) => { + e.stopPropagation(); + e.preventDefault(); + (e.currentTarget as HTMLElement).setPointerCapture(e.pointerId); + const iframe = iframeRef.current; + if (!iframe) return; + const wordEl = getWordEl(iframe, box.groupIndex, box.wordIndex); + const win = iframe.contentWindow; + if (!wordEl || !win) return; + const cx = box.x + box.width / 2; + const cy = box.y + box.height / 2; + const startAngle = Math.atan2(e.clientY - cy, e.clientX - cx) * (180 / Math.PI); + const state = readGsapTransform(getOrCreateWrapper(wordEl), win); + interactionRef.current = { + type: "rotate", wordEl, segmentId: box.segmentId, + centerX: cx, centerY: cy, + startAngle, origTX: state.x, origTY: state.y, + origRotation: state.rotation, origScale: state.scale, + }; + }, [iframeRef]); + + /** Get iframe contentWindow, needed for gsap calls */ + const getIframeWin = useCallback((): Window | null => { + try { return iframeRef.current?.contentWindow ?? null; } catch { return null; } + }, [iframeRef]); + + // --- Unified pointer move --- + const handlePointerMove = useCallback((e: React.PointerEvent) => { + const i = interactionRef.current; + if (!i) return; + const win = getIframeWin(); + if (!win) return; + + if (i.type === "move") { + const cssScale = getCssScale(); + const dx = (e.clientX - i.startMX) / cssScale; + const dy = (e.clientY - i.startMY) / cssScale; + writeTransform(i.wordEl, win, i.origTX + dx, i.origTY + dy, i.origScale, i.origRotation); + } else if (i.type === "scale") { + const dx = e.clientX - i.startMX; + const factor = 1 + dx / Math.max(i.startWidth, 50); + const newScale = Math.max(0.1, i.origScale * factor); + writeTransform(i.wordEl, win, i.origTX, i.origTY, newScale, i.origRotation); + } else if (i.type === "rotate") { + const angle = Math.atan2(e.clientY - i.centerY, e.clientX - i.centerX) * (180 / Math.PI); + const delta = angle - i.startAngle; + writeTransform(i.wordEl, win, i.origTX, i.origTY, i.origScale, i.origRotation + delta); + } + }, [getCssScale, getIframeWin]); + + // --- Unified pointer up — sync back to store --- + const handlePointerUp = useCallback(() => { + const i = interactionRef.current; + if (i) { + const win = getIframeWin(); + if (win) syncToStore(i.segmentId, i.wordEl, win); + interactionRef.current = null; + } + }, [getIframeWin]); + + const handleBackgroundClick = useCallback((e: React.MouseEvent) => { + if (e.target === e.currentTarget) clearSelection(); + }, [clearSelection]); + + if (!isEditMode) return null; + + return ( +
+ {wordBoxes.map((box) => { + const isSelected = selectedSegmentIds.has(box.segmentId); + return ( +
{ e.stopPropagation(); selectSegment(box.segmentId, e.shiftKey); }} + onPointerDown={(e) => { + if (isSelected) startMove(box.groupIndex, box.wordIndex, box.segmentId, e); + }} + > + {isSelected && ( + <> + {/* Rotation handle — circle above the box */} +
startRotate(box, e)} + /> + {/* Line from box to rotation handle */} +
+ {/* Scale handles — four corners */} + {[ + { right: -HANDLE / 2, bottom: -HANDLE / 2, cursor: "nwse-resize" }, + { left: -HANDLE / 2, top: -HANDLE / 2, cursor: "nwse-resize" }, + { right: -HANDLE / 2, top: -HANDLE / 2, cursor: "nesw-resize" }, + { left: -HANDLE / 2, bottom: -HANDLE / 2, cursor: "nesw-resize" }, + ].map((pos, idx) => ( +
startScale(box.groupIndex, box.wordIndex, box.segmentId, e)} + /> + ))} + + )} +
+ ); + })} +
+ ); +}); diff --git a/packages/studio/src/captions/components/CaptionPropertyPanel.tsx b/packages/studio/src/captions/components/CaptionPropertyPanel.tsx new file mode 100644 index 000000000..ba75a8222 --- /dev/null +++ b/packages/studio/src/captions/components/CaptionPropertyPanel.tsx @@ -0,0 +1,294 @@ +import { memo, useCallback, useState } from "react"; +import { useCaptionStore } from "../store"; +import type { CaptionStyle } from "../types"; +import { CaptionAnimationPanel } from "./CaptionAnimationPanel"; + +// --------------------------------------------------------------------------- +// Helper Components +// --------------------------------------------------------------------------- + +function Section({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+
+ + {label} + +
+
{children}
+
+ ); +} + +function Row({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ {label} +
{children}
+
+ ); +} + +const inputCls = + "w-full bg-neutral-900 border border-neutral-800 rounded px-1.5 py-0.5 text-2xs text-neutral-200 font-mono outline-none focus:border-neutral-600"; + +// --------------------------------------------------------------------------- +// Main component +// --------------------------------------------------------------------------- + +interface CaptionPropertyPanelProps { + iframeRef: React.RefObject; +} + +export const CaptionPropertyPanel = memo(function CaptionPropertyPanel({ + iframeRef, +}: CaptionPropertyPanelProps) { + const model = useCaptionStore((s) => s.model); + const selectedSegmentIds = useCaptionStore((s) => s.selectedSegmentIds); + const selectedGroupId = useCaptionStore((s) => s.selectedGroupId); + const updateSelectedStyle = useCaptionStore((s) => s.updateSelectedStyle); + const updateGroupStyle = useCaptionStore((s) => s.updateGroupStyle); + + const [activeTab, setActiveTab] = useState<"style" | "animation">("style"); + + // Resolve effective style for the first selected segment + const firstSegmentId = selectedSegmentIds.size > 0 ? [...selectedSegmentIds][0] : undefined; + const firstSegment = model?.segments.get(firstSegmentId ?? ""); + + // Find the group that owns the first segment + let ownerGroupId: string | null = null; + if (model && firstSegmentId) { + for (const gid of model.groupOrder) { + const group = model.groups.get(gid); + if (group && group.segmentIds.includes(firstSegmentId)) { + ownerGroupId = gid; + break; + } + } + } + + const groupStyle = ownerGroupId ? model?.groups.get(ownerGroupId)?.style : undefined; + const segmentOverrides = firstSegment?.style ?? {}; + + // Merge group style with segment overrides for display + const effectiveStyle: Partial = { + ...groupStyle, + ...segmentOverrides, + }; + + + /** + * Apply a CSS style change to selected word elements in the iframe DOM in real time. + * Maps CaptionStyle property names to CSS properties. + */ + const applyToIframeDom = useCallback( + (updates: Partial) => { + const iframe = iframeRef.current; + if (!iframe || !model) return; + let doc: Document | null = null; + try { + doc = iframe.contentDocument; + } catch { + return; + } + if (!doc) return; + + const groupEls = doc.querySelectorAll(".caption-group"); + + // Build list of word elements to update + const targetEls: HTMLElement[] = []; + for (const segId of selectedSegmentIds) { + for (let gi = 0; gi < model.groupOrder.length; gi++) { + const group = model.groups.get(model.groupOrder[gi]); + if (!group) continue; + const wi = group.segmentIds.indexOf(segId); + if (wi < 0) continue; + const groupEl = groupEls[gi]; + if (!groupEl) continue; + // Resolve word span, handling wrappers + const children = groupEl.children; + let idx = 0; + for (const child of children) { + const c = child as HTMLElement; + if (c.dataset.captionWrapper === "true") { + const inner = c.querySelector(":scope > span"); + if (inner && idx === wi) { targetEls.push(inner); break; } + } else if (c.tagName === "SPAN") { + if (idx === wi) { targetEls.push(c); break; } + } + idx++; + } + break; + } + } + + // Apply transform updates via gsap.set on the WRAPPER (not the word span) + const hasTransform = updates.x !== undefined || updates.y !== undefined || + updates.scaleX !== undefined || updates.scaleY !== undefined || updates.rotation !== undefined; + + if (hasTransform) { + try { + const iframeGsap = (iframeRef.current?.contentWindow as unknown as { + gsap?: { set: (el: HTMLElement, props: Record) => void; + getProperty: (el: HTMLElement, prop: string) => number }; + })?.gsap; + if (iframeGsap) { + for (const el of targetEls) { + // Get or create wrapper + let wrapper = el.parentElement; + if (!wrapper || wrapper.dataset.captionWrapper !== "true") { + wrapper = doc.createElement("span") as HTMLElement; + wrapper.style.display = "inline-block"; + wrapper.dataset.captionWrapper = "true"; + el.parentNode?.insertBefore(wrapper, el); + wrapper.appendChild(el); + } + // Read current wrapper state and merge with updates + const curX = iframeGsap.getProperty(wrapper, "x") || 0; + const curY = iframeGsap.getProperty(wrapper, "y") || 0; + const curScale = iframeGsap.getProperty(wrapper, "scale") || 1; + const curRotation = iframeGsap.getProperty(wrapper, "rotation") || 0; + iframeGsap.set(wrapper, { + x: updates.x ?? curX, + y: updates.y ?? curY, + scale: updates.scaleX ?? curScale, + rotation: updates.rotation ?? curRotation, + }); + } + } + } catch { /* cross-origin */ } + } + }, + [iframeRef, model, selectedSegmentIds], + ); + + // All hooks must be called before any early return + const handleStyleChange = useCallback( + (updates: Partial) => { + if (selectedGroupId) { + updateGroupStyle(selectedGroupId, updates); + } else { + updateSelectedStyle(updates); + } + applyToIframeDom(updates); + }, + [selectedGroupId, updateGroupStyle, updateSelectedStyle, applyToIframeDom], + ); + + // Empty state — after all hooks + if (selectedSegmentIds.size === 0) { + return ( +
+

Select caption words to edit their style

+
+ ); + } + + // --------------------------------------------------------------------------- + // Derived style values with fallbacks + // --------------------------------------------------------------------------- + + const x = effectiveStyle.x ?? 0; + const y = effectiveStyle.y ?? 0; + const rotation = effectiveStyle.rotation ?? 0; + const scaleX = effectiveStyle.scaleX ?? 1; + + // Count label + const countLabel = selectedSegmentIds.size === 1 + ? "1 word" + : `${selectedSegmentIds.size} words`; + + return ( +
+ {/* Header */} +
+
+ + {countLabel} + +
+ {/* Tab switcher */} +
+ + +
+
+ + {/* Animation tab */} + {activeTab === "animation" && } + + {/* Style tab — Transform only */} + {activeTab === "style" && ( +
+
+ + handleStyleChange({ x: Number(e.target.value) })} + className={inputCls} + /> + + + + handleStyleChange({ y: Number(e.target.value) })} + className={inputCls} + /> + +
+ +
+ + + handleStyleChange({ + scaleX: Number(e.target.value), + scaleY: Number(e.target.value), + }) + } + className={inputCls} + /> + + + + handleStyleChange({ rotation: Number(e.target.value) })} + className={inputCls} + /> + +
+
+ )} +
+ ); +}); diff --git a/packages/studio/src/captions/components/CaptionTimeline.tsx b/packages/studio/src/captions/components/CaptionTimeline.tsx new file mode 100644 index 000000000..dfa2a1773 --- /dev/null +++ b/packages/studio/src/captions/components/CaptionTimeline.tsx @@ -0,0 +1,187 @@ +import { memo, useCallback, useRef } from "react"; +import { useCaptionStore } from "../store"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const GROUP_COLORS = [ + "#3CE6AC", + "#FF6B6B", + "#4ECDC4", + "#FFE66D", + "#A78BFA", + "#F472B6", + "#34D399", + "#FB923C", + "#60A5FA", + "#C084FC", +]; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface CaptionTimelineProps { + pixelsPerSecond: number; + onSeek?: (time: number) => void; +} + +interface DragState { + segId: string; + edge: "start" | "end"; + originalStart: number; + originalEnd: number; + startX: number; +} + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- + +export const CaptionTimeline = memo(function CaptionTimeline({ + pixelsPerSecond, + onSeek, +}: CaptionTimelineProps) { + const model = useCaptionStore((s) => s.model); + const selectedSegmentIds = useCaptionStore((s) => s.selectedSegmentIds); + const selectSegment = useCaptionStore((s) => s.selectSegment); + const updateSegmentTiming = useCaptionStore((s) => s.updateSegmentTiming); + const splitGroup = useCaptionStore((s) => s.splitGroup); + + const dragRef = useRef(null); + + const handleEdgePointerDown = useCallback( + ( + e: React.PointerEvent, + segId: string, + edge: "start" | "end", + originalStart: number, + originalEnd: number, + ) => { + e.stopPropagation(); + e.preventDefault(); + (e.target as HTMLElement).setPointerCapture(e.pointerId); + dragRef.current = { segId, edge, originalStart, originalEnd, startX: e.clientX }; + }, + [], + ); + + const handlePointerMove = useCallback( + (e: React.PointerEvent) => { + const drag = dragRef.current; + if (!drag) return; + + const delta = (e.clientX - drag.startX) / pixelsPerSecond; + + if (drag.edge === "start") { + const newStart = Math.max(0, drag.originalStart + delta); + const clampedStart = Math.min(newStart, drag.originalEnd - 0.05); + updateSegmentTiming(drag.segId, clampedStart, drag.originalEnd); + } else { + const newEnd = Math.max(drag.originalStart + 0.05, drag.originalEnd + delta); + const clampedEnd = Math.max(0, newEnd); + updateSegmentTiming(drag.segId, drag.originalStart, clampedEnd); + } + }, + [pixelsPerSecond, updateSegmentTiming], + ); + + const handlePointerUp = useCallback(() => { + dragRef.current = null; + }, []); + + const handleBlockClick = useCallback( + (e: React.MouseEvent, segId: string) => { + e.stopPropagation(); + selectSegment(segId, e.shiftKey); + }, + [selectSegment], + ); + + const handleBlockDoubleClick = useCallback( + (e: React.MouseEvent, groupId: string, segId: string) => { + e.stopPropagation(); + splitGroup(groupId, segId); + }, + [splitGroup], + ); + + const handleTrackClick = useCallback( + (e: React.MouseEvent) => { + if (!onSeek) return; + const rect = (e.currentTarget as HTMLDivElement).getBoundingClientRect(); + const x = e.clientX - rect.left - 32; + const time = Math.max(0, x / pixelsPerSecond); + onSeek(time); + }, + [onSeek, pixelsPerSecond], + ); + + if (!model) return null; + + return ( +
+ {model.groupOrder.map((groupId, groupIdx) => { + const group = model.groups.get(groupId); + if (!group) return null; + const color = GROUP_COLORS[groupIdx % GROUP_COLORS.length]; + + return group.segmentIds.map((segId) => { + const seg = model.segments.get(segId); + if (!seg) return null; + + const left = 32 + seg.start * pixelsPerSecond; + const width = Math.max((seg.end - seg.start) * pixelsPerSecond, 4); + const isSelected = selectedSegmentIds.has(segId); + + return ( +
handleBlockClick(e, segId)} + onDoubleClick={(e) => handleBlockDoubleClick(e, groupId, segId)} + > + {/* Left edge drag handle */} +
handleEdgePointerDown(e, segId, "start", seg.start, seg.end)} + /> + + {/* Text label */} + + {seg.text} + + + {/* Right edge drag handle */} +
handleEdgePointerDown(e, segId, "end", seg.start, seg.end)} + /> +
+ ); + }); + })} +
+ ); +}); diff --git a/packages/studio/src/captions/generator.test.ts b/packages/studio/src/captions/generator.test.ts new file mode 100644 index 000000000..94c4bce47 --- /dev/null +++ b/packages/studio/src/captions/generator.test.ts @@ -0,0 +1,279 @@ +// @vitest-environment node +import { describe, it, expect } from "vitest"; +import { generateCaptionHtml } from "./generator.js"; +import { buildCaptionModel, TranscriptWord } from "./parser.js"; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const SAMPLE_TRANSCRIPT: TranscriptWord[] = [ + { text: "We", start: 0.1, end: 0.3 }, + { text: "asked", start: 0.4, end: 0.6 }, + { text: "what", start: 0.7, end: 0.9 }, + { text: "you", start: 1.0, end: 1.2 }, + { text: "needed.", start: 1.3, end: 1.8 }, + { text: "Forty-seven", start: 1.9, end: 2.3 }, + { text: "percent", start: 2.4, end: 2.7 }, +]; + +function buildTestModel(wordsPerGroup = 5) { + return buildCaptionModel(SAMPLE_TRANSCRIPT, { + width: 1920, + height: 1080, + duration: 16, + wordsPerGroup, + }); +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("generateCaptionHtml", () => { + describe("HTML structure", () => { + it("wraps output in a