feat(media-use): v2 media OS core (resolve cascade, providers, local generation, telemetry) + retire hyperframes-media

2026-09-11 06:30:03 +00:00 · 2026-07-06 23:41:05 -04:00
parent 676f461f6a
commit 5fe957363d
105 changed files with 2362 additions and 287 deletions
@@ -0,0 +1,279 @@
+// Declarative table of USER-INSTALLED local models, for the spec-gated fallback.
+//
+// These models run on the user's own machine for their own use; media-use
+// recommends, spec-checks, and assists install; it does not bundle, redistribute,
+// or sell them. Because nothing is redistributed, selection is purely by
+// quality / size / spec-fit / word-timestamp support (there is deliberately NO
+// license field gating availability).
+//
+// Tiers (`small`|`medium`|`large`|`xlarge`) are human labels; `needs.ramMB` is
+// what selection actually gates on. selectModel() returns the best model that
+// fits the machine's AVAILABLE RAM, best-first: by explicit `rank` when set
+// (quality that is NOT size, e.g. ASR), else by RAM footprint (the quality
+// proxy for generation). No fit -> recommend the CLI/cloud path.
+//
+// Picks reflect the 2026 research pass, verified live where noted.
+
+export const CAPABILITIES = ["tts", "asr", "upscale", "videogen", "imagegen"];
+
+const MODELS = {
+  tts: [
+    {
+      id: "kokoro",
+      tier: "medium",
+      sizeMB: 330,
+      needs: { ramMB: 2048, gpu: false },
+      wordTimestamps: "native",
+      install: "pip install kokoro",
+      invoke: "python -m kokoro --text {text} --voice {voice} --out {out}",
+      notes: "CPU, faster-than-realtime, native per-word timestamps. Default floor.",
+    },
+    {
+      id: "fish-speech",
+      tier: "large",
+      sizeMB: 1100,
+      needs: { ramMB: 16000, gpu: true, vramMB: 12000 },
+      wordTimestamps: "whisperx", // needs forced alignment (run ASR over output)
+      install: "pip install fish-speech",
+      invoke: "fish-speech synth --text {text} --ref {ref} --out {out}",
+      notes: "Expressive zero-shot voice cloning; meeting pick. WhisperX for word timing.",
+    },
+  ],
+  asr: [
+    // Parakeet is BETTER than Whisper yet SMALLER (0.6B vs 1.5B), so quality is
+    // not size here: `rank` pins it ahead of whisper regardless of footprint.
+    // Open ASR Leaderboard avg WER: Parakeet ~6.05% vs whisper-large-v3 7.44%
+    // (~19% better); on NOISY test-other 4.73% vs 5.96%, and whisper-v3
+    // hallucinated to 308% WER on meetings where Parakeet held. 5-10x faster.
+    //
+    // Cohere Transcribe 2B tops the leaderboard (5.42%) and is nominally the most
+    // accurate, but its mlx-audio community MLX quants (4bit AND 8bit, with and
+    // without --language en) produced multilingual token-soup garbage AND ran
+    // 40-70x slower than Parakeet on a 24GB Mac (live-tested 2026-07). Excluded
+    // until the mlx-audio Cohere decoder stabilizes; Parakeet is the default.
+    {
+      id: "parakeet-mlx",
+      tier: "small",
+      rank: 0,
+      sizeMB: 2400,
+      needs: { ramMB: 4000, gpu: true },
+      wordTimestamps: "tokens", // sub-word tokens; merged to words by parakeet-words.mjs
+      repo: "mlx-community/parakeet-tdt-0.6b-v3",
+      install:
+        "uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx",
+      invoke:
+        "parakeet-mlx {audio} --model mlx-community/parakeet-tdt-0.6b-v3 --output-format json --output-dir {outdir}",
+      notes:
+        "NVIDIA Parakeet-TDT 0.6B via parakeet-mlx. VERIFIED on 24GB: accurate transcript, ~3s (cached model) for 8s audio, word timestamps drive transcript-cut. English + 25 European languages. Beats whisper.cpp on accuracy (6.05% vs 7.44% WER) AND speed (5-10x).",
+    },
+    {
+      id: "whisperx",
+      tier: "medium",
+      rank: 1,
+      sizeMB: 1500,
+      needs: { ramMB: 4096, gpu: false },
+      wordTimestamps: "native", // faster-whisper + wav2vec2 forced alignment
+      install: "pip install whisperx",
+      invoke: "whisperx {audio} --output_format json --out {out}",
+      notes:
+        "CPU-only fallback (no GPU): faster-whisper + wav2vec2 forced alignment, native word timestamps. The packaged `hyperframes transcribe` (whisper.cpp) is the zero-setup baseline below this.",
+    },
+  ],
+  upscale: [
+    {
+      id: "real-esrgan",
+      tier: "medium",
+      sizeMB: 70,
+      needs: { ramMB: 2048, gpu: false },
+      wordTimestamps: false,
+      install: "brew install real-esrgan-ncnn-vulkan  # or download the ncnn binary",
+      invoke: "realesrgan-ncnn-vulkan -i {in} -o {out} -s 4",
+      notes: "ncnn-vulkan binary, CPU-capable. GFPGAN for faces.",
+    },
+    {
+      id: "seedvr2",
+      tier: "large",
+      sizeMB: 6000,
+      needs: { ramMB: 24000, gpu: true, vramMB: 16000 },
+      wordTimestamps: false,
+      install: "pip install seedvr2",
+      invoke: "seedvr2 upscale --in {in} --out {out}",
+      notes: "Diffusion upscaler, GPU-only. Video2X for video.",
+    },
+  ],
+  videogen: [
+    // 2026-07 X research pass + live verification on a 24GB M-series Mac.
+    // The Mac-local video story is LTX 2.3 on MLX via dgrauet/ltx-2-mlx (the
+    // pipeline these weights were converted for; also powers Phosphene).
+    // Wan 2.x MLX exists only as A14B conversions (too large for consumer
+    // unified memory); revisit when a 5B Wan MLX conversion lands.
+    // IMPORTANT: download the weights with a targeted include list first;
+    // pointing tools at the repo blind snapshot-downloads all 60 GB:
+    //   hf download dgrauet/ltx-2.3-mlx-q4 --include \
+    //     transformer-distilled-1.1.safetensors connector.safetensors \
+    //     "vae_*.safetensors" audio_vae.safetensors vocoder.safetensors "*.json"
+    {
+      id: "ltx-2.3-mlx-q4",
+      tier: "medium",
+      sizeMB: 20000, // distilled subset; gemma-3-12b-4bit text encoder adds ~7GB
+      needs: { ramMB: 16384, gpu: true },
+      wordTimestamps: false,
+      install:
+        "git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras",
+      invoke:
+        "ltx-2-mlx generate --prompt {prompt} --distilled --low-ram --model dgrauet/ltx-2.3-mlx-q4 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}",
+      notes:
+        "LTX 2.3 int4 on MLX. Verified on 24GB unified: 512x320 x 33 frames in ~19 min cold (incl. text-encoder download), t2v with audio. Dims must be multiples of 64. i2v, retake/extend, keyframe interpolation supported.",
+    },
+    {
+      id: "ltx-2.3-mlx-bf16",
+      tier: "large",
+      sizeMB: 45000,
+      needs: { ramMB: 32768, gpu: true },
+      wordTimestamps: false,
+      install:
+        "git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras",
+      invoke:
+        "ltx-2-mlx generate --prompt {prompt} --two-stage --model dgrauet/ltx-2.3-mlx-bf16 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}",
+      notes:
+        "Full-precision two-stage pipeline (upstream production default). 32GB with --low-ram block streaming; 64-128GB Macs for long/HD runs (the 25s multi-scene spots seen in the wild).",
+    },
+  ],
+  imagegen: [
+    // 2026-07 X research + live verification on a 24GB M-series Mac. mflux
+    // (FLUX-on-MLX) is the Mac-native runner; FLUX is the quality leader. Two
+    // hard-won findings baked into `needs.ramMB`:
+    //   1. The OFFICIAL FLUX repos are HF-gated (license wall). Point --path at a
+    //      non-gated community 4-bit re-upload (self-contained, incl. VAE).
+    //   2. Without --low-ram, FLUX's T5-XXL text encoder + transformer blow past
+    //      24GB into swap: a 768x512 run took 90 MINUTES. With --low-ram (streams
+    //      components from disk) the SAME machine did 512x512 in ~20s at 7.6GB
+    //      free. So the medium tier's needs.ramMB is the streamed floor, not the
+    //      resident footprint; the large tiers are the no-streaming thresholds.
+    // The runner resolves `repo` to a local snapshot (hf download) before --path;
+    // a bare repo id in --path breaks mlx unflatten.
+    {
+      id: "flux-schnell-mflux-q4",
+      tier: "medium",
+      sizeMB: 8700,
+      needs: { ramMB: 8000, gpu: true },
+      repo: "dhairyashil/FLUX.1-schnell-mflux-4bit",
+      wordTimestamps: false,
+      install: "uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6",
+      invoke:
+        "mflux-generate --model schnell --path {model_path} --low-ram --steps 4 --prompt {prompt} --width {w} --height {h} --seed {seed} --output {out}",
+      notes:
+        "FLUX.1 schnell int4. VERIFIED on 24GB (7.6GB free): --low-ram 512x512 in ~20s, photoreal. --low-ram is MANDATORY at this tier (streams to avoid swap). Few-step, fast.",
+    },
+    {
+      id: "flux2-klein-mflux-q4",
+      tier: "large",
+      sizeMB: 12000,
+      needs: { ramMB: 32000, gpu: true },
+      repo: "Runpod/FLUX.2-klein-4B-mflux-4bit",
+      wordTimestamps: false,
+      install: "uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux",
+      invoke:
+        "mflux-generate --base-model flux2-klein-4b --path {model_path} --steps 8 --prompt {prompt} --width {w} --height {h} --seed {seed} --output {out}",
+      notes:
+        "FLUX.2 Klein 4B int4 (most-downloaded mflux community repo). Newer, higher quality than schnell; full-resident (no streaming) so needs 32GB+ to stay fast. Needs mflux >= 0.18 for the flux2-klein base model.",
+    },
+    {
+      id: "qwen-image-mflux",
+      tier: "xlarge",
+      sizeMB: 40000,
+      needs: { ramMB: 64000, gpu: true },
+      repo: "Qwen/Qwen-Image",
+      wordTimestamps: false,
+      install: "uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux",
+      invoke:
+        "mflux-generate --base-model qwen --steps 20 --prompt {prompt} --width {w} --height {h} --seed {seed} --output {out}",
+      notes:
+        "Qwen-Image, top-tier quality. Heavy: 'several minutes' even on 128GB M4 Max, 'almost fried' a 32GB M4 Pro. 64GB+ only. Below that, the cloud upsell (codex) is faster and better.",
+    },
+  ],
+};
+
+function tableFor(capability) {
+  const t = MODELS[capability];
+  if (!t) throw new Error(`unknown local-model capability: ${capability}`);
+  return t;
+}
+
+/** All local models for a capability. */
+export function listModels(capability) {
+  return tableFor(capability).slice();
+}
+
+/** Does this machine meet a model's needs? Apple Silicon unified memory counts as VRAM. */
+export function meetsSpecs(model, specs) {
+  const n = model.needs || {};
+  // Gate on AVAILABLE RAM when the probe reported it (the real budget with the
+  // OS + open apps resident); fall back to total RAM otherwise. Older specs
+  // objects (and unit fixtures) that only set ramMB keep working unchanged.
+  const budget = specs.availableRamMB ?? specs.ramMB;
+  if (n.ramMB && budget < n.ramMB) return false;
+  if (n.gpu && !specs.gpu?.present) return false;
+  if (n.vramMB) {
+    const vram = specs.gpu?.vramMB ?? 0;
+    if (vram < n.vramMB) return false;
+  }
+  return true;
+}
+
+// "Best model the machine can run" == best-first among those that fit. Ordering:
+//   1. explicit `rank` (lower = better) when a model declares it. Needed where
+//      quality is NOT size: Parakeet-0.6B beats Whisper-large-1.5B at ASR, so
+//      footprint would pick the wrong one.
+//   2. otherwise RAM footprint descending, the quality proxy for generation
+//      (a 40GB image model out-renders a 12GB one).
+function rankedByPreference(table) {
+  return [...table].sort((a, b) => {
+    const ra = a.rank ?? Infinity;
+    const rb = b.rank ?? Infinity;
+    if (ra !== rb) return ra - rb;
+    return (b.needs?.ramMB ?? 0) - (a.needs?.ramMB ?? 0);
+  });
+}
+
+/**
+ * Pick the best local model the machine can run for a capability: the
+ * highest-footprint model that fits the available-RAM budget (and GPU/VRAM).
+ * `preferTier` pins the search to one tier (e.g. force a smaller/faster model).
+ * Returns `{ model, tier }`, or `{ recommend: "cli", reason }` when nothing fits.
+ */
+export function selectModel(capability, specs, { preferTier } = {}) {
+  const table = tableFor(capability);
+  const pool = preferTier ? table.filter((m) => m.tier === preferTier) : table;
+  for (const model of rankedByPreference(pool)) {
+    if (meetsSpecs(model, specs)) return { model, tier: model.tier };
+  }
+  const smallest = table.reduce((a, b) => (a.sizeMB <= b.sizeMB ? a : b));
+  return {
+    recommend: "cli",
+    reason: `machine does not meet specs for any local ${capability} model (smallest needs ~${smallest.needs.ramMB}MB RAM${smallest.needs.gpu ? " + GPU" : ""}); use the CLI path instead`,
+  };
+}
+
+/**
+ * Agent-facing ladder: every model for a capability, best-first, each flagged
+ * with whether it fits this machine and why. Lets the agent see the RAM-graded
+ * options and choose (e.g. trade the auto-picked best for a smaller/faster one,
+ * or step up to a cloud upsell) rather than only getting one auto-selection.
+ */
+export function describeModelLadder(capability, specs) {
+  const budget = specs.availableRamMB ?? specs.ramMB;
+  return rankedByPreference(tableFor(capability)).map((model) => {
+    const fits = meetsSpecs(model, specs);
+    return {
+      id: model.id,
+      tier: model.tier,
+      needsRamMB: model.needs?.ramMB ?? 0,
+      fits,
+      reason: fits
+        ? `fits (needs ~${model.needs?.ramMB}MB, ${budget}MB available)`
+        : `too big (needs ~${model.needs?.ramMB}MB${model.needs?.gpu ? " + GPU" : ""}, ${budget}MB available)`,
+      notes: model.notes,
+    };
+  });
+}