From 90df164a22089aa0c6cefdc7fc00ffdee54853a6 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 26 Aug 2026 22:03:15 +0000 Subject: [PATCH] fix(media-use): repoint the dead videogen tier, demote past unusable models `LOCAL_MODELS.videogen`'s `large` tier named `dgrauet/ltx-2.3-mlx-bf16`, which returns HTTP 401 and cannot be downloaded at all. It was not a dormant entry: `rankedByPreference` sorts by descending `needs.ramMB` when no `rank` is set, so the largest fitting tier is tried FIRST by design. Any machine clearing 32 GB *available* RAM selected the dead entry, `ltxVideoGenerate` caught the failure and returned a bare `null`, and since `ltx.local` is last in `["heygen.video", "ltx.local"]` and network providers are skipped under `--local-only` (`registry.mjs:206`), local video generation failed outright instead of falling back to the tier that works. It survived review because the table landed with "live verification on a 24GB M-series Mac" - and a 24 GB machine cannot select a 32 GB tier, so that entry was unreachable on the only machine that validated it. The unit fixtures inherit the same ceiling (`fittingSpecs` is 20000MB), so every existing test exercised the medium tier alone. Two changes: 1. Repoint to `dgrauet/ltx-2.3-mlx-q8` (reachable) and correct `sizeMB` from 45000 to 28800. Measured against the HF API: the q8 repo totals 87.5 GB, and the registry's own targeted `--include` subset is 28.76 GB. That matches the sibling q4 entry's convention (`sizeMB: 20000` vs a measured 19.48 GB subset), so 45000 was wrong under either reading. `--low-ram` is added because the entry's own note calls it required at this tier's 32 GB floor, and the invoke omitted it. 2. A repoint alone is one bad URL from a repeat, so add the missing recovery. `selectModelLadder` returns every fitting model best-first; `selectModel`'s pick is now defined as that list's head. All three sites that previously selected exactly one model and failed terminally walk the ladder instead, demoting past a tier that cannot run here - gated weights, runner off PATH, an OOM at a tier that nominally fits: - `ltx-video-provider.mjs` (videogen, the reported failure) - `mflux-provider.mjs` (imagegen - same shape, and its 32 GB/64 GB tiers are equally unverifiable on a 24 GB machine) - `local-run.mjs` (tts/asr/upscale - `fish-speech` missing should still get you Kokoro) Every demotion is logged rather than silent, so a quietly smaller model is never mistaken for the tier the machine nominally qualified for. Also fixes the `install` string both videogen entries share: it ended at `uv sync --all-extras`, which leaves the entry point in `.venv/bin`, so the "`ltx-2-mlx` not on PATH" hint named a command that following the instruction would not put on PATH. The q8 tier is NOT live-verified - no 32 GB+ Apple Silicon machine was available - and its notes say so. Shipping it unverified is safe precisely because of change 2: a wrong tier now costs one failed attempt, not the whole local path. - Rames Jusso Co-Authored-By: Claude Opus 5 (1M context) --- skills-manifest.json | 4 +- skills/media-use/scripts/lib/local-models.mjs | 47 ++++-- .../scripts/lib/local-models.test.mjs | 66 ++++++++ skills/media-use/scripts/lib/local-run.mjs | 66 ++++---- .../media-use/scripts/lib/local-run.test.mjs | 37 +++++ .../scripts/lib/ltx-video-provider.mjs | 106 ++++++------ .../scripts/lib/ltx-video-provider.test.mjs | 89 +++++++++- .../media-use/scripts/lib/mflux-provider.mjs | 152 +++++++++++------- .../scripts/lib/mflux-provider.test.mjs | 115 +++++++++++++ 9 files changed, 530 insertions(+), 152 deletions(-) create mode 100644 skills/media-use/scripts/lib/mflux-provider.test.mjs diff --git a/skills-manifest.json b/skills-manifest.json index 99ff50894..fd44eaec7 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -50,8 +50,8 @@ "files": 12 }, "media-use": { - "hash": "1b0ce647f5c7df95", - "files": 152 + "hash": "36eedcb77a5f40f3", + "files": 153 }, "motion-graphics": { "hash": "853ac75cbab69036", diff --git a/skills/media-use/scripts/lib/local-models.mjs b/skills/media-use/scripts/lib/local-models.mjs index af292ff32..ca392f705 100644 --- a/skills/media-use/scripts/lib/local-models.mjs +++ b/skills/media-use/scripts/lib/local-models.mjs @@ -12,6 +12,10 @@ // (quality that is NOT size, e.g. ASR), else by RAM footprint (the quality // proxy for generation). No fit -> recommend the CLI/cloud path. // +// selectModelLadder() returns EVERY fitting model in that same order. Callers +// that can retry walk it so ONE unusable entry (gated weights, a missing +// binary, an OOM) demotes to the next tier instead of killing the local path. +// // Picks reflect the 2026 research pass, verified live where noted. export const CAPABILITIES = ["tts", "asr", "upscale", "videogen", "imagegen"]; @@ -102,13 +106,17 @@ const MODELS = { }, ], videogen: [ - // 2026-07 X research pass + live verification on a 24GB M-series Mac. + // 2026-07 X research pass + live verification on a 24GB M-series Mac - + // which reaches the q4 tier only: a 24GB machine cannot select the 32GB + // entry below it, so that tier's claims stay unverified until someone + // runs it on a 32GB+ machine. // The Mac-local video story is LTX 2.3 on MLX via dgrauet/ltx-2-mlx (the // pipeline these weights were converted for; also powers Phosphene). // Wan 2.x MLX exists only as A14B conversions (too large for consumer // unified memory); revisit when a 5B Wan MLX conversion lands. // IMPORTANT: download the weights with a targeted include list first; - // pointing tools at the repo blind snapshot-downloads all 60 GB: + // pointing tools at the repo blind snapshot-downloads the lot (60 GB q4, + // 88 GB q8): // hf download dgrauet/ltx-2.3-mlx-q4 --include \ // transformer-distilled-1.1.safetensors connector.safetensors \ // "vae_*.safetensors" audio_vae.safetensors vocoder.safetensors "*.json" @@ -119,24 +127,24 @@ const MODELS = { needs: { ramMB: 16384, gpu: true }, wordTimestamps: false, install: - "git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras", + 'git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras && export PATH="$PWD/.venv/bin:$PATH"', invoke: "ltx-2-mlx generate --prompt {prompt} --distilled --low-ram --model dgrauet/ltx-2.3-mlx-q4 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}", notes: "LTX 2.3 int4 on MLX. Verified on 24GB unified: 512x320 x 33 frames in ~19 min cold (incl. text-encoder download), t2v with audio. Dims must be multiples of 64. i2v, retake/extend, keyframe interpolation supported.", }, { - id: "ltx-2.3-mlx-bf16", + id: "ltx-2.3-mlx-q8", tier: "large", - sizeMB: 45000, + sizeMB: 28800, needs: { ramMB: 32768, gpu: true }, wordTimestamps: false, install: - "git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras", + 'git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras && export PATH="$PWD/.venv/bin:$PATH"', invoke: - "ltx-2-mlx generate --prompt {prompt} --two-stage --model dgrauet/ltx-2.3-mlx-bf16 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}", + "ltx-2-mlx generate --prompt {prompt} --two-stage --low-ram --model dgrauet/ltx-2.3-mlx-q8 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}", notes: - "Full-precision two-stage pipeline (upstream production default). 32GB with --low-ram block streaming; 64-128GB Macs for long/HD runs (the 25s multi-scene spots seen in the wild).", + "LTX 2.3 int8 on MLX, two-stage (upstream production default; higher quality than the q4 distilled tier). Replaced dgrauet/ltx-2.3-mlx-bf16, which is gated (HTTP 401) and cannot be downloaded at all. sizeMB is the targeted include-list subset, measured 28.8GB. --low-ram matches this tier's 32GB floor (block streaming); 64-128GB Macs for long/HD runs. NOT live-verified on a 32GB+ machine - the q4 tier below is the verified one.", }, ], imagegen: [ @@ -247,6 +255,23 @@ function rankedByPreference(table) { }); } +/** + * Every local model for a capability this machine can actually run, best-first + * (same ordering as selectModel, whose pick is this list's head). + * + * Callers that can retry should walk the whole list: a table entry can be + * unusable for reasons no spec check can see - weights pulled or gated behind a + * login, the runner missing from PATH, an OOM at a tier that nominally fits. On + * a single-select call any one of those fails the entire local path, because the + * cascade cannot tell "this model is broken" from "nothing here fits you". + * Demoting to the next fitting tier is almost always what the user wanted. + */ +export function selectModelLadder(capability, specs, { preferTier } = {}) { + const table = tableFor(capability); + const pool = preferTier ? table.filter((m) => m.tier === preferTier) : table; + return rankedByPreference(pool).filter((model) => meetsSpecs(model, specs)); +} + /** * Pick the best local model the machine can run for a capability: the * highest-footprint model that fits the available-RAM budget (and GPU/VRAM). @@ -255,10 +280,8 @@ function rankedByPreference(table) { */ export function selectModel(capability, specs, { preferTier } = {}) { const table = tableFor(capability); - const pool = preferTier ? table.filter((m) => m.tier === preferTier) : table; - for (const model of rankedByPreference(pool)) { - if (meetsSpecs(model, specs)) return { model, tier: model.tier }; - } + const [model] = selectModelLadder(capability, specs, { preferTier }); + if (model) return { model, tier: model.tier }; const smallest = table.reduce((a, b) => (a.sizeMB <= b.sizeMB ? a : b)); return { recommend: "cli", diff --git a/skills/media-use/scripts/lib/local-models.test.mjs b/skills/media-use/scripts/lib/local-models.test.mjs index e1cc56368..f1e8d0204 100644 --- a/skills/media-use/scripts/lib/local-models.test.mjs +++ b/skills/media-use/scripts/lib/local-models.test.mjs @@ -4,6 +4,7 @@ import { listModels, meetsSpecs, selectModel, + selectModelLadder, describeModelLadder, CAPABILITIES, } from "./local-models.mjs"; @@ -152,3 +153,68 @@ test("ASR offers word-timestamp-capable models (better than plain whisper)", () "every ASR model must support word timestamps", ); }); + +// A machine that clears BOTH videogen tiers (the 32GB entry and the 16GB one). +// The existing fixtures deliberately sit under the large tier's floor, which is +// exactly how a dead 32GB entry stayed invisible: nothing could select it. +const bothVideogenTiers = { availableRamMB: 40000, gpu: { present: true } }; + +test("selectModelLadder returns every fitting model, best-first", () => { + const ladder = selectModelLadder("videogen", bothVideogenTiers); + assert.deepEqual( + ladder.map((m) => m.tier), + ["large", "medium"], + "both tiers fit 40GB, biggest first", + ); + assert.equal( + selectModel("videogen", bothVideogenTiers).model.id, + ladder[0].id, + "selectModel's pick is the ladder's head", + ); +}); + +test("selectModelLadder drops what the machine cannot run", () => { + const oneTier = selectModelLadder("videogen", { availableRamMB: 20000, gpu: { present: true } }); + assert.deepEqual( + oneTier.map((m) => m.tier), + ["medium"], + "20GB cannot reach the 32GB tier", + ); + assert.deepEqual( + selectModelLadder("videogen", { availableRamMB: 100, gpu: { present: true } }), + [], + "nothing fits -> empty ladder, and selectModel recommends the CLI", + ); + assert.equal( + selectModel("videogen", { availableRamMB: 100, gpu: { present: true } }).recommend, + "cli", + ); +}); + +test("selectModelLadder honours preferTier", () => { + const pinned = selectModelLadder("videogen", bothVideogenTiers, { preferTier: "medium" }); + assert.deepEqual( + pinned.map((m) => m.tier), + ["medium"], + "preferTier pins the ladder to one tier", + ); +}); + +test("an invoke that names an owner/repo model agrees with the entry id", () => { + // Guards a half-done repoint: moving an entry to different weights means + // changing BOTH the id and the --model argument. Change one and the table + // selects one model while the runner downloads another. + let checked = 0; + for (const cap of CAPABILITIES) { + for (const m of listModels(cap)) { + if (m.repo) continue; // entries with an explicit repo resolve through it + const named = /--model\s+(\S+)/.exec(m.invoke); + if (!named) continue; + const [, name] = named[1].split("/"); + if (!name) continue; // a bare model name, not an owner/repo id + assert.equal(name, m.id, `${cap}/${m.id}: invoke runs ${named[1]}`); + checked += 1; + } + } + assert.ok(checked > 0, "no entry pins an owner/repo model - guard would be vacuous"); +}); diff --git a/skills/media-use/scripts/lib/local-run.mjs b/skills/media-use/scripts/lib/local-run.mjs index b8b1bc6c2..4550f10f9 100644 --- a/skills/media-use/scripts/lib/local-run.mjs +++ b/skills/media-use/scripts/lib/local-run.mjs @@ -1,10 +1,12 @@ import { execFileSync } from "node:child_process"; -import { selectModel } from "./local-models.mjs"; +import { selectModel, selectModelLadder } from "./local-models.mjs"; import { probeSpecs } from "./specs.mjs"; // Run a USER-INSTALLED local model for a capability (tts/asr/upscale). -// Picks the best tier the machine supports (selectModel), checks the tool is on -// PATH, fills the model's invoke template, and runs it. Returns: +// Walks the tiers the machine supports best-first (selectModelLadder), checking +// the tool is on PATH, filling the model's invoke template, and running it. A +// tier whose tool is missing or whose run fails demotes to the next fitting +// tier, so one unusable entry does not fail the capability. Returns: // { model, tier, out } on success // { recommend:"install", model, command, reason } when the tool isn't installed // { recommend:"cli", reason } when no tier fits the machine @@ -35,30 +37,40 @@ export function runLocalModel(capability, opts = {}) { vars = {}, preferTier, } = opts; - const sel = selectModel(capability, specs, { preferTier }); - if (sel.recommend) return sel; // no tier fits -> recommend the CLI path + const ladder = selectModelLadder(capability, specs, { preferTier }); + // no tier fits at all -> recommend the CLI path (selectModel words the reason) + if (!ladder.length) return selectModel(capability, specs, { preferTier }); - const { model } = sel; - const bin = model.invoke.split(/\s+/)[0]; - try { - which(bin); - } catch { - return { - recommend: "install", - model: model.id, - command: model.install, - reason: `${model.id} not installed`, - }; + // Best tier first, demoting past any tier that cannot run here: a missing + // tool or a failed run at the top tier must not hide a lower tier that works + // (fish-speech absent should still get you Kokoro). The last tier's failure is + // what gets reported, since by then nothing local ran. + let lastFailure = null; + for (const model of ladder) { + const bin = model.invoke.split(/\s+/)[0]; + try { + which(bin); + } catch { + lastFailure = { + recommend: "install", + model: model.id, + command: model.install, + reason: `${model.id} not installed`, + }; + continue; + } + try { + exec(fill(model.invoke, vars)); + } catch (e) { + lastFailure = { + recommend: "install", + model: model.id, + command: model.install, + reason: e.message || String(e), + }; + continue; + } + return { model: model.id, tier: model.tier, out: vars.out }; } - try { - exec(fill(model.invoke, vars)); - } catch (e) { - return { - recommend: "install", - model: model.id, - command: model.install, - reason: e.message || String(e), - }; - } - return { model: model.id, tier: sel.tier, out: vars.out }; + return lastFailure; } diff --git a/skills/media-use/scripts/lib/local-run.test.mjs b/skills/media-use/scripts/lib/local-run.test.mjs index 7e9c883db..b88b24188 100644 --- a/skills/media-use/scripts/lib/local-run.test.mjs +++ b/skills/media-use/scripts/lib/local-run.test.mjs @@ -52,3 +52,40 @@ test("a failing run degrades to an install recommendation, never throws", () => }); assert.equal(r.recommend, "install"); }); + +test("a tier whose tool is missing demotes to the next tier that fits", () => { + // 64GB + GPU fits BOTH tts tiers, so the ladder has two rungs: fish-speech + // (its own binary) above Kokoro (`python -m kokoro`). fish-speech absent must + // not cost the user Kokoro. + const strongGpu = { ramMB: 64000, gpu: { present: true, vramMB: 24000 } }; + let ran = ""; + const r = runLocalModel("tts", { + specs: strongGpu, + which: (bin) => { + if (bin === "fish-speech") throw new Error("not found"); + }, + exec: (cmd) => { + ran = cmd; + }, + vars: { text: "hello", voice: "af_heart", out: "/tmp/v.wav" }, + }); + + assert.equal(r.model, "kokoro", "demoted past the missing fish-speech binary"); + assert.equal(r.tier, "medium"); + assert.match(ran, /kokoro/); +}); + +test("every fitting tier failing reports the last tier's install command", () => { + const strongGpu = { ramMB: 64000, gpu: { present: true, vramMB: 24000 } }; + const r = runLocalModel("tts", { + specs: strongGpu, + which: ok, + exec: () => { + throw new Error("boom"); + }, + vars: { text: "hi", out: "/tmp/v.wav" }, + }); + + assert.equal(r.recommend, "install"); + assert.equal(r.model, "kokoro", "the smallest fitting tier is the actionable one"); +}); diff --git a/skills/media-use/scripts/lib/ltx-video-provider.mjs b/skills/media-use/scripts/lib/ltx-video-provider.mjs index e7ca74748..9fb9a6755 100644 --- a/skills/media-use/scripts/lib/ltx-video-provider.mjs +++ b/skills/media-use/scripts/lib/ltx-video-provider.mjs @@ -3,7 +3,7 @@ import { existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { probeSpecs } from "./specs.mjs"; -import { buildArgv, selectModel } from "./local-models.mjs"; +import { buildArgv, selectModel, selectModelLadder } from "./local-models.mjs"; export async function ltxVideoGenerate( intent, @@ -12,59 +12,69 @@ export async function ltxVideoGenerate( pathExists = existsSync, ) { const specs = ctx?.specs || probeSpecs(); - const sel = selectModel("videogen", specs, { preferTier: ctx?.preferTier }); - if (sel.recommend) { + const ladder = selectModelLadder("videogen", specs, { preferTier: ctx?.preferTier }); + if (!ladder.length) { + const { reason } = selectModel("videogen", specs, { preferTier: ctx?.preferTier }); console.error( - `media-use: local video gen not enabled (${sel.reason}). Enable a fitting free on-device LTX model to use this provider.`, + `media-use: local video gen not enabled (${reason}). Enable a fitting free on-device LTX model to use this provider.`, ); return null; } - const { model } = sel; - const bin = model.invoke.trim().split(/\s+/)[0]; - try { - execFn("which", [bin], { stdio: ["ignore", "ignore", "ignore"] }); - } catch { - console.error( - `media-use: local video gen not enabled (\`${bin}\` not on PATH). Install for free on-device LTX: ${model.install}`, - ); - return null; - } + // Walk the whole ladder, best tier first. A tier that cannot run on this + // machine for a reason no spec check sees (runner off PATH, gated weights, an + // OOM) demotes to the next fitting tier instead of failing local video gen + // outright. Every demotion is reported: a silent drop to a smaller model + // leaves the caller wondering why the output looks the way it does. + for (const model of ladder) { + const bin = model.invoke.trim().split(/\s+/)[0]; + try { + execFn("which", [bin], { stdio: ["ignore", "ignore", "ignore"] }); + } catch { + console.error( + `media-use: local video gen not enabled (\`${bin}\` not on PATH). Install for free on-device LTX: ${model.install}`, + ); + continue; + } - const outPath = join(tmpdir(), `media-use-ltx-${process.pid}-${Date.now()}.mp4`); - const width = ctx?.width || 512; - const height = ctx?.height || 320; - const frames = ctx?.frames || 33; - const argv = buildArgv(model.invoke, { - prompt: intent, - w: width, - h: height, - frames, - out: outPath, - }); - argv.shift(); - - try { - execFn(bin, argv, { - encoding: "utf8", - timeout: 1_800_000, - stdio: ["ignore", "pipe", "pipe"], + const outPath = join(tmpdir(), `media-use-ltx-${process.pid}-${Date.now()}.mp4`); + const argv = buildArgv(model.invoke, { + prompt: intent, + w: ctx?.width || 512, + h: ctx?.height || 320, + frames: ctx?.frames || 33, + out: outPath, }); - } catch (err) { - console.error( - `media-use: local video gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`, - ); - return null; + argv.shift(); + + try { + execFn(bin, argv, { + encoding: "utf8", + timeout: 1_800_000, + stdio: ["ignore", "pipe", "pipe"], + }); + } catch (err) { + console.error( + `media-use: local video gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`, + ); + continue; + } + if (!pathExists(outPath)) { + console.error( + `media-use: local video gen (${model.id}) exited cleanly but wrote no output file`, + ); + continue; + } + return { + localPath: outPath, + ext: ".mp4", + source: "generated", + metadata: { + description: intent, + provider: "ltx.local", + provenance: { prompt: intent }, + }, + }; } - if (!pathExists(outPath)) return null; - return { - localPath: outPath, - ext: ".mp4", - source: "generated", - metadata: { - description: intent, - provider: "ltx.local", - provenance: { prompt: intent }, - }, - }; + return null; } diff --git a/skills/media-use/scripts/lib/ltx-video-provider.test.mjs b/skills/media-use/scripts/lib/ltx-video-provider.test.mjs index b24e35cda..bbd7a0112 100644 --- a/skills/media-use/scripts/lib/ltx-video-provider.test.mjs +++ b/skills/media-use/scripts/lib/ltx-video-provider.test.mjs @@ -124,7 +124,10 @@ test("generate failure returns null instead of throwing", async (t) => { assert.equal(calls, 2); }); -test("missing generated output returns null", async () => { +test("missing generated output returns null and says so", async (t) => { + const errors = []; + t.mock.method(console, "error", (message) => errors.push(message)); + const result = await ltxVideoGenerate( "storm clouds", { specs: fittingSpecs }, @@ -133,4 +136,88 @@ test("missing generated output returns null", async () => { ); assert.equal(result, null); + assert.equal(errors.length, 1); + assert.match(errors[0], /wrote no output file/); +}); + +// 40GB clears BOTH videogen tiers, so the ladder has two rungs. `fittingSpecs` +// above sits under the large tier's floor on purpose: every other test in this +// file exercises the medium tier alone, which is precisely why a broken large +// tier could sit in the table unnoticed. +const bothTiersSpecs = { availableRamMB: 40000, gpu: { present: true } }; + +const isGenerate = (call) => call[0] !== "which"; + +test("a top tier that cannot run demotes to the next fitting tier", async (t) => { + const errors = []; + t.mock.method(console, "error", (message) => errors.push(message)); + const calls = []; + // The runner is installed, but the large tier's weights are gated: the + // download 401s and `generate` exits non-zero. The medium tier then works. + const fakeExec = (...call) => { + calls.push(call); + if (isGenerate(call) && call[1].includes("dgrauet/ltx-2.3-mlx-q8")) { + const err = new Error("exit 1"); + err.stderr = "401 Client Error: Unauthorized for url: .../ltx-2.3-mlx-q8"; + throw err; + } + }; + + const result = await ltxVideoGenerate( + "storm clouds", + { specs: bothTiersSpecs }, + fakeExec, + () => true, + ); + + assert.ok(result, "the medium tier still produced a video"); + const generated = calls.filter(isGenerate).map((call) => call[1].join(" ")); + assert.equal(generated.length, 2, "large attempted first, then medium"); + assert.match(generated[0], /dgrauet\/ltx-2\.3-mlx-q8/); + assert.match(generated[1], /dgrauet\/ltx-2\.3-mlx-q4/); + // the demotion is reported, never silent: a smaller model changes the output + assert.equal(errors.length, 1); + assert.match(errors[0], /ltx-2\.3-mlx-q8\) failed/); + assert.match(errors[0], /401/); +}); + +test("every fitting tier failing returns null, one reason per tier", async (t) => { + const errors = []; + t.mock.method(console, "error", (message) => errors.push(message)); + const fakeExec = (...call) => { + if (isGenerate(call)) throw new Error("mlx out of memory"); + }; + + const result = await ltxVideoGenerate( + "storm clouds", + { specs: bothTiersSpecs }, + fakeExec, + () => true, + ); + + assert.equal(result, null); + assert.equal(errors.length, 2, "both tiers tried, both reported"); + assert.match(errors[0], /ltx-2\.3-mlx-q8/); + assert.match(errors[1], /ltx-2\.3-mlx-q4/); +}); + +test("preferTier pins the attempt to one tier instead of demoting", async (t) => { + t.mock.method(console, "error", () => {}); + const calls = []; + const fakeExec = (...call) => { + calls.push(call); + if (isGenerate(call)) throw new Error("boom"); + }; + + const result = await ltxVideoGenerate( + "storm clouds", + { specs: bothTiersSpecs, preferTier: "large" }, + fakeExec, + () => true, + ); + + assert.equal(result, null); + const generated = calls.filter(isGenerate).map((call) => call[1].join(" ")); + assert.equal(generated.length, 1, "pinned to large: no demotion to medium"); + assert.match(generated[0], /dgrauet\/ltx-2\.3-mlx-q8/); }); diff --git a/skills/media-use/scripts/lib/mflux-provider.mjs b/skills/media-use/scripts/lib/mflux-provider.mjs index 9de2f920b..cc7157a07 100644 --- a/skills/media-use/scripts/lib/mflux-provider.mjs +++ b/skills/media-use/scripts/lib/mflux-provider.mjs @@ -3,13 +3,14 @@ import { existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { probeSpecs } from "./specs.mjs"; -import { buildArgv, selectModel } from "./local-models.mjs"; +import { buildArgv, selectModelLadder } from "./local-models.mjs"; // Local image generation via mflux (FLUX-on-MLX), the Mac-native runner. -// Spec-gated: selectModel("imagegen", specs) returns the best FLUX-class model -// the machine's AVAILABLE RAM can actually run (medium FLUX-schnell --low-ram on -// ~24GB, up to Qwen-Image on 64GB+). When nothing local fits, this returns null -// so the registry falls through to the codex image upsell. +// Spec-gated: selectModelLadder("imagegen", specs) returns every FLUX-class +// model the machine's AVAILABLE RAM can actually run (medium FLUX-schnell +// --low-ram on ~24GB, up to Qwen-Image on 64GB+), best first. When nothing +// local fits, or no fitting tier can actually run here, this returns null so +// the registry falls through to the codex image upsell. // // The official FLUX repos are HF-gated, so the model entries point --path at // non-gated community 4-bit re-uploads; the repo is resolved to a local snapshot @@ -17,70 +18,97 @@ import { buildArgv, selectModel } from "./local-models.mjs"; // Resolve an HF repo to its local snapshot dir. `hf download` is idempotent and // prints the snapshot path as its last line. -function resolveSnapshot(repo) { - const out = execFileSync("hf", ["download", repo], { +function resolveSnapshot(repo, execFn, pathExists) { + const out = execFn("hf", ["download", repo], { encoding: "utf8", timeout: 1_800_000, stdio: ["ignore", "pipe", "pipe"], }); - const path = out.trim().split(/\r?\n/).pop()?.trim(); - return path && existsSync(path) ? path : null; + const path = out?.trim().split(/\r?\n/).pop()?.trim(); + return path && pathExists(path) ? path : null; } -export async function mfluxImageGenerate(intent, ctx) { +export async function mfluxImageGenerate( + intent, + ctx, + execFn = execFileSync, + pathExists = existsSync, +) { const specs = ctx?.specs || probeSpecs(); - const sel = selectModel("imagegen", specs, { preferTier: ctx?.preferTier }); - if (sel.recommend) return null; // no local model fits -> codex upsell/fallback + const ladder = selectModelLadder("imagegen", specs, { preferTier: ctx?.preferTier }); + if (!ladder.length) return null; // no local model fits -> codex upsell/fallback - const { model } = sel; - const bin = model.invoke.trim().split(/\s+/)[0]; - // Not installed? Surface the exact enable-command FIRST (before the model - // download) so the agent learns the free local path is available instead of - // silently taking the codex upsell. - try { - execFileSync("which", [bin], { stdio: ["ignore", "ignore", "ignore"] }); - } catch { - console.error( - `media-use: local image gen not enabled (\`${bin}\` not on PATH). Install for free on-device FLUX: ${model.install}`, - ); - return null; + // Best tier first, demoting past any tier that cannot run here (runner off + // PATH, a snapshot that won't download, an OOM) rather than failing local + // image gen outright. Every demotion is reported, so a quietly smaller model + // is never mistaken for the tier the machine nominally qualified for. + for (const model of ladder) { + const bin = model.invoke.trim().split(/\s+/)[0]; + // Not installed? Surface the exact enable-command (before the model + // download) so the agent learns the free local path is available instead of + // silently taking the codex upsell. + try { + execFn("which", [bin], { stdio: ["ignore", "ignore", "ignore"] }); + } catch { + console.error( + `media-use: local image gen not enabled (\`${bin}\` not on PATH). Install for free on-device FLUX: ${model.install}`, + ); + continue; + } + + const outPath = join(tmpdir(), `media-use-mflux-${process.pid}-${Date.now()}.png`); + const vars = { + prompt: intent, + w: ctx?.width || 512, + h: ctx?.height || 512, + seed: ctx?.seed ?? 42, + out: outPath, + }; + if (model.repo && model.invoke.includes("{model_path}")) { + let snap = null; + let why = `could not resolve a local snapshot of ${model.repo}`; + try { + snap = resolveSnapshot(model.repo, execFn, pathExists); + } catch (err) { + why = `hf download failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`; + } + if (!snap) { + console.error(`media-use: local image gen (${model.id}): ${why}`); + continue; + } + vars.model_path = snap; + } + + const argv = buildArgv(model.invoke, vars); + argv.shift(); // drop the bin (already validated) + try { + execFn(bin, argv, { + encoding: "utf8", + timeout: 1_800_000, + stdio: ["ignore", "pipe", "pipe"], + }); + } catch (err) { + console.error( + `media-use: local image gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`, + ); + continue; + } + if (!pathExists(outPath)) { + console.error( + `media-use: local image gen (${model.id}) exited cleanly but wrote no output file`, + ); + continue; + } + return { + localPath: outPath, + ext: ".png", + source: "generated", + metadata: { + description: intent, + provider: `mflux.${model.id}`, + provenance: { model: model.id, tier: model.tier, prompt: intent }, + }, + }; } - - const outPath = join(tmpdir(), `media-use-mflux-${process.pid}-${Date.now()}.png`); - const width = ctx?.width || 512; - const height = ctx?.height || 512; - const seed = ctx?.seed ?? 42; - - const vars = { prompt: intent, w: width, h: height, seed, out: outPath }; - if (model.repo && model.invoke.includes("{model_path}")) { - const snap = model.repo ? resolveSnapshot(model.repo) : null; - if (!snap) return null; - vars.model_path = snap; - } - - const argv = buildArgv(model.invoke, vars); - argv.shift(); // drop the bin (already validated) - try { - execFileSync(bin, argv, { - encoding: "utf8", - timeout: 1_800_000, - stdio: ["ignore", "pipe", "pipe"], - }); - } catch (err) { - console.error( - `media-use: local image gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`, - ); - return null; - } - if (!existsSync(outPath)) return null; - return { - localPath: outPath, - ext: ".png", - source: "generated", - metadata: { - description: intent, - provider: `mflux.${model.id}`, - provenance: { model: model.id, tier: model.tier, prompt: intent }, - }, - }; + return null; } diff --git a/skills/media-use/scripts/lib/mflux-provider.test.mjs b/skills/media-use/scripts/lib/mflux-provider.test.mjs new file mode 100644 index 000000000..9bb89c00a --- /dev/null +++ b/skills/media-use/scripts/lib/mflux-provider.test.mjs @@ -0,0 +1,115 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mfluxImageGenerate } from "./mflux-provider.mjs"; + +// 40GB clears the 32GB klein tier and the 8GB schnell tier below it; the 64GB +// qwen tier stays out of reach. Two rungs is what makes demotion observable. +const bothTiersSpecs = { availableRamMB: 40000, gpu: { present: true } }; +const SNAPSHOT = "/tmp/hf-snapshot"; + +// exec stub covering all three shells-out mflux does: the PATH probe, the +// idempotent `hf download`, and the generate itself. +function stubExec({ failGenerateFor = [], failWhichFor = [] } = {}) { + const calls = []; + const exec = (...call) => { + calls.push(call); + const [bin, argv] = call; + if (bin === "which") { + if (failWhichFor.includes(argv[0])) throw new Error("not found"); + return ""; + } + if (bin === "hf") return `Fetching 6 files...\n${SNAPSHOT}\n`; + if (failGenerateFor.some((id) => argv.join(" ").includes(id))) { + const err = new Error("exit 1"); + err.stderr = "mlx.core.metal: out of memory"; + throw err; + } + return ""; + }; + return { calls, exec }; +} + +const generateCalls = (calls) => + calls.filter(([bin]) => bin !== "which" && bin !== "hf").map(([, argv]) => argv.join(" ")); + +test("no local model fits: falls through to the upsell without shelling out", async () => { + const { calls, exec } = stubExec(); + + const result = await mfluxImageGenerate( + "a red bicycle", + { specs: { availableRamMB: 100, gpu: { present: true } } }, + exec, + () => true, + ); + + assert.equal(result, null); + assert.deepEqual(calls, []); +}); + +test("a top tier that cannot run demotes to the next fitting tier", async (t) => { + const errors = []; + t.mock.method(console, "error", (message) => errors.push(message)); + const { calls, exec } = stubExec({ failGenerateFor: ["flux2-klein-4b"] }); + + const result = await mfluxImageGenerate( + "a red bicycle", + { specs: bothTiersSpecs }, + exec, + () => true, + ); + + assert.ok(result, "the schnell tier still produced an image"); + assert.equal(result.metadata.provider, "mflux.flux-schnell-mflux-q4"); + const generated = generateCalls(calls); + assert.equal(generated.length, 2, "klein attempted first, then schnell"); + assert.match(generated[0], /flux2-klein-4b/); + assert.match(generated[1], /--model schnell/); + assert.equal(errors.length, 1, "the demotion is reported, not silent"); + assert.match(errors[0], /flux2-klein-mflux-q4\) failed/); +}); + +test("a snapshot that will not resolve demotes rather than failing outright", async (t) => { + const errors = []; + t.mock.method(console, "error", (message) => errors.push(message)); + const calls = []; + const exec = (...call) => { + calls.push(call); + const [bin, argv] = call; + if (bin === "which") return ""; + // klein's weights won't download; schnell's do + if (bin === "hf") { + if (argv[1].includes("FLUX.2-klein")) throw new Error("403 Forbidden"); + return `${SNAPSHOT}\n`; + } + return ""; + }; + + const result = await mfluxImageGenerate( + "a red bicycle", + { specs: bothTiersSpecs }, + exec, + () => true, + ); + + assert.ok(result, "demoted past the ungettable weights"); + assert.equal(result.metadata.provider, "mflux.flux-schnell-mflux-q4"); + assert.equal(errors.length, 1); + assert.match(errors[0], /hf download failed/); +}); + +test("runner missing from PATH reports the install hint per tier and returns null", async (t) => { + const errors = []; + t.mock.method(console, "error", (message) => errors.push(message)); + const { exec } = stubExec({ failWhichFor: ["mflux-generate"] }); + + const result = await mfluxImageGenerate( + "a red bicycle", + { specs: bothTiersSpecs }, + exec, + () => true, + ); + + assert.equal(result, null); + assert.equal(errors.length, 2, "both fitting tiers reported"); + assert.match(errors[0], /uv pip install mflux/); +});