mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-08-31 02:41:44 +00:00
fix(media-use): repoint the dead videogen tier, demote past unusable models
`LOCAL_MODELS.videogen`'s `large` tier named `dgrauet/ltx-2.3-mlx-bf16`, which
returns HTTP 401 and cannot be downloaded at all. It was not a dormant entry:
`rankedByPreference` sorts by descending `needs.ramMB` when no `rank` is set,
so the largest fitting tier is tried FIRST by design. Any machine clearing
32 GB *available* RAM selected the dead entry, `ltxVideoGenerate` caught the
failure and returned a bare `null`, and since `ltx.local` is last in
`["heygen.video", "ltx.local"]` and network providers are skipped under
`--local-only` (`registry.mjs:206`), local video generation failed outright
instead of falling back to the tier that works.
It survived review because the table landed with "live verification on a 24GB
M-series Mac" - and a 24 GB machine cannot select a 32 GB tier, so that entry
was unreachable on the only machine that validated it. The unit fixtures
inherit the same ceiling (`fittingSpecs` is 20000MB), so every existing test
exercised the medium tier alone.
Two changes:
1. Repoint to `dgrauet/ltx-2.3-mlx-q8` (reachable) and correct `sizeMB` from
45000 to 28800. Measured against the HF API: the q8 repo totals 87.5 GB,
and the registry's own targeted `--include` subset is 28.76 GB. That
matches the sibling q4 entry's convention (`sizeMB: 20000` vs a measured
19.48 GB subset), so 45000 was wrong under either reading. `--low-ram` is
added because the entry's own note calls it required at this tier's 32 GB
floor, and the invoke omitted it.
2. A repoint alone is one bad URL from a repeat, so add the missing recovery.
`selectModelLadder` returns every fitting model best-first;
`selectModel`'s pick is now defined as that list's head. All three sites
that previously selected exactly one model and failed terminally walk the
ladder instead, demoting past a tier that cannot run here - gated weights,
runner off PATH, an OOM at a tier that nominally fits:
- `ltx-video-provider.mjs` (videogen, the reported failure)
- `mflux-provider.mjs` (imagegen - same shape, and its 32 GB/64 GB tiers
are equally unverifiable on a 24 GB machine)
- `local-run.mjs` (tts/asr/upscale - `fish-speech` missing should still
get you Kokoro)
Every demotion is logged rather than silent, so a quietly smaller model is
never mistaken for the tier the machine nominally qualified for.
Also fixes the `install` string both videogen entries share: it ended at
`uv sync --all-extras`, which leaves the entry point in `.venv/bin`, so the
"`ltx-2-mlx` not on PATH" hint named a command that following the instruction
would not put on PATH.
The q8 tier is NOT live-verified - no 32 GB+ Apple Silicon machine was
available - and its notes say so. Shipping it unverified is safe precisely
because of change 2: a wrong tier now costs one failed attempt, not the whole
local path.
- Rames Jusso
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -50,8 +50,8 @@
|
||||
"files": 12
|
||||
},
|
||||
"media-use": {
|
||||
"hash": "1b0ce647f5c7df95",
|
||||
"files": 152
|
||||
"hash": "36eedcb77a5f40f3",
|
||||
"files": 153
|
||||
},
|
||||
"motion-graphics": {
|
||||
"hash": "853ac75cbab69036",
|
||||
|
||||
@@ -12,6 +12,10 @@
|
||||
// (quality that is NOT size, e.g. ASR), else by RAM footprint (the quality
|
||||
// proxy for generation). No fit -> recommend the CLI/cloud path.
|
||||
//
|
||||
// selectModelLadder() returns EVERY fitting model in that same order. Callers
|
||||
// that can retry walk it so ONE unusable entry (gated weights, a missing
|
||||
// binary, an OOM) demotes to the next tier instead of killing the local path.
|
||||
//
|
||||
// Picks reflect the 2026 research pass, verified live where noted.
|
||||
|
||||
export const CAPABILITIES = ["tts", "asr", "upscale", "videogen", "imagegen"];
|
||||
@@ -102,13 +106,17 @@ const MODELS = {
|
||||
},
|
||||
],
|
||||
videogen: [
|
||||
// 2026-07 X research pass + live verification on a 24GB M-series Mac.
|
||||
// 2026-07 X research pass + live verification on a 24GB M-series Mac -
|
||||
// which reaches the q4 tier only: a 24GB machine cannot select the 32GB
|
||||
// entry below it, so that tier's claims stay unverified until someone
|
||||
// runs it on a 32GB+ machine.
|
||||
// The Mac-local video story is LTX 2.3 on MLX via dgrauet/ltx-2-mlx (the
|
||||
// pipeline these weights were converted for; also powers Phosphene).
|
||||
// Wan 2.x MLX exists only as A14B conversions (too large for consumer
|
||||
// unified memory); revisit when a 5B Wan MLX conversion lands.
|
||||
// IMPORTANT: download the weights with a targeted include list first;
|
||||
// pointing tools at the repo blind snapshot-downloads all 60 GB:
|
||||
// pointing tools at the repo blind snapshot-downloads the lot (60 GB q4,
|
||||
// 88 GB q8):
|
||||
// hf download dgrauet/ltx-2.3-mlx-q4 --include \
|
||||
// transformer-distilled-1.1.safetensors connector.safetensors \
|
||||
// "vae_*.safetensors" audio_vae.safetensors vocoder.safetensors "*.json"
|
||||
@@ -119,24 +127,24 @@ const MODELS = {
|
||||
needs: { ramMB: 16384, gpu: true },
|
||||
wordTimestamps: false,
|
||||
install:
|
||||
"git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras",
|
||||
'git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras && export PATH="$PWD/.venv/bin:$PATH"',
|
||||
invoke:
|
||||
"ltx-2-mlx generate --prompt {prompt} --distilled --low-ram --model dgrauet/ltx-2.3-mlx-q4 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}",
|
||||
notes:
|
||||
"LTX 2.3 int4 on MLX. Verified on 24GB unified: 512x320 x 33 frames in ~19 min cold (incl. text-encoder download), t2v with audio. Dims must be multiples of 64. i2v, retake/extend, keyframe interpolation supported.",
|
||||
},
|
||||
{
|
||||
id: "ltx-2.3-mlx-bf16",
|
||||
id: "ltx-2.3-mlx-q8",
|
||||
tier: "large",
|
||||
sizeMB: 45000,
|
||||
sizeMB: 28800,
|
||||
needs: { ramMB: 32768, gpu: true },
|
||||
wordTimestamps: false,
|
||||
install:
|
||||
"git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras",
|
||||
'git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras && export PATH="$PWD/.venv/bin:$PATH"',
|
||||
invoke:
|
||||
"ltx-2-mlx generate --prompt {prompt} --two-stage --model dgrauet/ltx-2.3-mlx-bf16 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}",
|
||||
"ltx-2-mlx generate --prompt {prompt} --two-stage --low-ram --model dgrauet/ltx-2.3-mlx-q8 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}",
|
||||
notes:
|
||||
"Full-precision two-stage pipeline (upstream production default). 32GB with --low-ram block streaming; 64-128GB Macs for long/HD runs (the 25s multi-scene spots seen in the wild).",
|
||||
"LTX 2.3 int8 on MLX, two-stage (upstream production default; higher quality than the q4 distilled tier). Replaced dgrauet/ltx-2.3-mlx-bf16, which is gated (HTTP 401) and cannot be downloaded at all. sizeMB is the targeted include-list subset, measured 28.8GB. --low-ram matches this tier's 32GB floor (block streaming); 64-128GB Macs for long/HD runs. NOT live-verified on a 32GB+ machine - the q4 tier below is the verified one.",
|
||||
},
|
||||
],
|
||||
imagegen: [
|
||||
@@ -247,6 +255,23 @@ function rankedByPreference(table) {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Every local model for a capability this machine can actually run, best-first
|
||||
* (same ordering as selectModel, whose pick is this list's head).
|
||||
*
|
||||
* Callers that can retry should walk the whole list: a table entry can be
|
||||
* unusable for reasons no spec check can see - weights pulled or gated behind a
|
||||
* login, the runner missing from PATH, an OOM at a tier that nominally fits. On
|
||||
* a single-select call any one of those fails the entire local path, because the
|
||||
* cascade cannot tell "this model is broken" from "nothing here fits you".
|
||||
* Demoting to the next fitting tier is almost always what the user wanted.
|
||||
*/
|
||||
export function selectModelLadder(capability, specs, { preferTier } = {}) {
|
||||
const table = tableFor(capability);
|
||||
const pool = preferTier ? table.filter((m) => m.tier === preferTier) : table;
|
||||
return rankedByPreference(pool).filter((model) => meetsSpecs(model, specs));
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the best local model the machine can run for a capability: the
|
||||
* highest-footprint model that fits the available-RAM budget (and GPU/VRAM).
|
||||
@@ -255,10 +280,8 @@ function rankedByPreference(table) {
|
||||
*/
|
||||
export function selectModel(capability, specs, { preferTier } = {}) {
|
||||
const table = tableFor(capability);
|
||||
const pool = preferTier ? table.filter((m) => m.tier === preferTier) : table;
|
||||
for (const model of rankedByPreference(pool)) {
|
||||
if (meetsSpecs(model, specs)) return { model, tier: model.tier };
|
||||
}
|
||||
const [model] = selectModelLadder(capability, specs, { preferTier });
|
||||
if (model) return { model, tier: model.tier };
|
||||
const smallest = table.reduce((a, b) => (a.sizeMB <= b.sizeMB ? a : b));
|
||||
return {
|
||||
recommend: "cli",
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
listModels,
|
||||
meetsSpecs,
|
||||
selectModel,
|
||||
selectModelLadder,
|
||||
describeModelLadder,
|
||||
CAPABILITIES,
|
||||
} from "./local-models.mjs";
|
||||
@@ -152,3 +153,68 @@ test("ASR offers word-timestamp-capable models (better than plain whisper)", ()
|
||||
"every ASR model must support word timestamps",
|
||||
);
|
||||
});
|
||||
|
||||
// A machine that clears BOTH videogen tiers (the 32GB entry and the 16GB one).
|
||||
// The existing fixtures deliberately sit under the large tier's floor, which is
|
||||
// exactly how a dead 32GB entry stayed invisible: nothing could select it.
|
||||
const bothVideogenTiers = { availableRamMB: 40000, gpu: { present: true } };
|
||||
|
||||
test("selectModelLadder returns every fitting model, best-first", () => {
|
||||
const ladder = selectModelLadder("videogen", bothVideogenTiers);
|
||||
assert.deepEqual(
|
||||
ladder.map((m) => m.tier),
|
||||
["large", "medium"],
|
||||
"both tiers fit 40GB, biggest first",
|
||||
);
|
||||
assert.equal(
|
||||
selectModel("videogen", bothVideogenTiers).model.id,
|
||||
ladder[0].id,
|
||||
"selectModel's pick is the ladder's head",
|
||||
);
|
||||
});
|
||||
|
||||
test("selectModelLadder drops what the machine cannot run", () => {
|
||||
const oneTier = selectModelLadder("videogen", { availableRamMB: 20000, gpu: { present: true } });
|
||||
assert.deepEqual(
|
||||
oneTier.map((m) => m.tier),
|
||||
["medium"],
|
||||
"20GB cannot reach the 32GB tier",
|
||||
);
|
||||
assert.deepEqual(
|
||||
selectModelLadder("videogen", { availableRamMB: 100, gpu: { present: true } }),
|
||||
[],
|
||||
"nothing fits -> empty ladder, and selectModel recommends the CLI",
|
||||
);
|
||||
assert.equal(
|
||||
selectModel("videogen", { availableRamMB: 100, gpu: { present: true } }).recommend,
|
||||
"cli",
|
||||
);
|
||||
});
|
||||
|
||||
test("selectModelLadder honours preferTier", () => {
|
||||
const pinned = selectModelLadder("videogen", bothVideogenTiers, { preferTier: "medium" });
|
||||
assert.deepEqual(
|
||||
pinned.map((m) => m.tier),
|
||||
["medium"],
|
||||
"preferTier pins the ladder to one tier",
|
||||
);
|
||||
});
|
||||
|
||||
test("an invoke that names an owner/repo model agrees with the entry id", () => {
|
||||
// Guards a half-done repoint: moving an entry to different weights means
|
||||
// changing BOTH the id and the --model argument. Change one and the table
|
||||
// selects one model while the runner downloads another.
|
||||
let checked = 0;
|
||||
for (const cap of CAPABILITIES) {
|
||||
for (const m of listModels(cap)) {
|
||||
if (m.repo) continue; // entries with an explicit repo resolve through it
|
||||
const named = /--model\s+(\S+)/.exec(m.invoke);
|
||||
if (!named) continue;
|
||||
const [, name] = named[1].split("/");
|
||||
if (!name) continue; // a bare model name, not an owner/repo id
|
||||
assert.equal(name, m.id, `${cap}/${m.id}: invoke runs ${named[1]}`);
|
||||
checked += 1;
|
||||
}
|
||||
}
|
||||
assert.ok(checked > 0, "no entry pins an owner/repo model - guard would be vacuous");
|
||||
});
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { selectModel } from "./local-models.mjs";
|
||||
import { selectModel, selectModelLadder } from "./local-models.mjs";
|
||||
import { probeSpecs } from "./specs.mjs";
|
||||
|
||||
// Run a USER-INSTALLED local model for a capability (tts/asr/upscale).
|
||||
// Picks the best tier the machine supports (selectModel), checks the tool is on
|
||||
// PATH, fills the model's invoke template, and runs it. Returns:
|
||||
// Walks the tiers the machine supports best-first (selectModelLadder), checking
|
||||
// the tool is on PATH, filling the model's invoke template, and running it. A
|
||||
// tier whose tool is missing or whose run fails demotes to the next fitting
|
||||
// tier, so one unusable entry does not fail the capability. Returns:
|
||||
// { model, tier, out } on success
|
||||
// { recommend:"install", model, command, reason } when the tool isn't installed
|
||||
// { recommend:"cli", reason } when no tier fits the machine
|
||||
@@ -35,30 +37,40 @@ export function runLocalModel(capability, opts = {}) {
|
||||
vars = {},
|
||||
preferTier,
|
||||
} = opts;
|
||||
const sel = selectModel(capability, specs, { preferTier });
|
||||
if (sel.recommend) return sel; // no tier fits -> recommend the CLI path
|
||||
const ladder = selectModelLadder(capability, specs, { preferTier });
|
||||
// no tier fits at all -> recommend the CLI path (selectModel words the reason)
|
||||
if (!ladder.length) return selectModel(capability, specs, { preferTier });
|
||||
|
||||
const { model } = sel;
|
||||
const bin = model.invoke.split(/\s+/)[0];
|
||||
try {
|
||||
which(bin);
|
||||
} catch {
|
||||
return {
|
||||
recommend: "install",
|
||||
model: model.id,
|
||||
command: model.install,
|
||||
reason: `${model.id} not installed`,
|
||||
};
|
||||
// Best tier first, demoting past any tier that cannot run here: a missing
|
||||
// tool or a failed run at the top tier must not hide a lower tier that works
|
||||
// (fish-speech absent should still get you Kokoro). The last tier's failure is
|
||||
// what gets reported, since by then nothing local ran.
|
||||
let lastFailure = null;
|
||||
for (const model of ladder) {
|
||||
const bin = model.invoke.split(/\s+/)[0];
|
||||
try {
|
||||
which(bin);
|
||||
} catch {
|
||||
lastFailure = {
|
||||
recommend: "install",
|
||||
model: model.id,
|
||||
command: model.install,
|
||||
reason: `${model.id} not installed`,
|
||||
};
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
exec(fill(model.invoke, vars));
|
||||
} catch (e) {
|
||||
lastFailure = {
|
||||
recommend: "install",
|
||||
model: model.id,
|
||||
command: model.install,
|
||||
reason: e.message || String(e),
|
||||
};
|
||||
continue;
|
||||
}
|
||||
return { model: model.id, tier: model.tier, out: vars.out };
|
||||
}
|
||||
try {
|
||||
exec(fill(model.invoke, vars));
|
||||
} catch (e) {
|
||||
return {
|
||||
recommend: "install",
|
||||
model: model.id,
|
||||
command: model.install,
|
||||
reason: e.message || String(e),
|
||||
};
|
||||
}
|
||||
return { model: model.id, tier: sel.tier, out: vars.out };
|
||||
return lastFailure;
|
||||
}
|
||||
|
||||
@@ -52,3 +52,40 @@ test("a failing run degrades to an install recommendation, never throws", () =>
|
||||
});
|
||||
assert.equal(r.recommend, "install");
|
||||
});
|
||||
|
||||
test("a tier whose tool is missing demotes to the next tier that fits", () => {
|
||||
// 64GB + GPU fits BOTH tts tiers, so the ladder has two rungs: fish-speech
|
||||
// (its own binary) above Kokoro (`python -m kokoro`). fish-speech absent must
|
||||
// not cost the user Kokoro.
|
||||
const strongGpu = { ramMB: 64000, gpu: { present: true, vramMB: 24000 } };
|
||||
let ran = "";
|
||||
const r = runLocalModel("tts", {
|
||||
specs: strongGpu,
|
||||
which: (bin) => {
|
||||
if (bin === "fish-speech") throw new Error("not found");
|
||||
},
|
||||
exec: (cmd) => {
|
||||
ran = cmd;
|
||||
},
|
||||
vars: { text: "hello", voice: "af_heart", out: "/tmp/v.wav" },
|
||||
});
|
||||
|
||||
assert.equal(r.model, "kokoro", "demoted past the missing fish-speech binary");
|
||||
assert.equal(r.tier, "medium");
|
||||
assert.match(ran, /kokoro/);
|
||||
});
|
||||
|
||||
test("every fitting tier failing reports the last tier's install command", () => {
|
||||
const strongGpu = { ramMB: 64000, gpu: { present: true, vramMB: 24000 } };
|
||||
const r = runLocalModel("tts", {
|
||||
specs: strongGpu,
|
||||
which: ok,
|
||||
exec: () => {
|
||||
throw new Error("boom");
|
||||
},
|
||||
vars: { text: "hi", out: "/tmp/v.wav" },
|
||||
});
|
||||
|
||||
assert.equal(r.recommend, "install");
|
||||
assert.equal(r.model, "kokoro", "the smallest fitting tier is the actionable one");
|
||||
});
|
||||
|
||||
@@ -3,7 +3,7 @@ import { existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { probeSpecs } from "./specs.mjs";
|
||||
import { buildArgv, selectModel } from "./local-models.mjs";
|
||||
import { buildArgv, selectModel, selectModelLadder } from "./local-models.mjs";
|
||||
|
||||
export async function ltxVideoGenerate(
|
||||
intent,
|
||||
@@ -12,59 +12,69 @@ export async function ltxVideoGenerate(
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const specs = ctx?.specs || probeSpecs();
|
||||
const sel = selectModel("videogen", specs, { preferTier: ctx?.preferTier });
|
||||
if (sel.recommend) {
|
||||
const ladder = selectModelLadder("videogen", specs, { preferTier: ctx?.preferTier });
|
||||
if (!ladder.length) {
|
||||
const { reason } = selectModel("videogen", specs, { preferTier: ctx?.preferTier });
|
||||
console.error(
|
||||
`media-use: local video gen not enabled (${sel.reason}). Enable a fitting free on-device LTX model to use this provider.`,
|
||||
`media-use: local video gen not enabled (${reason}). Enable a fitting free on-device LTX model to use this provider.`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const { model } = sel;
|
||||
const bin = model.invoke.trim().split(/\s+/)[0];
|
||||
try {
|
||||
execFn("which", [bin], { stdio: ["ignore", "ignore", "ignore"] });
|
||||
} catch {
|
||||
console.error(
|
||||
`media-use: local video gen not enabled (\`${bin}\` not on PATH). Install for free on-device LTX: ${model.install}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
// Walk the whole ladder, best tier first. A tier that cannot run on this
|
||||
// machine for a reason no spec check sees (runner off PATH, gated weights, an
|
||||
// OOM) demotes to the next fitting tier instead of failing local video gen
|
||||
// outright. Every demotion is reported: a silent drop to a smaller model
|
||||
// leaves the caller wondering why the output looks the way it does.
|
||||
for (const model of ladder) {
|
||||
const bin = model.invoke.trim().split(/\s+/)[0];
|
||||
try {
|
||||
execFn("which", [bin], { stdio: ["ignore", "ignore", "ignore"] });
|
||||
} catch {
|
||||
console.error(
|
||||
`media-use: local video gen not enabled (\`${bin}\` not on PATH). Install for free on-device LTX: ${model.install}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const outPath = join(tmpdir(), `media-use-ltx-${process.pid}-${Date.now()}.mp4`);
|
||||
const width = ctx?.width || 512;
|
||||
const height = ctx?.height || 320;
|
||||
const frames = ctx?.frames || 33;
|
||||
const argv = buildArgv(model.invoke, {
|
||||
prompt: intent,
|
||||
w: width,
|
||||
h: height,
|
||||
frames,
|
||||
out: outPath,
|
||||
});
|
||||
argv.shift();
|
||||
|
||||
try {
|
||||
execFn(bin, argv, {
|
||||
encoding: "utf8",
|
||||
timeout: 1_800_000,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
const outPath = join(tmpdir(), `media-use-ltx-${process.pid}-${Date.now()}.mp4`);
|
||||
const argv = buildArgv(model.invoke, {
|
||||
prompt: intent,
|
||||
w: ctx?.width || 512,
|
||||
h: ctx?.height || 320,
|
||||
frames: ctx?.frames || 33,
|
||||
out: outPath,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`media-use: local video gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`,
|
||||
);
|
||||
return null;
|
||||
argv.shift();
|
||||
|
||||
try {
|
||||
execFn(bin, argv, {
|
||||
encoding: "utf8",
|
||||
timeout: 1_800_000,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`media-use: local video gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!pathExists(outPath)) {
|
||||
console.error(
|
||||
`media-use: local video gen (${model.id}) exited cleanly but wrote no output file`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
return {
|
||||
localPath: outPath,
|
||||
ext: ".mp4",
|
||||
source: "generated",
|
||||
metadata: {
|
||||
description: intent,
|
||||
provider: "ltx.local",
|
||||
provenance: { prompt: intent },
|
||||
},
|
||||
};
|
||||
}
|
||||
if (!pathExists(outPath)) return null;
|
||||
return {
|
||||
localPath: outPath,
|
||||
ext: ".mp4",
|
||||
source: "generated",
|
||||
metadata: {
|
||||
description: intent,
|
||||
provider: "ltx.local",
|
||||
provenance: { prompt: intent },
|
||||
},
|
||||
};
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -124,7 +124,10 @@ test("generate failure returns null instead of throwing", async (t) => {
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
|
||||
test("missing generated output returns null", async () => {
|
||||
test("missing generated output returns null and says so", async (t) => {
|
||||
const errors = [];
|
||||
t.mock.method(console, "error", (message) => errors.push(message));
|
||||
|
||||
const result = await ltxVideoGenerate(
|
||||
"storm clouds",
|
||||
{ specs: fittingSpecs },
|
||||
@@ -133,4 +136,88 @@ test("missing generated output returns null", async () => {
|
||||
);
|
||||
|
||||
assert.equal(result, null);
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(errors[0], /wrote no output file/);
|
||||
});
|
||||
|
||||
// 40GB clears BOTH videogen tiers, so the ladder has two rungs. `fittingSpecs`
|
||||
// above sits under the large tier's floor on purpose: every other test in this
|
||||
// file exercises the medium tier alone, which is precisely why a broken large
|
||||
// tier could sit in the table unnoticed.
|
||||
const bothTiersSpecs = { availableRamMB: 40000, gpu: { present: true } };
|
||||
|
||||
const isGenerate = (call) => call[0] !== "which";
|
||||
|
||||
test("a top tier that cannot run demotes to the next fitting tier", async (t) => {
|
||||
const errors = [];
|
||||
t.mock.method(console, "error", (message) => errors.push(message));
|
||||
const calls = [];
|
||||
// The runner is installed, but the large tier's weights are gated: the
|
||||
// download 401s and `generate` exits non-zero. The medium tier then works.
|
||||
const fakeExec = (...call) => {
|
||||
calls.push(call);
|
||||
if (isGenerate(call) && call[1].includes("dgrauet/ltx-2.3-mlx-q8")) {
|
||||
const err = new Error("exit 1");
|
||||
err.stderr = "401 Client Error: Unauthorized for url: .../ltx-2.3-mlx-q8";
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
const result = await ltxVideoGenerate(
|
||||
"storm clouds",
|
||||
{ specs: bothTiersSpecs },
|
||||
fakeExec,
|
||||
() => true,
|
||||
);
|
||||
|
||||
assert.ok(result, "the medium tier still produced a video");
|
||||
const generated = calls.filter(isGenerate).map((call) => call[1].join(" "));
|
||||
assert.equal(generated.length, 2, "large attempted first, then medium");
|
||||
assert.match(generated[0], /dgrauet\/ltx-2\.3-mlx-q8/);
|
||||
assert.match(generated[1], /dgrauet\/ltx-2\.3-mlx-q4/);
|
||||
// the demotion is reported, never silent: a smaller model changes the output
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(errors[0], /ltx-2\.3-mlx-q8\) failed/);
|
||||
assert.match(errors[0], /401/);
|
||||
});
|
||||
|
||||
test("every fitting tier failing returns null, one reason per tier", async (t) => {
|
||||
const errors = [];
|
||||
t.mock.method(console, "error", (message) => errors.push(message));
|
||||
const fakeExec = (...call) => {
|
||||
if (isGenerate(call)) throw new Error("mlx out of memory");
|
||||
};
|
||||
|
||||
const result = await ltxVideoGenerate(
|
||||
"storm clouds",
|
||||
{ specs: bothTiersSpecs },
|
||||
fakeExec,
|
||||
() => true,
|
||||
);
|
||||
|
||||
assert.equal(result, null);
|
||||
assert.equal(errors.length, 2, "both tiers tried, both reported");
|
||||
assert.match(errors[0], /ltx-2\.3-mlx-q8/);
|
||||
assert.match(errors[1], /ltx-2\.3-mlx-q4/);
|
||||
});
|
||||
|
||||
test("preferTier pins the attempt to one tier instead of demoting", async (t) => {
|
||||
t.mock.method(console, "error", () => {});
|
||||
const calls = [];
|
||||
const fakeExec = (...call) => {
|
||||
calls.push(call);
|
||||
if (isGenerate(call)) throw new Error("boom");
|
||||
};
|
||||
|
||||
const result = await ltxVideoGenerate(
|
||||
"storm clouds",
|
||||
{ specs: bothTiersSpecs, preferTier: "large" },
|
||||
fakeExec,
|
||||
() => true,
|
||||
);
|
||||
|
||||
assert.equal(result, null);
|
||||
const generated = calls.filter(isGenerate).map((call) => call[1].join(" "));
|
||||
assert.equal(generated.length, 1, "pinned to large: no demotion to medium");
|
||||
assert.match(generated[0], /dgrauet\/ltx-2\.3-mlx-q8/);
|
||||
});
|
||||
|
||||
@@ -3,13 +3,14 @@ import { existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { probeSpecs } from "./specs.mjs";
|
||||
import { buildArgv, selectModel } from "./local-models.mjs";
|
||||
import { buildArgv, selectModelLadder } from "./local-models.mjs";
|
||||
|
||||
// Local image generation via mflux (FLUX-on-MLX), the Mac-native runner.
|
||||
// Spec-gated: selectModel("imagegen", specs) returns the best FLUX-class model
|
||||
// the machine's AVAILABLE RAM can actually run (medium FLUX-schnell --low-ram on
|
||||
// ~24GB, up to Qwen-Image on 64GB+). When nothing local fits, this returns null
|
||||
// so the registry falls through to the codex image upsell.
|
||||
// Spec-gated: selectModelLadder("imagegen", specs) returns every FLUX-class
|
||||
// model the machine's AVAILABLE RAM can actually run (medium FLUX-schnell
|
||||
// --low-ram on ~24GB, up to Qwen-Image on 64GB+), best first. When nothing
|
||||
// local fits, or no fitting tier can actually run here, this returns null so
|
||||
// the registry falls through to the codex image upsell.
|
||||
//
|
||||
// The official FLUX repos are HF-gated, so the model entries point --path at
|
||||
// non-gated community 4-bit re-uploads; the repo is resolved to a local snapshot
|
||||
@@ -17,70 +18,97 @@ import { buildArgv, selectModel } from "./local-models.mjs";
|
||||
|
||||
// Resolve an HF repo to its local snapshot dir. `hf download` is idempotent and
|
||||
// prints the snapshot path as its last line.
|
||||
function resolveSnapshot(repo) {
|
||||
const out = execFileSync("hf", ["download", repo], {
|
||||
function resolveSnapshot(repo, execFn, pathExists) {
|
||||
const out = execFn("hf", ["download", repo], {
|
||||
encoding: "utf8",
|
||||
timeout: 1_800_000,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const path = out.trim().split(/\r?\n/).pop()?.trim();
|
||||
return path && existsSync(path) ? path : null;
|
||||
const path = out?.trim().split(/\r?\n/).pop()?.trim();
|
||||
return path && pathExists(path) ? path : null;
|
||||
}
|
||||
|
||||
export async function mfluxImageGenerate(intent, ctx) {
|
||||
export async function mfluxImageGenerate(
|
||||
intent,
|
||||
ctx,
|
||||
execFn = execFileSync,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const specs = ctx?.specs || probeSpecs();
|
||||
const sel = selectModel("imagegen", specs, { preferTier: ctx?.preferTier });
|
||||
if (sel.recommend) return null; // no local model fits -> codex upsell/fallback
|
||||
const ladder = selectModelLadder("imagegen", specs, { preferTier: ctx?.preferTier });
|
||||
if (!ladder.length) return null; // no local model fits -> codex upsell/fallback
|
||||
|
||||
const { model } = sel;
|
||||
const bin = model.invoke.trim().split(/\s+/)[0];
|
||||
// Not installed? Surface the exact enable-command FIRST (before the model
|
||||
// download) so the agent learns the free local path is available instead of
|
||||
// silently taking the codex upsell.
|
||||
try {
|
||||
execFileSync("which", [bin], { stdio: ["ignore", "ignore", "ignore"] });
|
||||
} catch {
|
||||
console.error(
|
||||
`media-use: local image gen not enabled (\`${bin}\` not on PATH). Install for free on-device FLUX: ${model.install}`,
|
||||
);
|
||||
return null;
|
||||
// Best tier first, demoting past any tier that cannot run here (runner off
|
||||
// PATH, a snapshot that won't download, an OOM) rather than failing local
|
||||
// image gen outright. Every demotion is reported, so a quietly smaller model
|
||||
// is never mistaken for the tier the machine nominally qualified for.
|
||||
for (const model of ladder) {
|
||||
const bin = model.invoke.trim().split(/\s+/)[0];
|
||||
// Not installed? Surface the exact enable-command (before the model
|
||||
// download) so the agent learns the free local path is available instead of
|
||||
// silently taking the codex upsell.
|
||||
try {
|
||||
execFn("which", [bin], { stdio: ["ignore", "ignore", "ignore"] });
|
||||
} catch {
|
||||
console.error(
|
||||
`media-use: local image gen not enabled (\`${bin}\` not on PATH). Install for free on-device FLUX: ${model.install}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const outPath = join(tmpdir(), `media-use-mflux-${process.pid}-${Date.now()}.png`);
|
||||
const vars = {
|
||||
prompt: intent,
|
||||
w: ctx?.width || 512,
|
||||
h: ctx?.height || 512,
|
||||
seed: ctx?.seed ?? 42,
|
||||
out: outPath,
|
||||
};
|
||||
if (model.repo && model.invoke.includes("{model_path}")) {
|
||||
let snap = null;
|
||||
let why = `could not resolve a local snapshot of ${model.repo}`;
|
||||
try {
|
||||
snap = resolveSnapshot(model.repo, execFn, pathExists);
|
||||
} catch (err) {
|
||||
why = `hf download failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`;
|
||||
}
|
||||
if (!snap) {
|
||||
console.error(`media-use: local image gen (${model.id}): ${why}`);
|
||||
continue;
|
||||
}
|
||||
vars.model_path = snap;
|
||||
}
|
||||
|
||||
const argv = buildArgv(model.invoke, vars);
|
||||
argv.shift(); // drop the bin (already validated)
|
||||
try {
|
||||
execFn(bin, argv, {
|
||||
encoding: "utf8",
|
||||
timeout: 1_800_000,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`media-use: local image gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!pathExists(outPath)) {
|
||||
console.error(
|
||||
`media-use: local image gen (${model.id}) exited cleanly but wrote no output file`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
return {
|
||||
localPath: outPath,
|
||||
ext: ".png",
|
||||
source: "generated",
|
||||
metadata: {
|
||||
description: intent,
|
||||
provider: `mflux.${model.id}`,
|
||||
provenance: { model: model.id, tier: model.tier, prompt: intent },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const outPath = join(tmpdir(), `media-use-mflux-${process.pid}-${Date.now()}.png`);
|
||||
const width = ctx?.width || 512;
|
||||
const height = ctx?.height || 512;
|
||||
const seed = ctx?.seed ?? 42;
|
||||
|
||||
const vars = { prompt: intent, w: width, h: height, seed, out: outPath };
|
||||
if (model.repo && model.invoke.includes("{model_path}")) {
|
||||
const snap = model.repo ? resolveSnapshot(model.repo) : null;
|
||||
if (!snap) return null;
|
||||
vars.model_path = snap;
|
||||
}
|
||||
|
||||
const argv = buildArgv(model.invoke, vars);
|
||||
argv.shift(); // drop the bin (already validated)
|
||||
try {
|
||||
execFileSync(bin, argv, {
|
||||
encoding: "utf8",
|
||||
timeout: 1_800_000,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`media-use: local image gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
if (!existsSync(outPath)) return null;
|
||||
return {
|
||||
localPath: outPath,
|
||||
ext: ".png",
|
||||
source: "generated",
|
||||
metadata: {
|
||||
description: intent,
|
||||
provider: `mflux.${model.id}`,
|
||||
provenance: { model: model.id, tier: model.tier, prompt: intent },
|
||||
},
|
||||
};
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mfluxImageGenerate } from "./mflux-provider.mjs";
|
||||
|
||||
// 40GB clears the 32GB klein tier and the 8GB schnell tier below it; the 64GB
|
||||
// qwen tier stays out of reach. Two rungs is what makes demotion observable.
|
||||
const bothTiersSpecs = { availableRamMB: 40000, gpu: { present: true } };
|
||||
const SNAPSHOT = "/tmp/hf-snapshot";
|
||||
|
||||
// exec stub covering all three shells-out mflux does: the PATH probe, the
|
||||
// idempotent `hf download`, and the generate itself.
|
||||
function stubExec({ failGenerateFor = [], failWhichFor = [] } = {}) {
|
||||
const calls = [];
|
||||
const exec = (...call) => {
|
||||
calls.push(call);
|
||||
const [bin, argv] = call;
|
||||
if (bin === "which") {
|
||||
if (failWhichFor.includes(argv[0])) throw new Error("not found");
|
||||
return "";
|
||||
}
|
||||
if (bin === "hf") return `Fetching 6 files...\n${SNAPSHOT}\n`;
|
||||
if (failGenerateFor.some((id) => argv.join(" ").includes(id))) {
|
||||
const err = new Error("exit 1");
|
||||
err.stderr = "mlx.core.metal: out of memory";
|
||||
throw err;
|
||||
}
|
||||
return "";
|
||||
};
|
||||
return { calls, exec };
|
||||
}
|
||||
|
||||
const generateCalls = (calls) =>
|
||||
calls.filter(([bin]) => bin !== "which" && bin !== "hf").map(([, argv]) => argv.join(" "));
|
||||
|
||||
test("no local model fits: falls through to the upsell without shelling out", async () => {
|
||||
const { calls, exec } = stubExec();
|
||||
|
||||
const result = await mfluxImageGenerate(
|
||||
"a red bicycle",
|
||||
{ specs: { availableRamMB: 100, gpu: { present: true } } },
|
||||
exec,
|
||||
() => true,
|
||||
);
|
||||
|
||||
assert.equal(result, null);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test("a top tier that cannot run demotes to the next fitting tier", async (t) => {
|
||||
const errors = [];
|
||||
t.mock.method(console, "error", (message) => errors.push(message));
|
||||
const { calls, exec } = stubExec({ failGenerateFor: ["flux2-klein-4b"] });
|
||||
|
||||
const result = await mfluxImageGenerate(
|
||||
"a red bicycle",
|
||||
{ specs: bothTiersSpecs },
|
||||
exec,
|
||||
() => true,
|
||||
);
|
||||
|
||||
assert.ok(result, "the schnell tier still produced an image");
|
||||
assert.equal(result.metadata.provider, "mflux.flux-schnell-mflux-q4");
|
||||
const generated = generateCalls(calls);
|
||||
assert.equal(generated.length, 2, "klein attempted first, then schnell");
|
||||
assert.match(generated[0], /flux2-klein-4b/);
|
||||
assert.match(generated[1], /--model schnell/);
|
||||
assert.equal(errors.length, 1, "the demotion is reported, not silent");
|
||||
assert.match(errors[0], /flux2-klein-mflux-q4\) failed/);
|
||||
});
|
||||
|
||||
test("a snapshot that will not resolve demotes rather than failing outright", async (t) => {
|
||||
const errors = [];
|
||||
t.mock.method(console, "error", (message) => errors.push(message));
|
||||
const calls = [];
|
||||
const exec = (...call) => {
|
||||
calls.push(call);
|
||||
const [bin, argv] = call;
|
||||
if (bin === "which") return "";
|
||||
// klein's weights won't download; schnell's do
|
||||
if (bin === "hf") {
|
||||
if (argv[1].includes("FLUX.2-klein")) throw new Error("403 Forbidden");
|
||||
return `${SNAPSHOT}\n`;
|
||||
}
|
||||
return "";
|
||||
};
|
||||
|
||||
const result = await mfluxImageGenerate(
|
||||
"a red bicycle",
|
||||
{ specs: bothTiersSpecs },
|
||||
exec,
|
||||
() => true,
|
||||
);
|
||||
|
||||
assert.ok(result, "demoted past the ungettable weights");
|
||||
assert.equal(result.metadata.provider, "mflux.flux-schnell-mflux-q4");
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(errors[0], /hf download failed/);
|
||||
});
|
||||
|
||||
test("runner missing from PATH reports the install hint per tier and returns null", async (t) => {
|
||||
const errors = [];
|
||||
t.mock.method(console, "error", (message) => errors.push(message));
|
||||
const { exec } = stubExec({ failWhichFor: ["mflux-generate"] });
|
||||
|
||||
const result = await mfluxImageGenerate(
|
||||
"a red bicycle",
|
||||
{ specs: bothTiersSpecs },
|
||||
exec,
|
||||
() => true,
|
||||
);
|
||||
|
||||
assert.equal(result, null);
|
||||
assert.equal(errors.length, 2, "both fitting tiers reported");
|
||||
assert.match(errors[0], /uv pip install mflux/);
|
||||
});
|
||||
Reference in New Issue
Block a user