mirror of
https://github.com/andrewyng/openworker.git
synced 2026-10-30 10:47:19 +00:00
Merge pull request #792 from andrewyng/feat/llamacpp-guided-setup
Guided llama.cpp setup on the Mac (UX-057)
This commit is contained in:
Rohit Prasad
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"version": 1,
|
||||
"about": "Models the llama.cpp setup suggests (UX-057). Read by coworker/providers/llamacpp_setup.py. Each names a Hugging Face repo and the GGUF files OpenWorker downloads into llama.cpp's own cache; `draft` is a second file for speculative decoding (`spec_type`). `size` is the bytes of all files, for the fit advice; a model too large for the machine is not offered. `ollama` is the same model's name in Ollama, for the download command shown next to it. `license` and `license_url` are shown before Download, read from the model card on Hugging Face (`license_source` says where; checked 2026-10-26). Edit in git; give a source.",
|
||||
"models": [
|
||||
{
|
||||
"id": "nemotron-3.5-lightning",
|
||||
"name": "Nemotron 3.5 Lightning",
|
||||
"vendor": "NVIDIA",
|
||||
"quant": "Q8 + MTP",
|
||||
"repo": "ggml-org/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF",
|
||||
"file": "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf",
|
||||
"draft": "mtp-NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf",
|
||||
"spec_type": "draft-mtp",
|
||||
"size": 35761208832,
|
||||
"context_max": 1048576,
|
||||
"features": ["thinking", "tools"],
|
||||
"license": "OpenMDW-1.1",
|
||||
"license_url": "https://openmdw.ai/license/1-1/",
|
||||
"license_source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 (model card: license: other, license_name: openmdw-1.1; the GGUF repo only says other)",
|
||||
"source": "https://huggingface.co/ggml-org/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF"
|
||||
},
|
||||
{
|
||||
"id": "qwen3-coder-30b",
|
||||
"name": "Qwen3 Coder 30B",
|
||||
"vendor": "Alibaba",
|
||||
"quant": "Q4",
|
||||
"repo": "unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF",
|
||||
"file": "Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf",
|
||||
"size": 18556689568,
|
||||
"context_max": 262144,
|
||||
"features": ["tools"],
|
||||
"license": "Apache-2.0",
|
||||
"license_url": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/main/LICENSE",
|
||||
"license_source": "https://huggingface.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF (model card: license: apache-2.0, license_link as above)",
|
||||
"ollama": "qwen3-coder:30b",
|
||||
"source": "https://huggingface.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF"
|
||||
},
|
||||
{
|
||||
"id": "gpt-oss-20b",
|
||||
"name": "gpt-oss 20B",
|
||||
"vendor": "OpenAI",
|
||||
"quant": "MXFP4",
|
||||
"repo": "ggml-org/gpt-oss-20b-GGUF",
|
||||
"file": "gpt-oss-20b-MXFP4.gguf",
|
||||
"size": 12109566624,
|
||||
"context_max": 131072,
|
||||
"features": ["reasoning", "tools"],
|
||||
"license": "Apache-2.0",
|
||||
"license_url": "https://huggingface.co/openai/gpt-oss-20b/blob/main/LICENSE",
|
||||
"license_source": "https://huggingface.co/ggml-org/gpt-oss-20b-GGUF (model card: license: apache-2.0)",
|
||||
"ollama": "gpt-oss:20b",
|
||||
"source": "https://huggingface.co/ggml-org/gpt-oss-20b-GGUF"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,911 @@
|
||||
"""llama.cpp on this Mac, set up and run by OpenWorker (UX-057).
|
||||
|
||||
The llama.cpp page offers three steps: get llama.cpp (Homebrew), pick a model sized to
|
||||
this machine, and start it. After that OpenWorker runs `llama-server` itself, one model at
|
||||
a time, and points the `llamacpp` provider at it. "Connect to it instead" (a server the
|
||||
user runs) stays the plain provider form; nothing here touches it unless the user starts
|
||||
a model.
|
||||
|
||||
Where models go: llama.cpp's own cache, the Hugging Face layout `llama-server -hf` uses
|
||||
(`$LLAMA_CACHE`, else the Hugging Face hub cache, `~/.cache/huggingface/hub`). OpenWorker
|
||||
downloads the files itself, into that layout, because `-hf` cannot fetch a model's second
|
||||
file (the MTP draft) and gives no progress; a later `llama-server -hf` finds them there.
|
||||
Models Ollama already downloaded are GGUF files too, and are loaded in place.
|
||||
|
||||
The desktop app ends its server with SIGKILL, so no shutdown code runs on quit. Two guards
|
||||
keep a model from holding memory after OpenWorker is gone: a small watcher process kills
|
||||
llama-server when this server exits, and the pid file lets the next start stop one left
|
||||
over.
|
||||
|
||||
The GUI polls `state()` while something runs, like the sandbox setup (setup_job.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from . import local_machine
|
||||
from .recommended import recommendation_for
|
||||
|
||||
_CATALOG = Path(__file__).with_name("llamacpp_models.json")
|
||||
HF = "https://huggingface.co"
|
||||
DEFAULT_PORT = 8080
|
||||
LOAD_TIMEOUT_S = 900.0
|
||||
CHECK_TIMEOUT_S = 180.0
|
||||
INSTALL_LINES = 6
|
||||
# Hugging Face answers 429 with how long to wait (per-IP windows of five minutes). Wait
|
||||
# that long, within reason, a few times; then give up with the error.
|
||||
RATE_LIMIT_RETRIES = 6
|
||||
RATE_LIMIT_WAIT_MAX_S = 120
|
||||
RATE_LIMIT_WAIT_DEFAULT_S = 5
|
||||
|
||||
# Where Homebrew puts programs on Apple silicon and on Intel Macs; the desktop app's PATH
|
||||
# often has neither.
|
||||
_BIN_DIRS = ("/opt/homebrew/bin", "/usr/local/bin")
|
||||
|
||||
|
||||
def supported() -> bool:
|
||||
"""The guided setup is Mac-only for now (ruled 2026-10-06)."""
|
||||
return sys.platform == "darwin"
|
||||
|
||||
|
||||
def _which(program: str) -> Optional[str]:
|
||||
found = shutil.which(program)
|
||||
if found:
|
||||
return found
|
||||
for d in _BIN_DIRS:
|
||||
p = Path(d) / program
|
||||
if p.is_file() and os.access(p, os.X_OK):
|
||||
return str(p)
|
||||
return None
|
||||
|
||||
|
||||
def find_llama_server() -> Optional[str]:
|
||||
return _which("llama-server")
|
||||
|
||||
|
||||
def find_brew() -> Optional[str]:
|
||||
return _which("brew")
|
||||
|
||||
|
||||
def llama_version(path: Optional[str]) -> Optional[str]:
|
||||
"""`llama-server --version` prints `version: 0.6.0 (build …)` on stderr."""
|
||||
if not path:
|
||||
return None
|
||||
try:
|
||||
out = subprocess.run([path, "--version"], capture_output=True, text=True, timeout=15)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
m = re.search(r"version:\s*([^\s(]+)", (out.stdout or "") + (out.stderr or ""))
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def cache_dir() -> Path:
|
||||
"""The folder llama.cpp keeps downloads in, resolved the way llama.cpp 0.6 does."""
|
||||
env = os.environ
|
||||
if env.get("LLAMA_CACHE"):
|
||||
return Path(env["LLAMA_CACHE"]).expanduser()
|
||||
for key in ("HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE"):
|
||||
if env.get(key):
|
||||
return Path(env[key]).expanduser()
|
||||
if env.get("HF_HOME"):
|
||||
return Path(env["HF_HOME"]).expanduser() / "hub"
|
||||
if env.get("XDG_CACHE_HOME"):
|
||||
return Path(env["XDG_CACHE_HOME"]).expanduser() / "huggingface" / "hub"
|
||||
return Path.home() / ".cache" / "huggingface" / "hub"
|
||||
|
||||
|
||||
def _pretty(path: Path) -> str:
|
||||
home = str(Path.home())
|
||||
s = str(path)
|
||||
return "~" + s[len(home):] if s.startswith(home) else s
|
||||
|
||||
|
||||
def _free_bytes(path: Path) -> Optional[int]:
|
||||
p = path
|
||||
while not p.exists() and p != p.parent:
|
||||
p = p.parent
|
||||
try:
|
||||
return shutil.disk_usage(p).free
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
class _Cancelled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# -- the models it offers --------------------------------------------------------------------
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def catalog() -> tuple[dict[str, Any], ...]:
|
||||
try:
|
||||
raw = json.loads(_CATALOG.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return ()
|
||||
return tuple(raw.get("models") or ())
|
||||
|
||||
|
||||
def _catalog_entry(model_id: str) -> Optional[dict[str, Any]]:
|
||||
return next((m for m in catalog() if m.get("id") == model_id), None)
|
||||
|
||||
|
||||
def _repo_dir(repo: str) -> Path:
|
||||
return cache_dir() / ("models--" + repo.replace("/", "--"))
|
||||
|
||||
|
||||
def _cached_file(repo: str, filename: str) -> Optional[Path]:
|
||||
"""A file already in the cache (any snapshot), else None."""
|
||||
snaps = _repo_dir(repo) / "snapshots"
|
||||
if not snaps.is_dir():
|
||||
return None
|
||||
for snap in snaps.iterdir():
|
||||
p = snap / filename
|
||||
if p.exists():
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def _downloaded(entry: dict[str, Any]) -> bool:
|
||||
files = [entry["file"], *([entry["draft"]] if entry.get("draft") else [])]
|
||||
return all(_cached_file(entry["repo"], f) for f in files)
|
||||
|
||||
|
||||
def ollama_models_dir() -> Path:
|
||||
env = os.environ.get("OLLAMA_MODELS", "").strip()
|
||||
return Path(env).expanduser() if env else Path.home() / ".ollama" / "models"
|
||||
|
||||
|
||||
def ollama_models() -> list[dict[str, Any]]:
|
||||
"""The models Ollama downloaded, each with its GGUF file, read from Ollama's manifests
|
||||
(no Ollama server needed). Names as Ollama shows them: `qwen3-coder:30b`."""
|
||||
root = ollama_models_dir()
|
||||
manifests = root / "manifests"
|
||||
if not manifests.is_dir():
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for path in sorted(manifests.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
parts = path.relative_to(manifests).parts # host / namespace / model / tag
|
||||
if len(parts) != 4:
|
||||
continue
|
||||
host, ns, model, tag = parts
|
||||
try:
|
||||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
layer = next(
|
||||
(l for l in doc.get("layers") or [] if l.get("mediaType") == "application/vnd.ollama.image.model"),
|
||||
None,
|
||||
)
|
||||
if not layer or not str(layer.get("digest", "")).startswith("sha256:"):
|
||||
continue
|
||||
blob = root / "blobs" / layer["digest"].replace(":", "-")
|
||||
if not blob.is_file():
|
||||
continue
|
||||
if host == "registry.ollama.ai":
|
||||
name = f"{model}:{tag}" if ns == "library" else f"{ns}/{model}:{tag}"
|
||||
else:
|
||||
name = f"{host}/{ns}/{model}:{tag}"
|
||||
out.append({"name": name, "path": str(blob), "size": int(layer.get("size") or 0)})
|
||||
return out
|
||||
|
||||
|
||||
def overview() -> dict[str, Any]:
|
||||
"""Everything the setup card shows: whether llama.cpp is here, where models go, what
|
||||
to run, and what is running."""
|
||||
memory = local_machine.model_memory_bytes()
|
||||
server = find_llama_server()
|
||||
have_ollama = {m["name"] for m in ollama_models()}
|
||||
suggested = []
|
||||
ollama_pull = []
|
||||
for m in catalog():
|
||||
fit = local_machine.fit_for(m.get("size"), memory)
|
||||
if fit == "too_large":
|
||||
continue
|
||||
suggested.append(
|
||||
{
|
||||
"id": m["id"],
|
||||
"name": m["name"],
|
||||
"vendor": m.get("vendor"),
|
||||
"quant": m.get("quant"),
|
||||
"features": m.get("features") or [],
|
||||
"size": m.get("size"),
|
||||
"fit": fit,
|
||||
"downloaded": _downloaded(m),
|
||||
"license": m.get("license"),
|
||||
"license_url": m.get("license_url"),
|
||||
}
|
||||
)
|
||||
if m.get("ollama") and m["ollama"] not in have_ollama:
|
||||
ollama_pull.append({"name": m["ollama"], "size": m.get("size"), "fit": fit, "command": f"ollama pull {m['ollama']}"})
|
||||
on_mac = [
|
||||
{**m, "fit": local_machine.fit_for(m["size"], memory)}
|
||||
for m in ollama_models()
|
||||
if local_machine.fit_for(m["size"], memory) != "too_large"
|
||||
]
|
||||
cache = cache_dir()
|
||||
return {
|
||||
"supported": supported(),
|
||||
"brew": find_brew(),
|
||||
"installed": {"path": server, "version": llama_version(server)} if server else None,
|
||||
"install_command": "brew install llama.cpp",
|
||||
"cache_dir": _pretty(cache),
|
||||
"cache_free_bytes": _free_bytes(cache),
|
||||
"suggested": suggested,
|
||||
"ollama": on_mac,
|
||||
"ollama_pull": ollama_pull,
|
||||
**runner().state(),
|
||||
}
|
||||
|
||||
|
||||
# -- downloads into llama.cpp's cache --------------------------------------------------------
|
||||
|
||||
|
||||
def _retry_after(resp: httpx.Response) -> int:
|
||||
"""Seconds to wait after a 429, from `Retry-After` (seconds or a date), `RateLimit-Reset`,
|
||||
or Hugging Face's `RateLimit: "resolvers";r=0;t=52` — bounded, at least one second."""
|
||||
h = resp.headers
|
||||
wait: Optional[float] = None
|
||||
raw = h.get("retry-after")
|
||||
if raw:
|
||||
try:
|
||||
wait = float(raw)
|
||||
except ValueError:
|
||||
try:
|
||||
wait = (parsedate_to_datetime(raw) - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
|
||||
except (TypeError, ValueError):
|
||||
wait = None
|
||||
if wait is None and h.get("ratelimit-reset"):
|
||||
try:
|
||||
wait = float(h["ratelimit-reset"])
|
||||
except ValueError:
|
||||
wait = None
|
||||
if wait is None and h.get("ratelimit"):
|
||||
m = re.search(r"\bt=(\d+)", h["ratelimit"])
|
||||
wait = float(m.group(1)) if m else None
|
||||
if wait is None:
|
||||
wait = RATE_LIMIT_WAIT_DEFAULT_S
|
||||
return int(min(max(wait, 1), RATE_LIMIT_WAIT_MAX_S))
|
||||
|
||||
|
||||
def _hf_request(method: str, url: str, *, on_wait: Optional[Callable[[int], None]] = None, **kw: Any) -> httpx.Response:
|
||||
"""One Hugging Face request; on 429 it waits as told and tries again, a few times."""
|
||||
kw.setdefault("timeout", 30)
|
||||
for attempt in range(RATE_LIMIT_RETRIES + 1):
|
||||
resp = httpx.request(method, url, **kw)
|
||||
if resp.status_code != 429 or attempt == RATE_LIMIT_RETRIES:
|
||||
return resp
|
||||
wait = _retry_after(resp)
|
||||
if on_wait:
|
||||
on_wait(wait)
|
||||
else:
|
||||
time.sleep(wait)
|
||||
return resp # pragma: no cover - the loop returns
|
||||
|
||||
|
||||
def _hf_files(repo: str, on_wait: Optional[Callable[[int], None]] = None) -> tuple[str, dict[str, dict[str, Any]]]:
|
||||
"""(commit, {path: {size, oid, sha256}}) for a repo's main branch. `sha256` is the LFS
|
||||
object id, the checksum the download is verified against; None for a non-LFS file."""
|
||||
rev = _hf_request("GET", f"{HF}/api/models/{repo}/revision/main", on_wait=on_wait, follow_redirects=True)
|
||||
rev.raise_for_status()
|
||||
commit = rev.json()["sha"]
|
||||
tree = _hf_request("GET", f"{HF}/api/models/{repo}/tree/{commit}", on_wait=on_wait, follow_redirects=True)
|
||||
tree.raise_for_status()
|
||||
files = {}
|
||||
for f in tree.json():
|
||||
lfs = f.get("lfs") or {}
|
||||
oid = lfs.get("oid") or f.get("oid")
|
||||
files[f["path"]] = {"size": int(lfs.get("size") or f.get("size") or 0), "oid": oid, "sha256": lfs.get("oid")}
|
||||
return commit, files
|
||||
|
||||
|
||||
def _hf_resolve_info(repo: str, filename: str, on_wait: Optional[Callable[[int], None]] = None) -> tuple[str, dict[str, Any]]:
|
||||
"""What the file's own download address says about it, for when the API is not
|
||||
answering: the 302 to the CDN carries the commit, the size and the ETag. The ETag names
|
||||
the blob as huggingface_hub does; it is not trusted as a checksum."""
|
||||
resp = _hf_request("HEAD", f"{HF}/{repo}/resolve/main/{filename}", on_wait=on_wait, follow_redirects=False)
|
||||
if resp.status_code not in (200, 302, 307):
|
||||
raise RuntimeError(f"download failed: HTTP {resp.status_code}")
|
||||
h = resp.headers
|
||||
etag = (h.get("x-linked-etag") or h.get("etag") or "").strip().removeprefix("W/").strip('"')
|
||||
size = int(h.get("x-linked-size") or h.get("content-length") or 0)
|
||||
if not etag or not size:
|
||||
raise RuntimeError(f"download failed: no size for {filename}")
|
||||
return h.get("x-repo-commit") or "main", {"size": size, "oid": etag, "sha256": None}
|
||||
|
||||
|
||||
def _sha256(path: Path, cancel: Optional[threading.Event] = None) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
while True:
|
||||
if cancel is not None and cancel.is_set():
|
||||
raise _Cancelled()
|
||||
chunk = fh.read(8 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _place(repo: str, commit: str, filename: str, blob: Path) -> Path:
|
||||
"""Link a downloaded blob into the snapshot folder, as huggingface_hub does."""
|
||||
snap = _repo_dir(repo) / "snapshots" / commit
|
||||
snap.mkdir(parents=True, exist_ok=True)
|
||||
target = snap / filename
|
||||
if not target.exists():
|
||||
try:
|
||||
target.symlink_to(os.path.relpath(blob, snap))
|
||||
except OSError:
|
||||
shutil.copyfile(blob, target)
|
||||
refs = _repo_dir(repo) / "refs"
|
||||
refs.mkdir(parents=True, exist_ok=True)
|
||||
(refs / "main").write_text(commit, encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
# -- the managed server ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def _state_path(name: str) -> Path:
|
||||
from ..secrets import state_dir
|
||||
|
||||
return state_dir() / name
|
||||
|
||||
|
||||
def _load_settings() -> dict[str, Any]:
|
||||
try:
|
||||
data = json.loads(_state_path("llamacpp.json").read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
|
||||
def _save_settings(data: dict[str, Any]) -> None:
|
||||
p = _state_path("llamacpp.json")
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def _port_free(port: int) -> bool:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
# As llama-server binds: a port a stopped server just left (TIME_WAIT) is free.
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
s.bind(("127.0.0.1", port))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _pick_port() -> int:
|
||||
for port in range(DEFAULT_PORT, DEFAULT_PORT + 20):
|
||||
if _port_free(port):
|
||||
return port
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _is_llama_server(pid: int) -> bool:
|
||||
try:
|
||||
out = subprocess.run(["ps", "-p", str(pid), "-o", "comm="], capture_output=True, text=True, timeout=5)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
return out.stdout.strip().endswith("llama-server")
|
||||
|
||||
|
||||
def _kill(pid: int) -> None:
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except OSError:
|
||||
return
|
||||
for _ in range(50):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _end(proc: subprocess.Popen) -> None:
|
||||
"""Stop a child of ours: SIGTERM, then SIGKILL after 5 s. Waiting on it also reaps it
|
||||
(a zombie still answers `kill -0`, so `_kill` would wait the full time)."""
|
||||
if proc.poll() is not None:
|
||||
return
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(5)
|
||||
|
||||
|
||||
class LlamaCppRunner:
|
||||
"""Installs llama.cpp and runs one llama-server. `phase` is one of idle, installing,
|
||||
downloading, loading, checking, running, failed. One thing at a time."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._cancel = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._proc: Optional[subprocess.Popen] = None
|
||||
self._watcher: Optional[subprocess.Popen] = None
|
||||
self.phase = "idle"
|
||||
self.lines: deque[str] = deque(maxlen=INSTALL_LINES)
|
||||
self.progress: Optional[dict[str, Any]] = None
|
||||
self.model: Optional[dict[str, Any]] = None
|
||||
self.error: Optional[str] = None
|
||||
self.check: Optional[dict[str, bool]] = None
|
||||
self.port: Optional[int] = None
|
||||
# "unverified" when a download could not be checked against its checksum.
|
||||
self.note: Optional[str] = None
|
||||
# Seconds until the next try while Hugging Face rate-limits us.
|
||||
self.retry_in: Optional[int] = None
|
||||
# Set by the manager: point the provider at the server and tick its model.
|
||||
self.on_running: Optional[Callable[[str, str], None]] = None
|
||||
|
||||
# -- reading -----------------------------------------------------------------------
|
||||
|
||||
def busy(self) -> bool:
|
||||
return self.phase in ("installing", "downloading", "loading", "checking")
|
||||
|
||||
def state(self) -> dict[str, Any]:
|
||||
settings = _load_settings()
|
||||
with self._lock:
|
||||
return {
|
||||
"phase": self.phase,
|
||||
"lines": list(self.lines),
|
||||
"progress": dict(self.progress) if self.progress else None,
|
||||
"model": dict(self.model) if self.model else None,
|
||||
"error": self.error,
|
||||
"check": dict(self.check) if self.check else None,
|
||||
"note": self.note,
|
||||
"retry_in": self.retry_in,
|
||||
"address": f"127.0.0.1:{self.port}" if self.port and self.phase == "running" else None,
|
||||
"autostart": bool(settings.get("autostart")),
|
||||
"last_choice": settings.get("choice"),
|
||||
}
|
||||
|
||||
# -- install ------------------------------------------------------------------------
|
||||
|
||||
def install(self) -> dict[str, Any]:
|
||||
brew = find_brew()
|
||||
if not brew:
|
||||
return {"ok": False, "error": "Homebrew is not installed"}
|
||||
return self._run("installing", lambda: self._install(brew))
|
||||
|
||||
def _install(self, brew: str) -> None:
|
||||
env = {**os.environ, "HOMEBREW_NO_ENV_HINTS": "1", "NONINTERACTIVE": "1"}
|
||||
env["PATH"] = ":".join([*_BIN_DIRS, env.get("PATH", "")])
|
||||
proc = subprocess.Popen(
|
||||
[brew, "install", "llama.cpp"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.DEVNULL,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert proc.stdout is not None
|
||||
for line in proc.stdout:
|
||||
if self._cancel.is_set():
|
||||
proc.kill()
|
||||
raise _Cancelled()
|
||||
line = line.rstrip()
|
||||
if line:
|
||||
with self._lock:
|
||||
self.lines.append(line[:200])
|
||||
proc.wait()
|
||||
if proc.returncode != 0 or not find_llama_server():
|
||||
last = self.lines[-1] if self.lines else f"brew exited with {proc.returncode}"
|
||||
raise RuntimeError(last)
|
||||
with self._lock:
|
||||
self.phase = "idle"
|
||||
|
||||
# -- start --------------------------------------------------------------------------
|
||||
|
||||
def start(self, choice: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Run a model: `{"kind": "suggested", "id": …}`, `{"kind": "ollama", "name": …}` or
|
||||
`{"kind": "file", "path": …}`. Stops the one running first."""
|
||||
try:
|
||||
plan = self._plan(choice)
|
||||
except ValueError as e:
|
||||
return {"ok": False, "error": str(e)}
|
||||
return self._run("downloading" if plan.get("download") else "loading", lambda: self._start(choice, plan))
|
||||
|
||||
def _plan(self, choice: dict[str, Any]) -> dict[str, Any]:
|
||||
kind = (choice or {}).get("kind")
|
||||
if kind == "suggested":
|
||||
entry = _catalog_entry(str(choice.get("id") or ""))
|
||||
if not entry:
|
||||
raise ValueError("unknown model")
|
||||
return {
|
||||
"name": entry["name"],
|
||||
"detail": entry.get("quant"),
|
||||
"alias": entry["id"],
|
||||
"download": entry,
|
||||
"spec_type": entry.get("spec_type"),
|
||||
"context_max": entry.get("context_max"),
|
||||
}
|
||||
if kind == "ollama":
|
||||
name = str(choice.get("name") or "")
|
||||
found = next((m for m in ollama_models() if m["name"] == name), None)
|
||||
if not found:
|
||||
raise ValueError(f"Ollama has no model named {name}")
|
||||
return {"name": name, "detail": "from Ollama", "alias": name, "file": found["path"]}
|
||||
if kind == "file":
|
||||
path = Path(str(choice.get("path") or "")).expanduser()
|
||||
if not path.is_file() or path.suffix.lower() != ".gguf":
|
||||
raise ValueError("choose a .gguf file")
|
||||
plan: dict[str, Any] = {"name": path.stem, "detail": "your file", "alias": path.stem, "file": str(path)}
|
||||
# A multi-token-prediction draft saved beside it (`mtp-<name>`), as ggml-org ships them.
|
||||
draft = path.with_name("mtp-" + path.name)
|
||||
if draft.is_file():
|
||||
plan.update(draft=str(draft), spec_type="draft-mtp", detail="your file + MTP")
|
||||
return plan
|
||||
raise ValueError("unknown choice")
|
||||
|
||||
def _start(self, choice: dict[str, Any], plan: dict[str, Any]) -> None:
|
||||
server = find_llama_server()
|
||||
if not server:
|
||||
raise RuntimeError("llama.cpp is not installed")
|
||||
self._stop_process()
|
||||
with self._lock:
|
||||
self.model = {
|
||||
"name": plan["name"],
|
||||
"detail": plan.get("detail"),
|
||||
"alias": plan["alias"],
|
||||
"download": bool(plan.get("download")) and not _downloaded(plan["download"]),
|
||||
}
|
||||
self.check = None
|
||||
if plan.get("download"):
|
||||
plan = {**plan, **self._download(plan["download"])}
|
||||
with self._lock:
|
||||
self.phase = "loading"
|
||||
self.progress = None
|
||||
memory = local_machine.model_memory_bytes()
|
||||
rec = recommendation_for(plan["alias"])
|
||||
model_max = plan.get("context_max") or (rec.context_max if rec else None)
|
||||
context = local_machine.recommended_context(model_max, memory) or 32_768
|
||||
port = _pick_port()
|
||||
args = [server, "-m", plan["file"], "--alias", plan["alias"], "-c", str(context), "--jinja",
|
||||
"--host", "127.0.0.1", "--port", str(port)]
|
||||
if plan.get("draft"):
|
||||
args += ["--spec-type", plan.get("spec_type") or "draft-mtp", "--spec-draft-model", plan["draft"]]
|
||||
log = _state_path("llamacpp-server.log")
|
||||
log.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(log, "wb") as fh:
|
||||
proc = subprocess.Popen(args, stdout=fh, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, start_new_session=True)
|
||||
watcher = subprocess.Popen(
|
||||
["/bin/sh", "-c", 'while kill -0 "$0" 2>/dev/null; do sleep 2; done; kill "$1" 2>/dev/null', str(os.getpid()), str(proc.pid)],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True,
|
||||
)
|
||||
_state_path("llamacpp.pid").write_text(json.dumps({"pid": proc.pid, "watcher": watcher.pid}), encoding="utf-8")
|
||||
with self._lock:
|
||||
self._proc, self._watcher, self.port = proc, watcher, port
|
||||
self.model = {**(self.model or {}), "context": context, "file": _pretty(Path(plan["file"]))}
|
||||
self._wait_loaded(proc, port, log)
|
||||
with self._lock:
|
||||
self.phase = "checking"
|
||||
check = self._check(port, plan["alias"])
|
||||
with self._lock:
|
||||
self.check = check
|
||||
if not check.get("answers"):
|
||||
self._stop_process()
|
||||
raise RuntimeError("the model loaded but did not answer")
|
||||
settings = _load_settings()
|
||||
settings["choice"] = choice
|
||||
_save_settings(settings)
|
||||
if self.on_running:
|
||||
self.on_running(f"http://127.0.0.1:{port}", plan["alias"])
|
||||
with self._lock:
|
||||
self.phase = "running"
|
||||
threading.Thread(target=self._watch_exit, args=(proc, log), daemon=True).start()
|
||||
|
||||
def _download(self, entry: dict[str, Any]) -> dict[str, Any]:
|
||||
wanted = [entry["file"], *([entry["draft"]] if entry.get("draft") else [])]
|
||||
have = {f: _cached_file(entry["repo"], f) for f in wanted}
|
||||
if all(have.values()):
|
||||
return {"file": str(have[entry["file"]]), "draft": str(have[entry["draft"]]) if entry.get("draft") else None}
|
||||
missing = [f for f in wanted if not have[f]]
|
||||
try:
|
||||
# Once, at the start: the commit, each file's size and its sha256 (the LFS id).
|
||||
commit, files = _hf_files(entry["repo"], self._rate_wait)
|
||||
except (httpx.HTTPError, KeyError, ValueError):
|
||||
# The API is not answering (down, or its own rate limit); the files come from
|
||||
# another address with its own limit, so download anyway, unverified.
|
||||
commit, files = "", {}
|
||||
for f in missing:
|
||||
c, info = _hf_resolve_info(entry["repo"], f, self._rate_wait)
|
||||
commit = commit or c
|
||||
files[f] = info
|
||||
with self._lock:
|
||||
self.note = "unverified"
|
||||
for f in missing:
|
||||
if f not in files:
|
||||
raise RuntimeError(f"{f} is no longer in {entry['repo']}")
|
||||
total = sum(files[f]["size"] for f in missing)
|
||||
free = _free_bytes(cache_dir())
|
||||
if free is not None and free < total:
|
||||
raise RuntimeError(f"not enough free space: {total // 10**9} GB needed, {free // 10**9} GB free")
|
||||
done_before = 0
|
||||
started = time.monotonic()
|
||||
for f in missing:
|
||||
blobs = _repo_dir(entry["repo"]) / "blobs"
|
||||
blobs.mkdir(parents=True, exist_ok=True)
|
||||
blob = blobs / files[f]["oid"]
|
||||
part = blob.with_name(blob.name + ".incomplete")
|
||||
size = files[f]["size"]
|
||||
if not (blob.is_file() and blob.stat().st_size == size):
|
||||
url = f"{HF}/{entry['repo']}/resolve/{commit}/{f}"
|
||||
for attempt in range(RATE_LIMIT_RETRIES + 1):
|
||||
wait = self._fetch(url, part, size, done_before, total, started)
|
||||
if wait is None:
|
||||
break
|
||||
if attempt == RATE_LIMIT_RETRIES:
|
||||
raise RuntimeError("download failed: HTTP 429 (Hugging Face is rate-limiting downloads)")
|
||||
self._rate_wait(wait)
|
||||
if part.stat().st_size != size:
|
||||
raise RuntimeError("download incomplete; start again to resume")
|
||||
if files[f].get("sha256"):
|
||||
with self._lock:
|
||||
self.progress = {**(self.progress or {}), "done": done_before + size, "total": total, "verifying": True}
|
||||
if _sha256(part, self._cancel) != files[f]["sha256"]:
|
||||
part.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"{f} did not match its checksum and was removed; start again to download it afresh")
|
||||
part.rename(blob)
|
||||
done_before += size
|
||||
self._set_progress(done_before, total, started)
|
||||
have[f] = _place(entry["repo"], commit, f, blob)
|
||||
return {"file": str(have[entry["file"]]), "draft": str(have[entry["draft"]]) if entry.get("draft") else None}
|
||||
|
||||
def _fetch(self, url: str, part: Path, size: int, done_before: int, total: int, started: float) -> Optional[int]:
|
||||
"""Download `url` into `part`, resuming what is there. None when done; on a 429,
|
||||
the seconds Hugging Face asks us to wait."""
|
||||
have_bytes = part.stat().st_size if part.is_file() else 0
|
||||
headers = {"Range": f"bytes={have_bytes}-"} if have_bytes else {}
|
||||
with httpx.stream("GET", url, headers=headers, follow_redirects=True, timeout=60) as resp:
|
||||
if resp.status_code == 429:
|
||||
return _retry_after(resp)
|
||||
if resp.status_code == 200:
|
||||
have_bytes = 0
|
||||
elif resp.status_code != 206:
|
||||
raise RuntimeError(f"download failed: HTTP {resp.status_code}")
|
||||
with open(part, "ab" if have_bytes else "wb") as out:
|
||||
got = have_bytes
|
||||
last = 0.0
|
||||
for chunk in resp.iter_bytes(1 << 20):
|
||||
if self._cancel.is_set():
|
||||
raise _Cancelled()
|
||||
out.write(chunk)
|
||||
got += len(chunk)
|
||||
now = time.monotonic()
|
||||
if now - last > 0.5:
|
||||
last = now
|
||||
self._set_progress(done_before + got, total, started)
|
||||
return None
|
||||
|
||||
def _rate_wait(self, seconds: int) -> None:
|
||||
"""Wait out a 429, a second at a time so the card can count down and Cancel works."""
|
||||
end = time.monotonic() + seconds
|
||||
while True:
|
||||
left = int(end - time.monotonic() + 0.999)
|
||||
if left <= 0:
|
||||
break
|
||||
with self._lock:
|
||||
self.retry_in = left
|
||||
if self._cancel.wait(min(1.0, left)):
|
||||
with self._lock:
|
||||
self.retry_in = None
|
||||
raise _Cancelled()
|
||||
with self._lock:
|
||||
self.retry_in = None
|
||||
|
||||
def _set_progress(self, done: int, total: int, started: float) -> None:
|
||||
elapsed = max(time.monotonic() - started, 0.001)
|
||||
rate = done / elapsed if done else 0
|
||||
left = int((total - done) / rate) if rate > 0 else None
|
||||
with self._lock:
|
||||
self.progress = {"done": done, "total": total, "seconds_left": left}
|
||||
|
||||
def _wait_loaded(self, proc: subprocess.Popen, port: int, log: Path) -> None:
|
||||
deadline = time.monotonic() + LOAD_TIMEOUT_S
|
||||
while time.monotonic() < deadline:
|
||||
if self._cancel.is_set():
|
||||
self._stop_process()
|
||||
raise _Cancelled()
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError(_last_error(log) or f"llama-server exited with {proc.returncode}")
|
||||
try:
|
||||
if httpx.get(f"http://127.0.0.1:{port}/health", timeout=2).status_code == 200:
|
||||
return
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
self._stop_process()
|
||||
raise RuntimeError("the model took too long to load")
|
||||
|
||||
def _check(self, port: int, alias: str) -> dict[str, bool]:
|
||||
"""One reply and one tool call, so a model that cannot drive the agent is caught
|
||||
here and not in the user's first session."""
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_time",
|
||||
"description": "The current time in a city.",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
},
|
||||
}
|
||||
def ask(body: dict[str, Any]) -> Optional[dict[str, Any]]:
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"http://127.0.0.1:{port}/v1/chat/completions",
|
||||
json={"model": alias, "max_tokens": 1024, **body},
|
||||
timeout=CHECK_TIMEOUT_S,
|
||||
)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
return (r.json().get("choices") or [{}])[0].get("message") or {}
|
||||
except Exception: # noqa: BLE001 - a failed check is reported, not raised
|
||||
return None
|
||||
|
||||
reply = ask({"messages": [{"role": "user", "content": "Say hello in one word."}]})
|
||||
if reply is None:
|
||||
return {"answers": False, "tools": False}
|
||||
call = ask({"messages": [{"role": "user", "content": "What time is it in Paris? Use the tool."}], "tools": [tool]})
|
||||
return {"answers": True, "tools": bool(call and call.get("tool_calls"))}
|
||||
|
||||
def _watch_exit(self, proc: subprocess.Popen, log: Path) -> None:
|
||||
proc.wait()
|
||||
with self._lock:
|
||||
if self._proc is not proc:
|
||||
return # stopped or replaced on purpose
|
||||
self._proc = None
|
||||
self.phase = "failed"
|
||||
self.error = "llama-server stopped: " + (_last_error(log) or f"exit {proc.returncode}")
|
||||
|
||||
# -- stop, cancel, autostart -------------------------------------------------------
|
||||
|
||||
def stop(self) -> dict[str, Any]:
|
||||
self._cancel.set()
|
||||
self._stop_process()
|
||||
with self._lock:
|
||||
self.phase, self.progress, self.error = "idle", None, None
|
||||
return {"ok": True, **self.state()}
|
||||
|
||||
def cancel(self) -> dict[str, Any]:
|
||||
self._cancel.set()
|
||||
return {"ok": True}
|
||||
|
||||
def _stop_process(self) -> None:
|
||||
with self._lock:
|
||||
proc, watcher = self._proc, self._watcher
|
||||
self._proc = self._watcher = None
|
||||
if watcher is not None:
|
||||
_end(watcher)
|
||||
if proc is not None:
|
||||
_end(proc)
|
||||
try:
|
||||
_state_path("llamacpp.pid").unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def set_autostart(self, on: bool) -> dict[str, Any]:
|
||||
settings = _load_settings()
|
||||
settings["autostart"] = bool(on)
|
||||
_save_settings(settings)
|
||||
return {"ok": True, **self.state()}
|
||||
|
||||
def on_app_start(self) -> None:
|
||||
"""At server start: stop a llama-server an earlier run left behind, then start the
|
||||
last model if the user turned on "Start with OpenWorker"."""
|
||||
try:
|
||||
left = json.loads(_state_path("llamacpp.pid").read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
left = None
|
||||
if isinstance(left, dict):
|
||||
pid = left.get("pid")
|
||||
if isinstance(pid, int) and _is_llama_server(pid):
|
||||
_kill(pid)
|
||||
try:
|
||||
_state_path("llamacpp.pid").unlink()
|
||||
except OSError:
|
||||
pass
|
||||
settings = _load_settings()
|
||||
if supported() and settings.get("autostart") and settings.get("choice") and find_llama_server():
|
||||
self.start(settings["choice"])
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._cancel.set()
|
||||
self._stop_process()
|
||||
|
||||
# -- the worker thread ----------------------------------------------------------------
|
||||
|
||||
def _run(self, phase: str, work: Callable[[], None]) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
if self.busy():
|
||||
return {"ok": False, "error": "busy"}
|
||||
self._cancel.clear()
|
||||
self.phase, self.error, self.progress = phase, None, None
|
||||
self.note = self.retry_in = None
|
||||
self.lines.clear()
|
||||
|
||||
def body() -> None:
|
||||
try:
|
||||
work()
|
||||
except _Cancelled:
|
||||
self._stop_process()
|
||||
with self._lock:
|
||||
self.phase, self.progress = "idle", None
|
||||
except Exception as e: # noqa: BLE001 - shown on the card
|
||||
with self._lock:
|
||||
self.phase, self.error = "failed", str(e) or type(e).__name__
|
||||
|
||||
self._thread = threading.Thread(target=body, name="llamacpp-setup", daemon=True)
|
||||
self._thread.start()
|
||||
return {"ok": True, **self.state()}
|
||||
|
||||
|
||||
def _last_error(log: Path) -> Optional[str]:
|
||||
"""The most telling line of llama-server's log: the last error, else the last line."""
|
||||
try:
|
||||
lines = [l.strip() for l in log.read_text(encoding="utf-8", errors="replace").splitlines() if l.strip()]
|
||||
except OSError:
|
||||
return None
|
||||
errors = [l for l in lines if " E " in l or "error" in l.lower()]
|
||||
pick = (errors or lines or [None])[-1]
|
||||
return pick[:200] if pick else None
|
||||
|
||||
|
||||
_runner: Optional[LlamaCppRunner] = None
|
||||
_runner_lock = threading.Lock()
|
||||
|
||||
|
||||
def runner() -> LlamaCppRunner:
|
||||
global _runner
|
||||
with _runner_lock:
|
||||
if _runner is None:
|
||||
_runner = LlamaCppRunner()
|
||||
return _runner
|
||||
|
||||
|
||||
def pick_gguf_file() -> dict[str, Any]:
|
||||
"""The Mac's file chooser, from the server (a web page cannot learn a file's path)."""
|
||||
if sys.platform != "darwin":
|
||||
return {"ok": False, "error": "Mac only"}
|
||||
script = 'POSIX path of (choose file with prompt "Choose a GGUF model file")'
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["osascript", "-e", 'tell application "System Events" to activate', "-e", script],
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return {"ok": False, "error": "no file chooser available"}
|
||||
path = (out.stdout or "").strip()
|
||||
if out.returncode != 0 or not path:
|
||||
return {"ok": False, "canceled": True}
|
||||
return {"ok": True, "path": path}
|
||||
@@ -216,6 +216,17 @@ def create_app(manager: SessionManager) -> FastAPI:
|
||||
except Exception: # never let a bad connector stop the server
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
try:
|
||||
# A llama-server left by an earlier run is stopped; "Start with OpenWorker"
|
||||
# starts the last model again (providers/llamacpp_setup.py).
|
||||
from ..providers import llamacpp_setup
|
||||
|
||||
if llamacpp_setup.supported():
|
||||
await asyncio.to_thread(manager.llamacpp_runner().on_app_start)
|
||||
except Exception:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
yield
|
||||
openrouter_auth.cancel()
|
||||
@@ -2415,6 +2426,40 @@ def create_app(manager: SessionManager) -> FastAPI:
|
||||
def system_get() -> dict[str, Any]:
|
||||
return manager.system_facts()
|
||||
|
||||
# UX-057: llama.cpp set up and run by OpenWorker. GET polls the state while it works.
|
||||
@app.get("/v1/llamacpp/setup")
|
||||
async def llamacpp_setup_get() -> dict[str, Any]:
|
||||
from ..providers import llamacpp_setup
|
||||
|
||||
manager.llamacpp_runner()
|
||||
return await asyncio.to_thread(llamacpp_setup.overview)
|
||||
|
||||
@app.post("/v1/llamacpp/install")
|
||||
def llamacpp_install() -> dict[str, Any]:
|
||||
return manager.llamacpp_runner().install()
|
||||
|
||||
@app.post("/v1/llamacpp/start")
|
||||
def llamacpp_start(body: dict) -> dict[str, Any]:
|
||||
return manager.llamacpp_runner().start((body or {}).get("choice") or {})
|
||||
|
||||
@app.post("/v1/llamacpp/stop")
|
||||
async def llamacpp_stop() -> dict[str, Any]:
|
||||
return await asyncio.to_thread(manager.llamacpp_stop)
|
||||
|
||||
@app.post("/v1/llamacpp/cancel")
|
||||
def llamacpp_cancel() -> dict[str, Any]:
|
||||
return manager.llamacpp_runner().cancel()
|
||||
|
||||
@app.post("/v1/llamacpp/autostart")
|
||||
def llamacpp_autostart(body: dict) -> dict[str, Any]:
|
||||
return manager.llamacpp_runner().set_autostart(bool((body or {}).get("on")))
|
||||
|
||||
@app.post("/v1/llamacpp/pick-file")
|
||||
async def llamacpp_pick_file() -> dict[str, Any]:
|
||||
from ..providers import llamacpp_setup
|
||||
|
||||
return await asyncio.to_thread(llamacpp_setup.pick_gguf_file)
|
||||
|
||||
@app.post("/v1/providers/verify")
|
||||
async def providers_verify(body: dict) -> dict[str, Any]:
|
||||
# Live read-only credential check (sync httpx) — run off the event loop.
|
||||
|
||||
@@ -4223,6 +4223,14 @@ class SessionManager:
|
||||
# A keyless server is "connected" when it answers (cached probes), not
|
||||
# when a form was saved; the page groups connected providers first.
|
||||
row["alive"] = self._ollama_alive() if d.name == "ollama" else self._local_server_alive(d.name)
|
||||
if d.name == "llamacpp":
|
||||
# UX-057: OpenWorker can set it up and run it here (Mac for now).
|
||||
from ..providers import llamacpp_setup
|
||||
|
||||
if llamacpp_setup.supported():
|
||||
st = self.llamacpp_runner().state()
|
||||
row["setup_here"] = True
|
||||
row["managed"] = {"phase": st["phase"], "model": (st["model"] or {}).get("name")}
|
||||
if d.auth == "oauth":
|
||||
# Sign-in state instead of key state; the token values themselves
|
||||
# never leave the SecretStore.
|
||||
@@ -4795,6 +4803,39 @@ class SessionManager:
|
||||
|
||||
return local_machine.system_facts()
|
||||
|
||||
def llamacpp_runner(self):
|
||||
"""The llama-server OpenWorker runs (UX-057), wired to point the `llamacpp`
|
||||
provider at it and tick its model once it answers."""
|
||||
from ..providers import llamacpp_setup
|
||||
|
||||
runner = llamacpp_setup.runner()
|
||||
if runner.on_running is None:
|
||||
runner.on_running = self._llamacpp_running
|
||||
return runner
|
||||
|
||||
def llamacpp_stop(self) -> dict[str, Any]:
|
||||
from ..providers import local_server
|
||||
|
||||
result = self.llamacpp_runner().stop()
|
||||
# The cached probe would say "running" for up to 30 s after the server is gone.
|
||||
local_server.forget_all()
|
||||
self._local_alive_cache = {}
|
||||
return result
|
||||
|
||||
def _llamacpp_running(self, base_url: str, alias: str) -> None:
|
||||
from ..providers import local_server
|
||||
|
||||
local_server.forget_all()
|
||||
self._local_alive_cache = {}
|
||||
model = f"llamacpp:{alias}"
|
||||
# Switching models: the one it served before is gone, so it leaves the picker.
|
||||
previous = self._prefs.get("llamacpp_managed_model")
|
||||
if previous and previous != model:
|
||||
self.remove_model(previous)
|
||||
self._prefs["llamacpp_managed_model"] = model
|
||||
self.set_provider("llamacpp", {"base_url": base_url})
|
||||
self.add_model(model)
|
||||
|
||||
def model_selectable(self, model: str) -> bool:
|
||||
"""Can this machine run `model` right now? Its provider has a key — or, for the
|
||||
keyless Ollama, a local Ollama answers (cached liveness probe)."""
|
||||
@@ -6255,6 +6296,9 @@ class SessionManager:
|
||||
await self.stop_gateway()
|
||||
await self.mcp.aclose()
|
||||
self.audit_store.close()
|
||||
from ..providers import llamacpp_setup
|
||||
|
||||
llamacpp_setup.runner().shutdown()
|
||||
|
||||
# -- automation (scheduled tasks) -------------------------------------------
|
||||
def worker_call_for(self, arguments: dict[str, Any], *, lead_session: str | None = None) -> dict[str, Any] | None:
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 119 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 116 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 116 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 96 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 96 KiB |
@@ -0,0 +1,230 @@
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
import { test } from "./fixtures";
|
||||
|
||||
// UX-057: on a Mac, OpenWorker sets llama.cpp up and runs it — get llama.cpp, pick a
|
||||
// model, start it — and "Connect to it instead" keeps the plain form.
|
||||
|
||||
const GB = 1024 ** 3;
|
||||
|
||||
const provider = (managed: { phase: string; model: string | null }) => ({
|
||||
name: "llamacpp", title: "llama.cpp", kind: "local", needs_key: false, configured: false, values: {},
|
||||
suggested_models: [], recommended_model: null, alive: managed.phase === "running", setup_here: true, managed,
|
||||
fields: [
|
||||
{ key: "base_url", label: "Server address", secret: false, required: false, help: "", placeholder: "http://localhost:8080" },
|
||||
{ key: "api_key", label: "API key (only if the server was started with one)", secret: true, required: false, help: "", placeholder: "" },
|
||||
],
|
||||
});
|
||||
|
||||
function base(over: Record<string, unknown> = {}) {
|
||||
return {
|
||||
supported: true, brew: "/opt/homebrew/bin/brew", installed: null, install_command: "brew install llama.cpp",
|
||||
cache_dir: "~/.cache/huggingface/hub", cache_free_bytes: 56 * GB,
|
||||
suggested: [
|
||||
{ id: "nemotron-3.5-lightning", name: "Nemotron 3.5 Lightning", vendor: "NVIDIA", quant: "Q8 + MTP", features: ["thinking", "tools"], size: 36e9, fit: "runs_well", downloaded: false,
|
||||
license: "OpenMDW-1.1", license_url: "https://openmdw.ai/license/1-1/" },
|
||||
{ id: "gpt-oss-20b", name: "gpt-oss 20B", vendor: "OpenAI", quant: "MXFP4", features: ["reasoning", "tools"], size: 12e9, fit: "runs_well", downloaded: false,
|
||||
license: "Apache-2.0", license_url: "https://huggingface.co/openai/gpt-oss-20b/blob/main/LICENSE" },
|
||||
],
|
||||
ollama: [{ name: "qwen3-coder:30b", path: "/Users/me/.ollama/models/blobs/sha256-1194", size: 18e9, fit: "runs_well" }],
|
||||
ollama_pull: [{ name: "gpt-oss:20b", size: 12e9, fit: "runs_well", command: "ollama pull gpt-oss:20b" }],
|
||||
phase: "idle", lines: [], progress: null, model: null, error: null, check: null, note: null, retry_in: null, address: null,
|
||||
autostart: false, last_choice: null,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
const RUNNING = {
|
||||
phase: "running",
|
||||
model: { name: "qwen3-coder:30b", detail: "from Ollama", alias: "qwen3-coder:30b", download: false, context: 131072, file: "~/.ollama/models/blobs/sha256-1194" },
|
||||
check: { answers: true, tools: true },
|
||||
address: "127.0.0.1:8080",
|
||||
};
|
||||
|
||||
async function openPage(page: Page) {
|
||||
await page.goto("/");
|
||||
await page.getByTestId("account-row").click();
|
||||
await page.getByRole("button", { name: "Settings", exact: true }).click();
|
||||
await page.getByRole("button", { name: "Models & Keys" }).click();
|
||||
}
|
||||
|
||||
test("llama.cpp: install, pick a model Ollama has, start it, and it runs", async ({ page }) => {
|
||||
let state = base();
|
||||
const posts: { path: string; body: unknown }[] = [];
|
||||
await page.route("**/v1/providers", (route) =>
|
||||
route.fulfill({ json: [provider({ phase: state.phase, model: (state.model as { name: string } | null)?.name ?? null })] }),
|
||||
);
|
||||
await page.route("**/v1/providers/llamacpp/models", (route) => route.fulfill({ json: { provider: "llamacpp", alive: true, models: [] } }));
|
||||
await page.route("**/v1/llamacpp/**", (route) => {
|
||||
const path = new URL(route.request().url()).pathname.replace("/v1/llamacpp/", "");
|
||||
if (route.request().method() === "GET") {
|
||||
// Each poll moves the work along, as the server's thread would.
|
||||
const now = state;
|
||||
if (state.phase === "installing") state = base({ installed: { path: "/opt/homebrew/bin/llama-server", version: "0.6.0" } });
|
||||
if (state.phase === "loading") state = base({ installed: { path: "/opt/homebrew/bin/llama-server", version: "0.6.0" }, ...RUNNING });
|
||||
return route.fulfill({ json: now });
|
||||
}
|
||||
posts.push({ path, body: route.request().postDataJSON() });
|
||||
if (path === "install") state = base({ phase: "installing", lines: ["==> Fetching downloads for: llama.cpp", "==> Pouring llama.cpp--0.6.0.arm64_tahoe.bottle.tar.gz"] });
|
||||
if (path === "start") state = base({ installed: { path: "/opt/homebrew/bin/llama-server", version: "0.6.0" }, phase: "loading", model: { ...RUNNING.model } });
|
||||
if (path === "autostart") state = { ...state, autostart: (route.request().postDataJSON() as { on: boolean }).on };
|
||||
return route.fulfill({ json: { ok: true, ...state } });
|
||||
});
|
||||
|
||||
await openPage(page);
|
||||
await expect(page.getByTestId("set-provider-llamacpp")).toContainText("Not set up · Set up on this Mac");
|
||||
await page.getByTestId("set-provider-llamacpp").click();
|
||||
const main = page.locator("main");
|
||||
await expect(main).toContainText("Runs open models on this Mac");
|
||||
await expect(page.getByTestId("llamacpp-step-get")).toHaveAttribute("data-state", "now");
|
||||
await expect(page.getByTestId("llamacpp-step-get")).toContainText("brew install llama.cpp");
|
||||
// The guided card replaces the address form until the user asks for it.
|
||||
await expect(page.getByTestId("set-field-api_key")).toHaveCount(0);
|
||||
await expect(page.getByTestId("set-endpoint-link")).toHaveCount(0);
|
||||
|
||||
await page.getByTestId("llamacpp-install").click();
|
||||
await expect(page.getByTestId("llamacpp-install-log")).toContainText("Pouring llama.cpp");
|
||||
await expect(page.getByTestId("llamacpp-step-get")).toHaveAttribute("data-state", "done");
|
||||
await expect(main).toContainText("Installed · 0.6.0");
|
||||
|
||||
// Pick: suggested models, what Ollama has, the pull command, and where models go.
|
||||
const pick = page.getByTestId("llamacpp-step-pick");
|
||||
await expect(pick).toHaveAttribute("data-state", "now");
|
||||
await expect(pick).toContainText("Nemotron 3.5 Lightning");
|
||||
await expect(page.getByTestId("llamacpp-pull")).toContainText("ollama pull gpt-oss:20b");
|
||||
await expect(page.getByTestId("llamacpp-cache")).toContainText("~/.cache/huggingface/hub");
|
||||
// Each model that downloads names its licence, linked, before Download.
|
||||
await expect(page.getByTestId("llamacpp-license-nemotron-3.5-lightning")).toContainText("Licence: OpenMDW-1.1");
|
||||
await expect(page.getByTestId("llamacpp-license-gpt-oss-20b")).toContainText("Licence: Apache-2.0");
|
||||
await expect(page.getByTestId("llamacpp-start")).toHaveText("Download and start");
|
||||
await page.getByTestId("llamacpp-model-o:qwen3-coder:30b").click();
|
||||
await expect(page.getByTestId("llamacpp-start")).toHaveText("Start");
|
||||
await page.getByTestId("llamacpp-start").click();
|
||||
|
||||
const running = page.getByTestId("llamacpp-running");
|
||||
await expect(running).toContainText("qwen3-coder:30b");
|
||||
await expect(running).toContainText("128K");
|
||||
await expect(running).toContainText("127.0.0.1:8080");
|
||||
await expect(page.getByTestId("llamacpp-no-tools")).toHaveCount(0);
|
||||
await expect(main).toContainText("✓ Running · 1 model");
|
||||
// "Start with OpenWorker" is off until the user turns it on.
|
||||
const auto = page.getByTestId("llamacpp-autostart");
|
||||
await expect(auto).toHaveAttribute("aria-checked", "false");
|
||||
await auto.click();
|
||||
await expect(auto).toHaveAttribute("aria-checked", "true");
|
||||
|
||||
expect(posts.map((p) => p.path)).toEqual(["install", "start", "autostart"]);
|
||||
expect(posts[1].body).toEqual({ choice: { kind: "ollama", name: "qwen3-coder:30b" } });
|
||||
expect(posts[2].body).toEqual({ on: true });
|
||||
|
||||
await page.getByTestId("set-back").click();
|
||||
await expect(page.getByTestId("set-provider-llamacpp")).toContainText("✓ Running · qwen3-coder:30b");
|
||||
});
|
||||
|
||||
test("llama.cpp: a model that cannot call tools runs with a warning", async ({ page }) => {
|
||||
const state = base({ installed: { path: "/opt/homebrew/bin/llama-server", version: "0.6.0" }, ...RUNNING, check: { answers: true, tools: false } });
|
||||
await page.route("**/v1/providers", (route) => route.fulfill({ json: [provider({ phase: "running", model: "qwen3-coder:30b" })] }));
|
||||
await page.route("**/v1/providers/llamacpp/models", (route) => route.fulfill({ json: { provider: "llamacpp", alive: true, models: [] } }));
|
||||
await page.route("**/v1/llamacpp/**", (route) => route.fulfill({ json: state }));
|
||||
await openPage(page);
|
||||
await page.getByTestId("set-provider-llamacpp").click();
|
||||
await expect(page.getByTestId("llamacpp-no-tools")).toContainText("did not call a tool");
|
||||
});
|
||||
|
||||
test("llama.cpp: Connect to it instead shows the address form, and back", async ({ page }) => {
|
||||
await page.route("**/v1/providers", (route) => route.fulfill({ json: [provider({ phase: "idle", model: null })] }));
|
||||
await page.route("**/v1/providers/llamacpp/models", (route) => route.fulfill({ json: { provider: "llamacpp", alive: false, models: [] } }));
|
||||
await page.route("**/v1/llamacpp/**", (route) => route.fulfill({ json: base({ brew: null }) }));
|
||||
await openPage(page);
|
||||
await page.getByTestId("set-provider-llamacpp").click();
|
||||
// No Homebrew: it says so instead of offering Install.
|
||||
await expect(page.getByTestId("llamacpp-no-brew")).toBeVisible();
|
||||
await expect(page.getByTestId("llamacpp-install")).toHaveCount(0);
|
||||
await page.getByTestId("llamacpp-connect-instead").click();
|
||||
// Today's form: the key field, the address behind "Custom endpoint", the help line.
|
||||
await expect(page.getByTestId("set-field-api_key")).toBeVisible();
|
||||
await expect(page.getByTestId("set-endpoint-link")).toBeVisible();
|
||||
await expect(page.locator("main")).toContainText("llama-server");
|
||||
await page.getByTestId("llamacpp-setup-here").click();
|
||||
await expect(page.getByTestId("llamacpp-setup")).toBeVisible();
|
||||
});
|
||||
|
||||
const INSTALLED = { installed: { path: "/opt/homebrew/bin/llama-server", version: "0.6.0" } };
|
||||
const DOWNLOADING = {
|
||||
...INSTALLED,
|
||||
phase: "downloading",
|
||||
model: { name: "Nemotron 3.5 Lightning", detail: "Q8 + MTP", alias: "nemotron-3.5-lightning", download: true },
|
||||
progress: { done: 12.4e9, total: 35.8e9, seconds_left: 840 },
|
||||
};
|
||||
|
||||
test("llama.cpp: a download waits out a rate limit, verifies the file, and says when it could not", async ({ page }) => {
|
||||
// Each poll moves the download along: rate-limited, then verifying with the API gone.
|
||||
const states = [
|
||||
base({ ...DOWNLOADING, retry_in: 42 }),
|
||||
base({ ...DOWNLOADING, retry_in: 42 }),
|
||||
base({ ...DOWNLOADING, progress: { done: 35.8e9, total: 35.8e9, seconds_left: 0, verifying: true }, note: "unverified" }),
|
||||
];
|
||||
let poll = 0;
|
||||
await page.route("**/v1/providers", (route) => route.fulfill({ json: [provider({ phase: "downloading", model: "Nemotron 3.5 Lightning" })] }));
|
||||
await page.route("**/v1/providers/llamacpp/models", (route) => route.fulfill({ json: { provider: "llamacpp", alive: false, models: [] } }));
|
||||
await page.route("**/v1/llamacpp/**", (route) => route.fulfill({ json: states[Math.min(poll++, states.length - 1)] }));
|
||||
await openPage(page);
|
||||
await expect(page.getByTestId("set-provider-llamacpp")).toContainText("Starting…");
|
||||
await page.getByTestId("set-provider-llamacpp").click();
|
||||
await expect(page.getByTestId("llamacpp-step-start")).toHaveAttribute("data-state", "now");
|
||||
const progress = page.getByTestId("llamacpp-progress");
|
||||
await expect(progress).toHaveText("Hugging Face is rate-limiting downloads, retrying in 42s");
|
||||
await expect(progress).toHaveText("Verifying the download…");
|
||||
await expect(page.getByTestId("llamacpp-note")).toContainText("checksum was not checked");
|
||||
await expect(page.getByTestId("llamacpp-cancel")).toBeVisible();
|
||||
});
|
||||
|
||||
// The screenshots in docs/assets (light and dark): the three-step card before anything is
|
||||
// installed, the model picker, and the running card.
|
||||
for (const theme of ["light", "dark"] as const) {
|
||||
test(`llama.cpp: screenshots (${theme})`, async ({ page }) => {
|
||||
let state = base();
|
||||
await page.addInitScript((value) => localStorage.setItem("openwork-theme", value), theme);
|
||||
await page.setViewportSize({ width: 1280, height: 900 });
|
||||
await page.route("**/v1/providers", (route) =>
|
||||
route.fulfill({ json: [provider({ phase: state.phase, model: (state.model as { name: string } | null)?.name ?? null })] }),
|
||||
);
|
||||
await page.route("**/v1/providers/llamacpp/models", (route) =>
|
||||
route.fulfill({
|
||||
json: {
|
||||
provider: "llamacpp", alive: state.phase === "running",
|
||||
models: state.phase === "running"
|
||||
? [{ model: "llamacpp:qwen3-coder:30b", name: "qwen3-coder:30b", size_bytes: 18e9, tools: true, thinking: false, vision: null, remote: false,
|
||||
parameter_size: null, quantization: null, context_max: 262144, context: 131072, context_from: "server", fit: "runs_well", recommendation: null }]
|
||||
: [],
|
||||
},
|
||||
}),
|
||||
);
|
||||
await page.route("**/v1/llamacpp/**", (route) => route.fulfill({ json: state }));
|
||||
// "Your system", as a 96 GB Mac reports it.
|
||||
await page.route("**/v1/system", (route) =>
|
||||
route.fulfill({
|
||||
json: { processor: "Apple M4 Max", graphics: "Apple M4 Max (40-core GPU)", kind: "apple_silicon", memory_bytes: 96 * GB, gpu_memory_bytes: null,
|
||||
storage_free_bytes: 420 * GB, storage_total_bytes: 926 * GB, model_memory_bytes: 72 * GB, runs_well_up_to_bytes: 64 * GB },
|
||||
}),
|
||||
);
|
||||
const suffix = theme === "dark" ? "-dark" : "";
|
||||
|
||||
await openPage(page);
|
||||
await page.getByTestId("set-provider-llamacpp").click();
|
||||
await expect(page.getByTestId("llamacpp-step-get")).toHaveAttribute("data-state", "now");
|
||||
await page.screenshot({ path: `test-results/models-llamacpp-setup${suffix}.png` });
|
||||
|
||||
state = base(INSTALLED);
|
||||
await page.getByTestId("set-back").click();
|
||||
await page.getByTestId("set-provider-llamacpp").click();
|
||||
await expect(page.getByTestId("llamacpp-step-pick")).toHaveAttribute("data-state", "now");
|
||||
await expect(page.getByTestId("llamacpp-license-nemotron-3.5-lightning")).toBeVisible();
|
||||
await page.screenshot({ path: `test-results/models-llamacpp-pick${suffix}.png` });
|
||||
|
||||
state = base({ ...INSTALLED, ...RUNNING });
|
||||
await page.getByTestId("set-back").click();
|
||||
await page.getByTestId("set-provider-llamacpp").click();
|
||||
await expect(page.getByTestId("llamacpp-running")).toContainText("qwen3-coder:30b");
|
||||
await page.screenshot({ path: `test-results/models-llamacpp-running${suffix}.png` });
|
||||
});
|
||||
}
|
||||
@@ -2520,6 +2520,9 @@ export interface ProviderInfo {
|
||||
is_custom?: boolean; // user-created OpenAI-compatible provider
|
||||
// Local servers only: whether the server answers right now (cached probe).
|
||||
alive?: boolean;
|
||||
// llama.cpp on a Mac (UX-057): OpenWorker can set it up and run it, and what it runs.
|
||||
setup_here?: boolean;
|
||||
managed?: { phase: LlamaCppPhase; model: string | null };
|
||||
signed_in?: boolean;
|
||||
account?: string | null; // signed-in account label (email or id)
|
||||
authorizing?: boolean;
|
||||
@@ -2717,6 +2720,77 @@ export async function getLocalModels(
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// -- llama.cpp set up and run by OpenWorker (UX-057) ---------------------------------
|
||||
export type LlamaCppPhase = "idle" | "installing" | "downloading" | "loading" | "checking" | "running" | "failed";
|
||||
export type ModelFit = "runs_well" | "tight" | "too_large" | "unknown";
|
||||
export type LlamaCppChoice =
|
||||
| { kind: "suggested"; id: string }
|
||||
| { kind: "ollama"; name: string }
|
||||
| { kind: "file"; path: string };
|
||||
|
||||
export interface LlamaCppSetup {
|
||||
supported: boolean;
|
||||
brew: string | null;
|
||||
installed: { path: string; version: string | null } | null;
|
||||
install_command: string;
|
||||
cache_dir: string;
|
||||
cache_free_bytes: number | null;
|
||||
suggested: {
|
||||
id: string;
|
||||
name: string;
|
||||
vendor: string | null;
|
||||
quant: string | null;
|
||||
features: string[];
|
||||
size: number | null;
|
||||
fit: ModelFit;
|
||||
downloaded: boolean;
|
||||
// The model's licence, named and linked before Download (from its Hugging Face card).
|
||||
license: string | null;
|
||||
license_url: string | null;
|
||||
}[];
|
||||
ollama: { name: string; path: string; size: number; fit: ModelFit }[];
|
||||
ollama_pull: { name: string; size: number | null; fit: ModelFit; command: string }[];
|
||||
phase: LlamaCppPhase;
|
||||
lines: string[];
|
||||
// `verifying`: the file is down and its sha256 is being checked against Hugging Face's.
|
||||
progress: { done: number; total: number; seconds_left: number | null; verifying?: boolean } | null;
|
||||
model: { name: string; detail: string | null; alias: string; download?: boolean; context?: number; file?: string } | null;
|
||||
error: string | null;
|
||||
check: { answers: boolean; tools: boolean } | null;
|
||||
// "unverified": the Hugging Face API was unreachable, so the download's checksum was not checked.
|
||||
note: "unverified" | null;
|
||||
// Seconds until the next try while Hugging Face rate-limits the download.
|
||||
retry_in: number | null;
|
||||
address: string | null;
|
||||
autostart: boolean;
|
||||
last_choice: LlamaCppChoice | null;
|
||||
}
|
||||
|
||||
async function llamacppPost(path: string, body?: unknown): Promise<LlamaCppSetup & { ok?: boolean; error?: string }> {
|
||||
const res = await fetch(`${httpBase()}/v1/llamacpp/${path}`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body ?? {}),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function getLlamaCppSetup(): Promise<LlamaCppSetup> {
|
||||
const res = await fetch(`${httpBase()}/v1/llamacpp/setup`, { signal: AbortSignal.timeout(15_000) });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
export const installLlamaCpp = () => llamacppPost("install");
|
||||
export const startLlamaCpp = (choice: LlamaCppChoice) => llamacppPost("start", { choice });
|
||||
export const stopLlamaCpp = () => llamacppPost("stop");
|
||||
export const cancelLlamaCpp = () => llamacppPost("cancel");
|
||||
export const setLlamaCppAutostart = (on: boolean) => llamacppPost("autostart", { on });
|
||||
export async function pickGgufFile(): Promise<{ ok: boolean; path?: string; canceled?: boolean; error?: string }> {
|
||||
const res = await fetch(`${httpBase()}/v1/llamacpp/pick-file`, { method: "POST" });
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function getProviders(): Promise<ProviderInfo[]> {
|
||||
const res = await fetch(`${httpBase()}/v1/providers`);
|
||||
const data = await res.json();
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
import { CloudSignInInline, CloudStatusPending } from "./connectors/CloudSignIn";
|
||||
import { Icon } from "./Icon";
|
||||
import { ModelChecklist } from "./ModelChecklist";
|
||||
import { LlamaCppPage } from "./models/LlamaCppSetup";
|
||||
import { LocalModelsTable, YourSystem } from "./models/LocalModels";
|
||||
import { PickModelDialog } from "./models/PickModelDialog";
|
||||
import { ProviderGroups } from "./models/ProviderGroups";
|
||||
@@ -209,13 +210,34 @@ export function ModelsTab() {
|
||||
);
|
||||
}
|
||||
|
||||
const removeKeyFooter = <ProviderKeyFooter info={info} credentialed={ps.credentialed} onRemove={ps.removeKey} />;
|
||||
const localTables = (key = "", withTable = true) => (
|
||||
<>
|
||||
<YourSystem />
|
||||
{withTable && <LocalModelsTable
|
||||
key={key}
|
||||
provider={ps.sel!}
|
||||
curated={settings.models}
|
||||
defaultModel={settings.model}
|
||||
onChanged={refreshSettings}
|
||||
onConfigure={(m) => setDialog(m)}
|
||||
/>}
|
||||
{dialogEl}
|
||||
</>
|
||||
);
|
||||
|
||||
if (ps.sel === "llamacpp" && info?.setup_here) {
|
||||
// UX-057: OpenWorker sets llama.cpp up and runs it on this Mac.
|
||||
return (
|
||||
<div>
|
||||
<LlamaCppPage ps={ps} tp="set" footer={removeKeyFooter} onChanged={refreshSettings} after={localTables} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<ProviderForm
|
||||
ps={ps}
|
||||
tp="set"
|
||||
footer={<ProviderKeyFooter info={info} credentialed={ps.credentialed} onRemove={ps.removeKey} />}
|
||||
/>
|
||||
<ProviderForm ps={ps} tp="set" footer={removeKeyFooter} />
|
||||
|
||||
{info?.env_key && (
|
||||
<p className="text-meta text-muted mt-3 leading-relaxed" data-testid="set-env-key-note">
|
||||
@@ -230,17 +252,7 @@ export function ModelsTab() {
|
||||
|
||||
{info?.kind === "local" ? (
|
||||
// A local provider: the machine, then what the server holds (UX-055).
|
||||
<>
|
||||
<YourSystem />
|
||||
<LocalModelsTable
|
||||
provider={ps.sel}
|
||||
curated={settings.models}
|
||||
defaultModel={settings.model}
|
||||
onChanged={refreshSettings}
|
||||
onConfigure={(m) => setDialog(m)}
|
||||
/>
|
||||
{dialogEl}
|
||||
</>
|
||||
localTables()
|
||||
) : info?.configured ? (
|
||||
<div className="mt-6">
|
||||
<div className={SEC_H + " mb-1.5"}>{t("manage.models")}</div>
|
||||
|
||||
@@ -0,0 +1,554 @@
|
||||
// llama.cpp set up and run by OpenWorker on a Mac (UX-057): get llama.cpp, pick a model
|
||||
// sized to this Mac, start it. Then a card for the running server. Polls the state while
|
||||
// something runs, like the sandbox setup. "Connect to it instead" keeps the plain form.
|
||||
|
||||
import { useCallback, useEffect, useRef, useState, type ReactNode } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import {
|
||||
cancelLlamaCpp,
|
||||
getLlamaCppSetup,
|
||||
installLlamaCpp,
|
||||
pickGgufFile,
|
||||
setLlamaCppAutostart,
|
||||
startLlamaCpp,
|
||||
stopLlamaCpp,
|
||||
type LlamaCppChoice,
|
||||
type LlamaCppSetup,
|
||||
type ModelFit,
|
||||
} from "../../api";
|
||||
import { ProviderForm, type ProviderSetupState } from "../../providers/ProviderSetup";
|
||||
import { openExternal } from "../../tauri";
|
||||
import { fmtBytes, fmtContext } from "./LocalModels";
|
||||
|
||||
const BUSY = new Set(["installing", "downloading", "loading", "checking"]);
|
||||
|
||||
function choiceKey(c: LlamaCppChoice | null): string {
|
||||
if (!c) return "";
|
||||
return c.kind === "suggested" ? `s:${c.id}` : c.kind === "ollama" ? `o:${c.name}` : `f:${c.path}`;
|
||||
}
|
||||
|
||||
function fromKey(key: string): LlamaCppChoice | null {
|
||||
const v = key.slice(2);
|
||||
if (key.startsWith("s:")) return { kind: "suggested", id: v };
|
||||
if (key.startsWith("o:")) return { kind: "ollama", name: v };
|
||||
if (key.startsWith("f:")) return { kind: "file", path: v };
|
||||
return null;
|
||||
}
|
||||
|
||||
function minutes(s: number | null | undefined): number | null {
|
||||
return s == null ? null : Math.max(1, Math.round(s / 60));
|
||||
}
|
||||
|
||||
/** The llama.cpp page when OpenWorker can run it here: the provider head, the guided
|
||||
* card, then `after` (Your system, and the models table once a server runs), re-mounted
|
||||
* when what runs changes so it reads the new server. */
|
||||
export function LlamaCppPage({
|
||||
ps,
|
||||
tp,
|
||||
footer,
|
||||
onChanged,
|
||||
after,
|
||||
}: {
|
||||
ps: ProviderSetupState;
|
||||
tp: string;
|
||||
footer?: ReactNode;
|
||||
onChanged: () => void;
|
||||
after: (key: string, withTable: boolean) => ReactNode;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const [setup, setSetup] = useState<LlamaCppSetup | null>(null);
|
||||
const [loadError, setLoadError] = useState(false);
|
||||
// A server the user runs themselves answers and OpenWorker runs nothing: their form.
|
||||
const [connect, setConnect] = useState<boolean | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
const s = await getLlamaCppSetup();
|
||||
setSetup(s);
|
||||
setLoadError(false);
|
||||
return s;
|
||||
} catch {
|
||||
setLoadError(true);
|
||||
return null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
useEffect(() => {
|
||||
if (connect !== null || !setup) return;
|
||||
setConnect(!!ps.info?.alive && setup.phase !== "running" && !BUSY.has(setup.phase));
|
||||
}, [setup, connect, ps.info?.alive]);
|
||||
|
||||
// Poll while something runs; when it settles, refresh the providers and the picker.
|
||||
const phase = setup?.phase;
|
||||
const wasBusy = useRef(false);
|
||||
useEffect(() => {
|
||||
const busy = !!phase && BUSY.has(phase);
|
||||
if (!busy) {
|
||||
if (wasBusy.current) {
|
||||
wasBusy.current = false;
|
||||
void ps.refreshProviders();
|
||||
onChanged();
|
||||
}
|
||||
return;
|
||||
}
|
||||
wasBusy.current = true;
|
||||
const id = window.setInterval(() => void load(), 1000);
|
||||
return () => window.clearInterval(id);
|
||||
}, [phase, load, ps, onChanged]);
|
||||
|
||||
const status = (() => {
|
||||
if (!setup || connect) return undefined;
|
||||
const cls = "block text-meta text-muted";
|
||||
if (setup.phase === "running") return <span className="block text-meta text-ok font-medium">{t("llamacpp.status_running")}</span>;
|
||||
if (setup.phase === "installing") return <span className={cls}>{t("llamacpp.status_installing")}</span>;
|
||||
if (BUSY.has(setup.phase)) return <span className={cls}>{t("llamacpp.status_starting")}</span>;
|
||||
if (setup.installed)
|
||||
return <span className={cls}>{t("llamacpp.status_installed", { version: setup.installed.version || "" })}</span>;
|
||||
return <span className={cls}>{t("provider.not_set_up")}</span>;
|
||||
})();
|
||||
|
||||
const guided =
|
||||
connect === false && setup ? (
|
||||
<SetupCard setup={setup} onState={setSetup} reload={load} onConnectInstead={() => setConnect(true)} onChanged={() => { void ps.refreshProviders(); onChanged(); }} />
|
||||
) : connect === null && loadError ? (
|
||||
<p className="text-meta text-muted mt-4" data-testid="llamacpp-load-failed">
|
||||
{t("llamacpp.load_failed")}{" "}
|
||||
<button className="underline underline-offset-2 hover:text-ink" onClick={() => void load()}>
|
||||
{t("manage.retry")}
|
||||
</button>
|
||||
</p>
|
||||
) : connect === null ? (
|
||||
<p className="text-meta text-faint mt-4">{t("llamacpp.loading")}</p>
|
||||
) : null;
|
||||
|
||||
return (
|
||||
<>
|
||||
<ProviderForm
|
||||
ps={ps}
|
||||
tp={tp}
|
||||
footer={footer}
|
||||
status={status}
|
||||
blurb={connect ? undefined : t("llamacpp.blurb")}
|
||||
guided={guided ?? undefined}
|
||||
/>
|
||||
{connect && (
|
||||
<p className="text-meta text-faint mt-3">
|
||||
<button className="text-muted underline decoration-line underline-offset-2 hover:text-ink" onClick={() => setConnect(false)} data-testid="llamacpp-setup-here">
|
||||
{t("llamacpp.setup_here")}
|
||||
</button>
|
||||
</p>
|
||||
)}
|
||||
{/* What the server reports, once there is a server: one it runs, or the user's own. */}
|
||||
{after(`${setup?.phase === "running" ? setup.model?.alias : ""}:${setup?.address ?? ""}`, !!connect || setup?.phase === "running")}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function SetupCard({
|
||||
setup,
|
||||
onState,
|
||||
reload,
|
||||
onConnectInstead,
|
||||
onChanged,
|
||||
}: {
|
||||
setup: LlamaCppSetup;
|
||||
onState: (s: LlamaCppSetup) => void;
|
||||
reload: () => Promise<LlamaCppSetup | null>;
|
||||
onConnectInstead: () => void;
|
||||
onChanged: () => void;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const [picked, setPicked] = useState<string>(() => choiceKey(setup.last_choice) || (setup.suggested[0] ? `s:${setup.suggested[0].id}` : ""));
|
||||
const [file, setFile] = useState<string | null>(setup.last_choice?.kind === "file" ? setup.last_choice.path : null);
|
||||
const [switching, setSwitching] = useState(false);
|
||||
const [lastStart, setLastStart] = useState<LlamaCppChoice | null>(null);
|
||||
const [copied, setCopied] = useState("");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const { phase } = setup;
|
||||
const installed = !!setup.installed;
|
||||
const busy = BUSY.has(phase);
|
||||
|
||||
const act = async (fn: () => Promise<LlamaCppSetup & { ok?: boolean; error?: string }>) => {
|
||||
setError(null);
|
||||
try {
|
||||
const r = await fn();
|
||||
if (r.ok === false && r.error) setError(r.error);
|
||||
if (r.phase) onState(r);
|
||||
else await reload();
|
||||
// Stop and the switch settle at once; the poll covers the rest.
|
||||
if (r.phase && !BUSY.has(r.phase)) onChanged();
|
||||
} catch (e) {
|
||||
setError(String((e as Error).message || e));
|
||||
}
|
||||
};
|
||||
|
||||
const copy = (text: string) => {
|
||||
void navigator.clipboard?.writeText(text);
|
||||
setCopied(text);
|
||||
window.setTimeout(() => setCopied(""), 1500);
|
||||
};
|
||||
|
||||
const start = (choice: LlamaCppChoice | null) => {
|
||||
if (!choice) return;
|
||||
setLastStart(choice);
|
||||
setSwitching(false);
|
||||
void act(() => startLlamaCpp(choice));
|
||||
};
|
||||
|
||||
const chooseFile = async () => {
|
||||
const r = await pickGgufFile();
|
||||
if (r.ok && r.path) {
|
||||
setFile(r.path);
|
||||
setPicked(`f:${r.path}`);
|
||||
}
|
||||
};
|
||||
|
||||
// -- the running server (screen 5) ---------------------------------------------------
|
||||
if (phase === "running" && !switching && setup.model) {
|
||||
const m = setup.model;
|
||||
return (
|
||||
<div className="mt-4 rounded-xl border border-line bg-panel px-5 py-4" data-testid="llamacpp-running">
|
||||
<div className="flex items-center gap-2.5">
|
||||
<span className="w-[7px] h-[7px] rounded-full bg-ok shrink-0" />
|
||||
<span className="text-ui font-semibold truncate">{m.name}</span>
|
||||
{m.detail && <span className="text-meta text-muted truncate">{m.detail}</span>}
|
||||
<span className="ml-auto flex gap-2 shrink-0">
|
||||
<button className={BTN} onClick={() => setSwitching(true)} data-testid="llamacpp-switch">
|
||||
{t("llamacpp.switch_model")}
|
||||
</button>
|
||||
<button className={BTN} onClick={() => void act(stopLlamaCpp)} data-testid="llamacpp-stop">
|
||||
{t("llamacpp.stop")}
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
{setup.check && !setup.check.tools && (
|
||||
<p className="text-meta text-warnInk mt-2.5" data-testid="llamacpp-no-tools">
|
||||
{t("llamacpp.no_tools")}
|
||||
</p>
|
||||
)}
|
||||
{setup.note === "unverified" && <p className="text-meta text-faint mt-2.5" data-testid="llamacpp-note">{t("llamacpp.note_unverified")}</p>}
|
||||
<div className="grid grid-cols-[120px_1fr] gap-x-3 gap-y-1 text-ui mt-3">
|
||||
<span className="text-muted">{t("llamacpp.context")}</span>
|
||||
<span>{fmtContext(m.context)}</span>
|
||||
<span className="text-muted">{t("llamacpp.address")}</span>
|
||||
<span className="font-mono text-meta">{setup.address}</span>
|
||||
<span className="text-muted">{t("llamacpp.model_file")}</span>
|
||||
<span className="font-mono text-meta truncate" title={m.file}>{m.file}</span>
|
||||
</div>
|
||||
<div className="border-t border-line my-3.5" />
|
||||
<label className="flex items-center gap-2.5 text-meta cursor-pointer">
|
||||
{t("llamacpp.autostart")}
|
||||
<Switch on={setup.autostart} onChange={(on) => void act(() => setLlamaCppAutostart(on))} testid="llamacpp-autostart" />
|
||||
</label>
|
||||
<p className="text-meta text-faint mt-1">{t(setup.autostart ? "llamacpp.autostart_on" : "llamacpp.autostart_off")}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// -- the three steps (screens 1–4) ---------------------------------------------------
|
||||
const pickNow = installed && !busy && phase !== "installing";
|
||||
const startNow = busy && phase !== "installing";
|
||||
const failedStart = phase === "failed" && installed && (lastStart || setup.model);
|
||||
const choice = fromKey(picked);
|
||||
const pickedSuggested = choice?.kind === "suggested" ? setup.suggested.find((s) => s.id === choice.id) : undefined;
|
||||
const pickedSummary = setup.model ? [setup.model.name, setup.model.detail].filter(Boolean).join(" · ") : "";
|
||||
const p = setup.progress;
|
||||
|
||||
return (
|
||||
<div className="mt-4">
|
||||
<div className="rounded-xl border border-line bg-panel px-5 py-4" data-testid="llamacpp-setup">
|
||||
<ol className="relative pl-[26px] before:content-[''] before:absolute before:left-[7px] before:top-2 before:bottom-2 before:w-px before:bg-lineStrong">
|
||||
{/* 1. Get llama.cpp */}
|
||||
<Step state={installed ? "done" : "now"} title={t("llamacpp.step_get")} sum={installed ? t("llamacpp.installed_sum", { version: setup.installed?.version || "" }) : phase === "installing" ? setup.install_command : undefined} testid="llamacpp-step-get">
|
||||
{!installed && phase !== "installing" && (
|
||||
<div className="mt-2 text-ui">
|
||||
{setup.brew ? (
|
||||
<>
|
||||
<p className="text-muted">{t("llamacpp.not_installed")}</p>
|
||||
<Cmd text={setup.install_command} copied={copied === setup.install_command} onCopy={copy} t={t} />
|
||||
<div className="flex items-center gap-2.5 mt-2.5">
|
||||
<button className={BTN_PRIMARY} onClick={() => void act(installLlamaCpp)} data-testid="llamacpp-install">
|
||||
{phase === "failed" ? t("manage.retry") : t("llamacpp.install")}
|
||||
</button>
|
||||
<span className="text-meta text-faint">{t("llamacpp.install_hint")}</span>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<p className="text-muted" data-testid="llamacpp-no-brew">
|
||||
{t("llamacpp.no_brew")}{" "}
|
||||
<button className="underline underline-offset-2 hover:text-ink" onClick={() => openExternal("https://brew.sh")}>brew.sh ↗</button>
|
||||
</p>
|
||||
)}
|
||||
{phase === "failed" && setup.error && <p className="text-meta text-danger mt-2" data-testid="llamacpp-error">{setup.error}</p>}
|
||||
</div>
|
||||
)}
|
||||
{phase === "installing" && (
|
||||
<div className="mt-2">
|
||||
<pre className="rounded-lg bg-[#1d1f23] text-[#c9ced6] px-3 py-2 text-[11.5px] leading-relaxed font-mono whitespace-pre-wrap break-all" data-testid="llamacpp-install-log">
|
||||
{setup.lines.length ? setup.lines.join("\n") : "…"}
|
||||
</pre>
|
||||
<button className={BTN + " mt-2"} onClick={() => void act(cancelLlamaCpp)} data-testid="llamacpp-cancel">
|
||||
{t("llamacpp.cancel")}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</Step>
|
||||
|
||||
{/* 2. Pick a model */}
|
||||
<Step
|
||||
state={pickNow ? "now" : startNow ? "done" : "todo"}
|
||||
title={t("llamacpp.step_pick")}
|
||||
sum={startNow ? pickedSummary : undefined}
|
||||
testid="llamacpp-step-pick"
|
||||
>
|
||||
{pickNow && (
|
||||
<div className="mt-2">
|
||||
<Group label={t("llamacpp.suggested")} />
|
||||
<div className="rounded-[10px] border border-line">
|
||||
{setup.suggested.map((s) => (
|
||||
<ModelRow
|
||||
key={s.id}
|
||||
id={`s:${s.id}`}
|
||||
picked={picked}
|
||||
onPick={setPicked}
|
||||
name={s.name}
|
||||
sub={[s.vendor, s.quant, ...s.features.map((f) => t(`llamacpp.feature_${f}`))].filter(Boolean).join(" · ")}
|
||||
size={s.size}
|
||||
fit={s.fit}
|
||||
extra={s.downloaded ? t("llamacpp.downloaded") : undefined}
|
||||
line={
|
||||
s.license ? (
|
||||
<button
|
||||
className="text-label text-faint underline decoration-line underline-offset-2 hover:text-ink"
|
||||
onClick={(e) => { e.stopPropagation(); if (s.license_url) openExternal(s.license_url); }}
|
||||
data-testid={`llamacpp-license-${s.id}`}
|
||||
>
|
||||
{t("llamacpp.license", { name: s.license })} ↗
|
||||
</button>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
))}
|
||||
{setup.suggested.length === 0 && <p className="px-3 py-2.5 text-meta text-faint">{t("llamacpp.none_fit")}</p>}
|
||||
</div>
|
||||
<Group label={t("llamacpp.on_this_mac")} />
|
||||
<div className="rounded-[10px] border border-line">
|
||||
{setup.ollama.map((o) => (
|
||||
<ModelRow key={o.name} id={`o:${o.name}`} picked={picked} onPick={setPicked} name={o.name} sub={t("llamacpp.from_ollama")} size={o.size} fit={o.fit} />
|
||||
))}
|
||||
<ModelRow
|
||||
id={file ? `f:${file}` : "f:"}
|
||||
picked={picked}
|
||||
onPick={() => (file ? setPicked(`f:${file}`) : void chooseFile())}
|
||||
name={file ? file.split("/").pop() || file : t("llamacpp.choose_file")}
|
||||
sub={file ? file : t("llamacpp.choose_file_sub")}
|
||||
action={file ? <button className="text-meta text-muted hover:text-ink" onClick={(e) => { e.stopPropagation(); void chooseFile(); }}>{t("llamacpp.change")}</button> : undefined}
|
||||
/>
|
||||
</div>
|
||||
{setup.ollama_pull.length > 0 && (
|
||||
<>
|
||||
<Group label={t("llamacpp.not_downloaded")} />
|
||||
<div className="rounded-[10px] border border-line" data-testid="llamacpp-pull">
|
||||
{setup.ollama_pull.map((o) => (
|
||||
<div key={o.name} className="flex items-center gap-3 px-3 py-2.5 border-t border-line first:border-t-0">
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="block text-ui font-medium">{o.name}</span>
|
||||
<span className="block text-label text-faint">{t("llamacpp.pull_sub", { size: fmtBytes(o.size), fit: t(`manage.fit_${o.fit}`) })}</span>
|
||||
</span>
|
||||
<Cmd text={o.command} copied={copied === o.command} onCopy={copy} t={t} compact />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<p className="text-meta text-faint mt-2" data-testid="llamacpp-cache">
|
||||
{t("llamacpp.cache_where")} <code className="font-mono">{setup.cache_dir}</code>
|
||||
{setup.cache_free_bytes != null && <> · {t("llamacpp.free", { size: fmtBytes(setup.cache_free_bytes) })}</>}
|
||||
</p>
|
||||
{failedStart && setup.error && (
|
||||
<p className="text-meta text-danger mt-2" data-testid="llamacpp-error">{setup.error}</p>
|
||||
)}
|
||||
<div className="flex items-center gap-2.5 mt-3">
|
||||
<button className={BTN_PRIMARY} disabled={!choice || picked === "f:"} onClick={() => start(choice)} data-testid="llamacpp-start">
|
||||
{pickedSuggested && !pickedSuggested.downloaded ? t("llamacpp.download_and_start") : t("llamacpp.start")}
|
||||
</button>
|
||||
{switching && (
|
||||
<button className={BTN} onClick={() => setSwitching(false)}>
|
||||
{t("llamacpp.cancel")}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</Step>
|
||||
|
||||
{/* 3. Start it */}
|
||||
<Step state={startNow ? "now" : "todo"} title={t("llamacpp.step_start")} testid="llamacpp-step-start">
|
||||
{startNow && (
|
||||
<div className="mt-2">
|
||||
{phase === "downloading" && (
|
||||
<>
|
||||
<p className="text-meta text-muted" data-testid="llamacpp-progress">
|
||||
{setup.retry_in != null
|
||||
? t("llamacpp.rate_limited", { seconds: setup.retry_in })
|
||||
: p?.verifying
|
||||
? t("llamacpp.verifying")
|
||||
: p
|
||||
? t("llamacpp.downloading", { done: fmtBytes(p.done), total: fmtBytes(p.total) }) +
|
||||
(minutes(p.seconds_left) != null ? " · " + t("llamacpp.minutes_left", { count: minutes(p.seconds_left) ?? 0 }) : "")
|
||||
: t("llamacpp.download_starting")}
|
||||
</p>
|
||||
<div className="h-1.5 rounded bg-paper overflow-hidden my-2">
|
||||
<i className="block h-full bg-muted rounded" style={{ width: `${p && p.total ? Math.round((100 * p.done) / p.total) : 0}%` }} />
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<ul className="text-meta space-y-0.5">
|
||||
{setup.model?.download && <Check done={phase !== "downloading"} now={phase === "downloading"} text={t("llamacpp.check_download")} />}
|
||||
<Check done={phase === "checking"} now={phase === "loading"} text={t("llamacpp.check_load")} />
|
||||
<Check done={false} now={phase === "checking"} text={t("llamacpp.check_answers")} />
|
||||
<Check done={false} now={false} text={t("llamacpp.check_tools")} />
|
||||
</ul>
|
||||
{setup.note === "unverified" && <p className="text-meta text-faint mt-2" data-testid="llamacpp-note">{t("llamacpp.note_unverified")}</p>}
|
||||
<div className="flex items-center gap-2.5 mt-2.5">
|
||||
<button className={BTN} onClick={() => void act(cancelLlamaCpp)} data-testid="llamacpp-cancel">
|
||||
{t("llamacpp.cancel")}
|
||||
</button>
|
||||
<span className="text-meta text-faint">{t("llamacpp.leave_ok")}</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</Step>
|
||||
</ol>
|
||||
</div>
|
||||
<p className="text-meta text-faint mt-3">
|
||||
{t("llamacpp.own_server")}{" "}
|
||||
<button className="text-muted underline decoration-line underline-offset-2 hover:text-ink" onClick={onConnectInstead} data-testid="llamacpp-connect-instead">
|
||||
{t("llamacpp.connect_instead")}
|
||||
</button>
|
||||
</p>
|
||||
{error && <p className="text-meta text-danger mt-2">{error}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const BTN = "rounded-lg border border-lineStrong bg-panel px-3 py-1.5 text-ui text-ink hover:border-ink/40 disabled:opacity-40";
|
||||
const BTN_PRIMARY = "rounded-lg border border-ink bg-ink px-3 py-1.5 text-ui font-medium text-panel hover:brightness-110 disabled:opacity-40";
|
||||
|
||||
function Step({ state, title, sum, testid, children }: { state: "done" | "now" | "todo"; title: string; sum?: string; testid: string; children?: ReactNode }) {
|
||||
return (
|
||||
<li className="relative pb-4 last:pb-0" data-testid={testid} data-state={state}>
|
||||
<span
|
||||
className={
|
||||
"absolute -left-[26px] top-[3px] w-[15px] h-[15px] rounded-full border-[1.5px] grid place-items-center text-[9px] " +
|
||||
(state === "now" ? "border-ink bg-ink" : state === "done" ? "border-lineStrong bg-paper text-muted" : "border-lineStrong bg-panel")
|
||||
}
|
||||
>
|
||||
{state === "done" ? "✓" : ""}
|
||||
</span>
|
||||
<div className="flex items-baseline gap-2">
|
||||
<span className={"text-ui " + (state === "todo" ? "text-faint" : "font-semibold")}>{title}</span>
|
||||
{sum && <span className="text-meta text-muted truncate">{sum}</span>}
|
||||
</div>
|
||||
{children}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
function Group({ label }: { label: string }) {
|
||||
return <div className="text-[11px] tracking-wide uppercase text-faint font-semibold mt-3 mb-1">{label}</div>;
|
||||
}
|
||||
|
||||
function ModelRow({
|
||||
id,
|
||||
picked,
|
||||
onPick,
|
||||
name,
|
||||
sub,
|
||||
size,
|
||||
fit,
|
||||
extra,
|
||||
action,
|
||||
line,
|
||||
}: {
|
||||
id: string;
|
||||
picked: string;
|
||||
onPick: (id: string) => void;
|
||||
name: string;
|
||||
sub?: string;
|
||||
size?: number | null;
|
||||
fit?: ModelFit;
|
||||
extra?: string;
|
||||
action?: ReactNode;
|
||||
// A third line under the name: the licence, for a model that downloads.
|
||||
line?: ReactNode;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const on = picked === id && id !== "f:";
|
||||
return (
|
||||
<div
|
||||
role="radio"
|
||||
aria-checked={on}
|
||||
tabIndex={0}
|
||||
className="grid grid-cols-[18px_1fr_auto_auto] gap-2.5 items-center px-3 py-2.5 border-t border-line first:border-t-0 cursor-pointer hover:bg-paper/50"
|
||||
onClick={() => onPick(id)}
|
||||
onKeyDown={(e) => (e.key === "Enter" || e.key === " ") && onPick(id)}
|
||||
data-testid={`llamacpp-model-${id}`}
|
||||
>
|
||||
<span className={"w-3.5 h-3.5 rounded-full " + (on ? "border-[4.5px] border-ink" : "border-[1.5px] border-lineStrong")} />
|
||||
<span className="min-w-0">
|
||||
<span className="block text-ui font-medium truncate">{name}</span>
|
||||
{sub && <span className="block text-label text-faint truncate">{sub}</span>}
|
||||
{line && <span className="block truncate">{line}</span>}
|
||||
</span>
|
||||
<span className="text-meta text-muted text-right">{action ?? (size ? fmtBytes(size) : "")}</span>
|
||||
<span className={"text-meta text-right min-w-[72px] " + (fit === "runs_well" ? "text-ok" : "text-warnInk")}>
|
||||
{extra ? <span className="text-muted">{extra}</span> : fit && fit !== "unknown" ? t(`manage.fit_${fit}`) : ""}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Cmd({ text, copied, onCopy, t, compact }: { text: string; copied: boolean; onCopy: (s: string) => void; t: (k: string) => string; compact?: boolean }) {
|
||||
return (
|
||||
<button
|
||||
className={"flex items-center gap-2 rounded-lg bg-paper font-mono text-meta text-ink text-left " + (compact ? "px-2 py-1 shrink-0" : "w-full px-2.5 py-2 mt-2")}
|
||||
onClick={() => onCopy(text)}
|
||||
title={t("provider.copy_command")}
|
||||
data-testid={`llamacpp-copy-${text.split(" ").pop()}`}
|
||||
>
|
||||
<span className="truncate">{text}</span>
|
||||
<span className="ml-auto font-sans text-label text-faint pl-2">{copied ? t("llamacpp.copied") : t("llamacpp.copy")}</span>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function Check({ done, now, text }: { done: boolean; now: boolean; text: string }) {
|
||||
return (
|
||||
<li className={"flex gap-2 " + (done || now ? "text-ink" : "text-muted")}>
|
||||
<span className="w-3.5 text-center text-muted">
|
||||
{done ? "✓" : now ? <span className="inline-block w-2.5 h-2.5 rounded-full border-2 border-lineStrong border-t-ink animate-spin" /> : "·"}
|
||||
</span>
|
||||
{text}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
function Switch({ on, onChange, testid }: { on: boolean; onChange: (on: boolean) => void; testid: string }) {
|
||||
return (
|
||||
<button
|
||||
role="switch"
|
||||
aria-checked={on}
|
||||
className={"ml-auto relative w-[34px] h-5 rounded-full transition-colors " + (on ? "bg-ink" : "bg-lineStrong")}
|
||||
onClick={() => onChange(!on)}
|
||||
data-testid={testid}
|
||||
>
|
||||
<span className={"absolute top-0.5 w-4 h-4 rounded-full bg-panel transition-all " + (on ? "right-0.5" : "left-0.5")} />
|
||||
</button>
|
||||
);
|
||||
}
|
||||
@@ -22,7 +22,7 @@ export function providerKind(p: ProviderInfo): Kind {
|
||||
* answers (the setup state's `keylessOk`). */
|
||||
export function providerConnected(p: ProviderInfo, keylessOk: Set<string>): boolean {
|
||||
if (p.auth === "oauth") return !!p.signed_in;
|
||||
if (!p.needs_key) return !!p.alive || keylessOk.has(p.name);
|
||||
if (!p.needs_key) return !!p.alive || keylessOk.has(p.name) || p.managed?.phase === "running";
|
||||
return p.configured;
|
||||
}
|
||||
|
||||
@@ -70,14 +70,18 @@ export function ProviderGroups({
|
||||
<ProviderMark name={p.name} title={p.title} />
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className={"block text-ui font-semibold leading-tight truncate" + (connected ? "" : " text-muted")}>{p.title}</span>
|
||||
{connected && count > 0 ? (
|
||||
{p.managed?.phase === "running" && p.managed.model ? (
|
||||
<span className="block text-meta text-ok font-medium truncate">{t("llamacpp.card_running", { model: p.managed.model })}</span>
|
||||
) : p.managed && !connected && ["installing", "downloading", "loading", "checking"].includes(p.managed.phase) ? (
|
||||
<span className="block text-meta text-muted truncate">{t(p.managed.phase === "installing" ? "llamacpp.status_installing" : "llamacpp.status_starting")}</span>
|
||||
) : connected && count > 0 ? (
|
||||
<span className="block text-meta text-ok font-medium truncate">
|
||||
{t(p.needs_key || p.auth === "oauth" ? "provider.connected_count" : "provider.running_count", { count })}
|
||||
</span>
|
||||
) : connected && !p.needs_key && p.auth !== "oauth" ? (
|
||||
<span className="block text-meta text-ok font-medium truncate">{t("provider.running")}</span>
|
||||
) : !connected && !p.needs_key && p.auth !== "oauth" && p.kind === "local" && p.name !== "ollama" ? (
|
||||
<span className="block text-meta text-faint truncate">{t("provider.not_connected")}</span>
|
||||
<span className="block text-meta text-faint truncate">{t(p.setup_here ? "llamacpp.card_not_set_up" : "provider.not_connected")}</span>
|
||||
) : (
|
||||
ps.statusFor(p, { lastUsed: true })
|
||||
)}
|
||||
|
||||
@@ -2874,8 +2874,71 @@
|
||||
"next": "Next",
|
||||
"add": "Add",
|
||||
"save": "Save"
|
||||
}
|
||||
,
|
||||
},
|
||||
"llamacpp": {
|
||||
"blurb": "Runs open models on this Mac. OpenWorker can set it up and keep it running for you.",
|
||||
"loading": "Checking this Mac…",
|
||||
"load_failed": "Couldn't read the llama.cpp setup.",
|
||||
"status_installing": "Installing…",
|
||||
"status_starting": "Starting…",
|
||||
"status_installed": "Installed · {{version}}",
|
||||
"status_running": "✓ Running · 1 model",
|
||||
"card_running": "✓ Running · {{model}}",
|
||||
"card_not_set_up": "Not set up · Set up on this Mac",
|
||||
"step_get": "Get llama.cpp",
|
||||
"step_pick": "Pick a model",
|
||||
"step_start": "Start it",
|
||||
"installed_sum": "{{version}}, Homebrew",
|
||||
"not_installed": "llama.cpp is not installed. OpenWorker installs it with Homebrew:",
|
||||
"install": "Install",
|
||||
"install_hint": "About 30 MB. You can watch the output.",
|
||||
"no_brew": "llama.cpp installs with Homebrew, which is not on this Mac. Install Homebrew first:",
|
||||
"cancel": "Cancel",
|
||||
"copy": "Copy",
|
||||
"copied": "Copied",
|
||||
"suggested": "Suggested for this Mac",
|
||||
"on_this_mac": "Already on this Mac",
|
||||
"not_downloaded": "Recommended, not downloaded yet",
|
||||
"pull_sub": "{{size}} · {{fit}} · in Ollama, then it shows above",
|
||||
"none_fit": "None of the suggested models fits this Mac's memory.",
|
||||
"from_ollama": "From Ollama · nothing to download",
|
||||
"downloaded": "Downloaded",
|
||||
"license": "Licence: {{name}}",
|
||||
"choose_file": "Choose a GGUF file…",
|
||||
"choose_file_sub": "A model file you downloaded yourself",
|
||||
"change": "Change",
|
||||
"feature_thinking": "thinking",
|
||||
"feature_tools": "tools",
|
||||
"feature_reasoning": "reasoning levels",
|
||||
"cache_where": "Models download to llama.cpp's own cache,",
|
||||
"free": "{{size}} free",
|
||||
"download_and_start": "Download and start",
|
||||
"start": "Start",
|
||||
"download_starting": "Starting the download…",
|
||||
"downloading": "Downloading · {{done}} of {{total}}",
|
||||
"verifying": "Verifying the download…",
|
||||
"rate_limited": "Hugging Face is rate-limiting downloads, retrying in {{seconds}}s",
|
||||
"note_unverified": "Hugging Face's API couldn't be reached, so this download's checksum was not checked.",
|
||||
"minutes_left_one": "about {{count}} min left",
|
||||
"minutes_left_other": "about {{count}} min left",
|
||||
"check_download": "Download the model",
|
||||
"check_load": "Load the model",
|
||||
"check_answers": "Check that it answers",
|
||||
"check_tools": "Check that it calls a tool",
|
||||
"leave_ok": "You can leave this page; it keeps going.",
|
||||
"switch_model": "Switch model",
|
||||
"stop": "Stop",
|
||||
"no_tools": "It answers, but it did not call a tool when asked. Coworkers use tools on every task, so pick a larger model for real work.",
|
||||
"context": "Context",
|
||||
"address": "Address",
|
||||
"model_file": "Model file",
|
||||
"autostart": "Start with OpenWorker",
|
||||
"autostart_off": "Off: it runs only after you press Start. It always stops when OpenWorker quits.",
|
||||
"autostart_on": "On: it starts when OpenWorker opens. It always stops when OpenWorker quits.",
|
||||
"own_server": "Already run llama-server yourself?",
|
||||
"connect_instead": "Connect to it instead",
|
||||
"setup_here": "Set up llama.cpp on this Mac instead"
|
||||
},
|
||||
"customization": {
|
||||
"title": "Customization",
|
||||
"lead": "Pick the services OpenWorker uses behind its tools. Each choice applies to new sessions on this machine.",
|
||||
|
||||
@@ -2834,8 +2834,71 @@
|
||||
"next": "下一步",
|
||||
"add": "添加",
|
||||
"save": "保存"
|
||||
}
|
||||
,
|
||||
},
|
||||
"llamacpp": {
|
||||
"blurb": "在这台 Mac 上运行开源模型。OpenWorker 可以为你安装并保持运行。",
|
||||
"loading": "正在检查这台 Mac…",
|
||||
"load_failed": "无法读取 llama.cpp 设置。",
|
||||
"status_installing": "正在安装…",
|
||||
"status_starting": "正在启动…",
|
||||
"status_installed": "已安装 · {{version}}",
|
||||
"status_running": "✓ 运行中 · 1 个模型",
|
||||
"card_running": "✓ 运行中 · {{model}}",
|
||||
"card_not_set_up": "未设置 · 在这台 Mac 上设置",
|
||||
"step_get": "获取 llama.cpp",
|
||||
"step_pick": "选择模型",
|
||||
"step_start": "启动",
|
||||
"installed_sum": "{{version}}，Homebrew",
|
||||
"not_installed": "尚未安装 llama.cpp。OpenWorker 会用 Homebrew 安装：",
|
||||
"install": "安装",
|
||||
"install_hint": "约 30 MB。可以查看输出。",
|
||||
"no_brew": "llama.cpp 通过 Homebrew 安装，但这台 Mac 上没有 Homebrew。请先安装 Homebrew：",
|
||||
"cancel": "取消",
|
||||
"copy": "复制",
|
||||
"copied": "已复制",
|
||||
"suggested": "推荐给这台 Mac",
|
||||
"on_this_mac": "这台 Mac 上已有",
|
||||
"not_downloaded": "推荐，尚未下载",
|
||||
"pull_sub": "{{size}} · {{fit}} · 下载到 Ollama 后会显示在上方",
|
||||
"none_fit": "推荐的模型都超出这台 Mac 的内存。",
|
||||
"from_ollama": "来自 Ollama · 无需下载",
|
||||
"downloaded": "已下载",
|
||||
"license": "许可证：{{name}}",
|
||||
"choose_file": "选择 GGUF 文件…",
|
||||
"choose_file_sub": "你自己下载的模型文件",
|
||||
"change": "更改",
|
||||
"feature_thinking": "思考",
|
||||
"feature_tools": "工具",
|
||||
"feature_reasoning": "推理级别",
|
||||
"cache_where": "模型下载到 llama.cpp 自己的缓存：",
|
||||
"free": "剩余 {{size}}",
|
||||
"download_and_start": "下载并启动",
|
||||
"start": "启动",
|
||||
"download_starting": "正在开始下载…",
|
||||
"downloading": "正在下载 · {{done}} / {{total}}",
|
||||
"verifying": "正在校验下载…",
|
||||
"rate_limited": "Hugging Face 正在限制下载，{{seconds}} 秒后重试",
|
||||
"note_unverified": "无法连接 Hugging Face API，未校验此次下载的校验和。",
|
||||
"minutes_left_one": "约剩 {{count}} 分钟",
|
||||
"minutes_left_other": "约剩 {{count}} 分钟",
|
||||
"check_download": "下载模型",
|
||||
"check_load": "加载模型",
|
||||
"check_answers": "检查能否回答",
|
||||
"check_tools": "检查能否调用工具",
|
||||
"leave_ok": "可以离开此页面，它会继续进行。",
|
||||
"switch_model": "切换模型",
|
||||
"stop": "停止",
|
||||
"no_tools": "它能回答，但在要求时没有调用工具。同事在每个任务中都会使用工具，正式工作请选择更大的模型。",
|
||||
"context": "上下文",
|
||||
"address": "地址",
|
||||
"model_file": "模型文件",
|
||||
"autostart": "随 OpenWorker 启动",
|
||||
"autostart_off": "关闭：只有按下启动后才运行。OpenWorker 退出时总会停止。",
|
||||
"autostart_on": "开启：OpenWorker 打开时启动。OpenWorker 退出时总会停止。",
|
||||
"own_server": "已经自己运行 llama-server？",
|
||||
"connect_instead": "改为连接到它",
|
||||
"setup_here": "改为在这台 Mac 上设置 llama.cpp"
|
||||
},
|
||||
"customization": {
|
||||
"title": "自定义",
|
||||
"lead": "选择 OpenWorker 各工具背后使用的服务。所做选择对本机的新会话生效。",
|
||||
|
||||
@@ -482,10 +482,18 @@ export function ProviderForm({
|
||||
ps,
|
||||
tp,
|
||||
footer,
|
||||
status,
|
||||
blurb,
|
||||
guided,
|
||||
}: {
|
||||
ps: ProviderSetupState;
|
||||
tp: string;
|
||||
footer?: ReactNode;
|
||||
// A guided setup (llama.cpp on a Mac, UX-057) replaces the status line, the blurb and
|
||||
// the fields; the footer stays.
|
||||
status?: ReactNode;
|
||||
blurb?: string;
|
||||
guided?: ReactNode;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const { info, sel } = ps;
|
||||
@@ -568,10 +576,11 @@ export function ProviderForm({
|
||||
<ProviderMark name={info?.name || ""} title={info?.title || ""} size={36} />
|
||||
<span className="min-w-0">
|
||||
<span className="block text-body font-semibold leading-tight">{info?.title}</span>
|
||||
{info ? ps.statusFor(info) : null}
|
||||
{status ?? (info ? ps.statusFor(info) : null)}
|
||||
</span>
|
||||
</div>
|
||||
{info?.blurb && <p className="text-meta text-faint mt-1">{info.blurb}</p>}
|
||||
{(blurb ?? info?.blurb) && <p className="text-meta text-faint mt-1">{blurb ?? info?.blurb}</p>}
|
||||
{guided}
|
||||
|
||||
{info?.auth === "oauth" &&
|
||||
(info.name === "openrouter-account" ? (
|
||||
@@ -580,7 +589,7 @@ export function ProviderForm({
|
||||
<OAuthSignIn info={info} tp={tp} onChanged={ps.refreshProviders} />
|
||||
))}
|
||||
|
||||
{fieldsAll
|
||||
{!guided && fieldsAll
|
||||
.filter(
|
||||
(f) =>
|
||||
!f.show_when &&
|
||||
@@ -674,7 +683,7 @@ export function ProviderForm({
|
||||
{t("provider.takes_a_minute")}
|
||||
</p>
|
||||
)}
|
||||
{info && !info.needs_key && info.auth !== "oauth" && LOCAL_HELP[info.name] && (
|
||||
{!guided && info && !info.needs_key && info.auth !== "oauth" && LOCAL_HELP[info.name] && (
|
||||
<p className="text-meta text-faint mt-2">
|
||||
{t(LOCAL_HELP[info.name].desc)}{" "}
|
||||
<button
|
||||
@@ -691,7 +700,7 @@ export function ProviderForm({
|
||||
(owner calls 2026-07-18 + 2026-07-19). */}
|
||||
{(() => {
|
||||
const keyed = (info?.fields || []).some((x) => x.secret);
|
||||
const ep = keyed ? (info?.fields || []).find((f) => f.key === "base_url") : undefined;
|
||||
const ep = keyed && !guided ? (info?.fields || []).find((f) => f.key === "base_url") : undefined;
|
||||
if (!ep) return null;
|
||||
if (!ps.showEndpoint)
|
||||
return (
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
"""llama.cpp set up and run by OpenWorker (providers/llamacpp_setup.py, UX-057): where
|
||||
models go, what it offers, the download into llama.cpp's cache, and the managed server's
|
||||
start, check, switch, stop and clean-up — against a stand-in llama-server."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from coworker.providers import llamacpp_setup as setup
|
||||
from coworker.providers import local_machine
|
||||
|
||||
GB = 1024**3
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _state(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
|
||||
monkeypatch.setenv("LLAMA_CACHE", str(tmp_path / "cache"))
|
||||
monkeypatch.setenv("OLLAMA_MODELS", str(tmp_path / "ollama"))
|
||||
monkeypatch.setattr(local_machine, "model_memory_bytes", lambda: 32 * GB)
|
||||
monkeypatch.setattr(setup, "_runner", None)
|
||||
yield
|
||||
if setup._runner is not None:
|
||||
setup._runner.shutdown()
|
||||
|
||||
|
||||
def test_cache_dir_follows_llama_cpp(monkeypatch, tmp_path):
|
||||
for key in ("LLAMA_CACHE", "HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE", "HF_HOME", "XDG_CACHE_HOME"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
assert setup.cache_dir() == Path.home() / ".cache" / "huggingface" / "hub"
|
||||
monkeypatch.setenv("HF_HOME", str(tmp_path / "hf"))
|
||||
assert setup.cache_dir() == tmp_path / "hf" / "hub"
|
||||
monkeypatch.setenv("LLAMA_CACHE", str(tmp_path / "llama"))
|
||||
assert setup.cache_dir() == tmp_path / "llama"
|
||||
|
||||
|
||||
def _ollama_model(root: Path, name: str, tag: str, size: int, ns: str = "library") -> Path:
|
||||
digest = f"sha256:{name}{tag}".replace(".", "")
|
||||
manifest = root / "manifests" / "registry.ollama.ai" / ns / name / tag
|
||||
manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||
layers = [
|
||||
{"mediaType": "application/vnd.ollama.image.license", "digest": "sha256:lic", "size": 10},
|
||||
{"mediaType": "application/vnd.ollama.image.model", "digest": digest, "size": size},
|
||||
]
|
||||
manifest.write_text(json.dumps({"layers": layers}))
|
||||
blob = root / "blobs" / digest.replace(":", "-")
|
||||
blob.parent.mkdir(parents=True, exist_ok=True)
|
||||
blob.write_bytes(b"GGUF")
|
||||
return blob
|
||||
|
||||
|
||||
def test_ollama_models_are_read_from_its_manifests(tmp_path):
|
||||
root = tmp_path / "ollama"
|
||||
blob = _ollama_model(root, "qwen3-coder", "30b", 18 * GB)
|
||||
_ollama_model(root, "mymodel", "latest", GB, ns="someone")
|
||||
# A manifest whose model file is missing is not offered.
|
||||
gone = root / "manifests" / "registry.ollama.ai" / "library" / "gone" / "1b"
|
||||
gone.parent.mkdir(parents=True)
|
||||
gone.write_text(json.dumps({"layers": [{"mediaType": "application/vnd.ollama.image.model", "digest": "sha256:nope", "size": 1}]}))
|
||||
models = {m["name"]: m for m in setup.ollama_models()}
|
||||
assert set(models) == {"qwen3-coder:30b", "someone/mymodel:latest"}
|
||||
assert models["qwen3-coder:30b"]["path"] == str(blob)
|
||||
assert models["qwen3-coder:30b"]["size"] == 18 * GB
|
||||
|
||||
|
||||
def test_overview_offers_what_fits_and_the_pull_commands(tmp_path, monkeypatch):
|
||||
_ollama_model(tmp_path / "ollama", "qwen3-coder", "30b", 18 * GB)
|
||||
_ollama_model(tmp_path / "ollama", "huge", "1t", 400 * GB)
|
||||
monkeypatch.setattr(setup, "find_llama_server", lambda: None)
|
||||
o = setup.overview()
|
||||
ids = [m["id"] for m in o["suggested"]]
|
||||
# 32 GB: the 36 GB Nemotron is too large; the others are offered.
|
||||
assert "nemotron-3.5-lightning" not in ids
|
||||
assert "gpt-oss-20b" in ids and "qwen3-coder-30b" in ids
|
||||
assert [m["name"] for m in o["ollama"]] == ["qwen3-coder:30b"]
|
||||
# Ollama already has qwen3-coder, so only gpt-oss gets a download command.
|
||||
assert [p["command"] for p in o["ollama_pull"]] == ["ollama pull gpt-oss:20b"]
|
||||
assert o["installed"] is None and o["install_command"] == "brew install llama.cpp"
|
||||
assert o["cache_dir"].endswith("cache") and o["phase"] == "idle" and o["autostart"] is False
|
||||
|
||||
|
||||
def test_a_file_with_an_mtp_draft_beside_it_runs_with_the_draft(tmp_path):
|
||||
model = tmp_path / "Nemo-Q8_0.gguf"
|
||||
model.write_bytes(b"GGUF")
|
||||
plan = setup.runner()._plan({"kind": "file", "path": str(model)})
|
||||
assert "draft" not in plan
|
||||
(tmp_path / "mtp-Nemo-Q8_0.gguf").write_bytes(b"GGUF")
|
||||
plan = setup.runner()._plan({"kind": "file", "path": str(model)})
|
||||
assert plan["draft"].endswith("mtp-Nemo-Q8_0.gguf") and plan["spec_type"] == "draft-mtp"
|
||||
with pytest.raises(ValueError):
|
||||
setup.runner()._plan({"kind": "file", "path": str(tmp_path / "x.bin")})
|
||||
|
||||
|
||||
def test_download_lands_in_the_hugging_face_layout_and_resumes(tmp_path, monkeypatch):
|
||||
entry = {"id": "t", "name": "T", "repo": "org/t-GGUF", "file": "t.gguf", "draft": "mtp-t.gguf"}
|
||||
blobs = {"t.gguf": b"M" * 5000, "mtp-t.gguf": b"D" * 300}
|
||||
monkeypatch.setattr(
|
||||
setup, "_hf_files", lambda repo, *a: ("c0ffee", {f: {"size": len(b), "oid": f"oid-{f}"} for f, b in blobs.items()})
|
||||
)
|
||||
# Half of the main file is already there from an interrupted run.
|
||||
part = setup._repo_dir("org/t-GGUF") / "blobs" / "oid-t.gguf.incomplete"
|
||||
part.parent.mkdir(parents=True)
|
||||
part.write_bytes(blobs["t.gguf"][:2000])
|
||||
ranges = []
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
name = req.url.path.rsplit("/", 1)[-1]
|
||||
assert "/resolve/c0ffee/" in req.url.path
|
||||
body = blobs[name]
|
||||
rng = req.headers.get("range")
|
||||
ranges.append((name, rng))
|
||||
if rng:
|
||||
start = int(rng.split("=")[1].rstrip("-"))
|
||||
return httpx.Response(206, content=body[start:])
|
||||
return httpx.Response(200, content=body)
|
||||
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
monkeypatch.setattr(httpx, "stream", lambda method, url, headers=None, **kw: client.stream(method, url, headers=headers))
|
||||
got = setup.runner()._download(entry)
|
||||
assert ranges == [("t.gguf", "bytes=2000-"), ("mtp-t.gguf", None)]
|
||||
snap = setup._repo_dir("org/t-GGUF") / "snapshots" / "c0ffee"
|
||||
assert Path(got["file"]) == snap / "t.gguf" and (snap / "t.gguf").is_symlink()
|
||||
assert (snap / "t.gguf").read_bytes() == blobs["t.gguf"]
|
||||
assert Path(got["draft"]).read_bytes() == blobs["mtp-t.gguf"]
|
||||
assert (setup._repo_dir("org/t-GGUF") / "refs" / "main").read_text() == "c0ffee"
|
||||
assert setup._downloaded(entry)
|
||||
assert setup.runner().progress["done"] == setup.runner().progress["total"] == 5300
|
||||
|
||||
|
||||
def test_the_catalog_names_each_model_licence_and_the_card_shows_it(monkeypatch):
|
||||
for m in setup.catalog():
|
||||
assert m["license"] and m["license_url"].startswith("https://"), m["id"]
|
||||
assert m["license_source"].startswith("https://huggingface.co/"), m["id"]
|
||||
monkeypatch.setattr(local_machine, "model_memory_bytes", lambda: 96 * GB)
|
||||
monkeypatch.setattr(setup, "find_llama_server", lambda: None)
|
||||
rows = {m["id"]: m for m in setup.overview()["suggested"]}
|
||||
assert rows["nemotron-3.5-lightning"]["license"] == "OpenMDW-1.1"
|
||||
assert rows["qwen3-coder-30b"]["license"] == "Apache-2.0" and rows["gpt-oss-20b"]["license"] == "Apache-2.0"
|
||||
assert all(r["license_url"].startswith("https://") for r in rows.values())
|
||||
|
||||
|
||||
def test_retry_after_is_read_from_the_headers_and_bounded():
|
||||
r = lambda **h: httpx.Response(429, headers=h) # noqa: E731
|
||||
assert setup._retry_after(r(**{"Retry-After": "7"})) == 7
|
||||
assert setup._retry_after(r(**{"RateLimit-Reset": "12"})) == 12
|
||||
# Hugging Face's own header: remaining and seconds to the window's end.
|
||||
assert setup._retry_after(r(**{"RateLimit": '"resolvers";r=0;t=52'})) == 52
|
||||
assert setup._retry_after(r(**{"Retry-After": "0"})) == 1
|
||||
assert setup._retry_after(r(**{"Retry-After": "9000"})) == setup.RATE_LIMIT_WAIT_MAX_S
|
||||
assert setup._retry_after(r()) == setup.RATE_LIMIT_WAIT_DEFAULT_S
|
||||
from email.utils import format_datetime
|
||||
import datetime as dt
|
||||
|
||||
soon = format_datetime(dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=30), usegmt=True)
|
||||
assert 27 <= setup._retry_after(r(**{"Retry-After": soon})) <= 30
|
||||
|
||||
|
||||
def _serve(monkeypatch, handler, *, api=True, files=True):
|
||||
"""Route the module's Hugging Face calls (API and file downloads) to `handler`."""
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
if api:
|
||||
monkeypatch.setattr(httpx, "request", lambda method, url, **kw: client.request(method, url, headers=kw.get("headers")))
|
||||
if files:
|
||||
monkeypatch.setattr(httpx, "stream", lambda method, url, headers=None, **kw: client.stream(method, url, headers=headers))
|
||||
|
||||
|
||||
def test_download_verifies_the_checksum_and_removes_a_bad_file(tmp_path, monkeypatch):
|
||||
good = b"G" * 4000
|
||||
sha = hashlib.sha256(good).hexdigest()
|
||||
entry = {"id": "t", "name": "T", "repo": "org/t-GGUF", "file": "t.gguf"}
|
||||
monkeypatch.setattr(setup, "_hf_files", lambda repo, *a: ("c0ffee", {"t.gguf": {"size": 4000, "oid": sha, "sha256": sha}}))
|
||||
body = {"ok": good, "bad": b"B" * 4000}
|
||||
serving = ["ok"]
|
||||
_serve(monkeypatch, lambda req: httpx.Response(200, content=body[serving[0]]), api=False)
|
||||
r = setup.runner()
|
||||
got = r._download(entry)
|
||||
assert Path(got["file"]).read_bytes() == good
|
||||
assert (setup._repo_dir("org/t-GGUF") / "blobs" / sha).is_file()
|
||||
assert r.progress["done"] == r.progress["total"] == 4000 and r.state()["note"] is None
|
||||
|
||||
# The same size, different bytes: the file is refused and removed, nothing is placed.
|
||||
serving[0] = "bad"
|
||||
monkeypatch.setattr(setup, "_hf_files", lambda repo, *a: ("c0ffee", {"t.gguf": {"size": 4000, "oid": "x" * 64, "sha256": "x" * 64}}))
|
||||
with pytest.raises(RuntimeError, match="checksum"):
|
||||
setup.LlamaCppRunner()._download({**entry, "repo": "org/u-GGUF"})
|
||||
blobs = setup._repo_dir("org/u-GGUF") / "blobs"
|
||||
assert not list(blobs.iterdir()) and not (setup._repo_dir("org/u-GGUF") / "snapshots").exists()
|
||||
|
||||
|
||||
def test_without_the_api_the_download_is_unverified_and_says_so(tmp_path, monkeypatch):
|
||||
data = b"Q" * 2500
|
||||
entry = {"id": "t", "name": "T", "repo": "org/t-GGUF", "file": "t.gguf"}
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if req.url.path.startswith("/api/"):
|
||||
raise httpx.ConnectError("api down")
|
||||
if req.method == "HEAD":
|
||||
# What huggingface.co's 302 to the CDN says about the file.
|
||||
return httpx.Response(302, headers={"Location": "https://cdn.example/x", "X-Repo-Commit": "abc123",
|
||||
"X-Linked-Size": str(len(data)), "X-Linked-ETag": '"etag-t"'})
|
||||
assert req.url.path == "/org/t-GGUF/resolve/abc123/t.gguf"
|
||||
return httpx.Response(200, content=data)
|
||||
|
||||
_serve(monkeypatch, handler)
|
||||
r = setup.runner()
|
||||
got = r._download(entry)
|
||||
assert Path(got["file"]) == setup._repo_dir("org/t-GGUF") / "snapshots" / "abc123" / "t.gguf"
|
||||
assert Path(got["file"]).read_bytes() == data
|
||||
assert (setup._repo_dir("org/t-GGUF") / "blobs" / "etag-t").is_file()
|
||||
assert r.state()["note"] == "unverified"
|
||||
|
||||
|
||||
def test_a_429_is_waited_out_as_hugging_face_asks(tmp_path, monkeypatch):
|
||||
data = b"R" * 1200
|
||||
sha = hashlib.sha256(data).hexdigest()
|
||||
entry = {"id": "t", "name": "T", "repo": "org/t-GGUF", "file": "t.gguf"}
|
||||
limited = {"/api/models/org/t-GGUF/revision/main": 1, "/org/t-GGUF/resolve/c0ffee/t.gguf": 1}
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
path = req.url.path
|
||||
if limited.get(path):
|
||||
limited[path] -= 1
|
||||
hdr = {"Retry-After": "1"} if path.startswith("/api/") else {"RateLimit": '"resolvers";r=0;t=1'}
|
||||
return httpx.Response(429, headers=hdr)
|
||||
if path.endswith("/revision/main"):
|
||||
return httpx.Response(200, json={"sha": "c0ffee"})
|
||||
if "/tree/" in path:
|
||||
return httpx.Response(200, json=[{"path": "t.gguf", "size": len(data), "lfs": {"oid": sha, "size": len(data)}}])
|
||||
return httpx.Response(200, content=data)
|
||||
|
||||
_serve(monkeypatch, handler)
|
||||
r = setup.runner()
|
||||
waits = []
|
||||
real = r._rate_wait
|
||||
monkeypatch.setattr(r, "_rate_wait", lambda s: (waits.append(s), real(s)))
|
||||
got = r._download(entry)
|
||||
assert waits == [1, 1]
|
||||
assert Path(got["file"]).read_bytes() == data and r.state()["retry_in"] is None
|
||||
|
||||
# While it waits, the card can count down; Cancel ends the wait at once.
|
||||
t = threading.Thread(target=lambda: pytest.raises(setup._Cancelled, real, 30))
|
||||
t.start()
|
||||
time.sleep(0.3)
|
||||
assert r.state()["retry_in"] == 30
|
||||
r.cancel()
|
||||
t.join(5)
|
||||
assert not t.is_alive() and r.state()["retry_in"] is None
|
||||
|
||||
|
||||
def test_too_many_429s_end_with_a_clear_error(tmp_path, monkeypatch):
|
||||
entry = {"id": "t", "name": "T", "repo": "org/t-GGUF", "file": "t.gguf"}
|
||||
monkeypatch.setattr(setup, "_hf_files", lambda repo, *a: ("c0ffee", {"t.gguf": {"size": 10, "oid": "o", "sha256": None}}))
|
||||
_serve(monkeypatch, lambda req: httpx.Response(429, headers={"Retry-After": "1"}), api=False)
|
||||
monkeypatch.setattr(setup, "RATE_LIMIT_RETRIES", 1)
|
||||
r = setup.runner()
|
||||
monkeypatch.setattr(r, "_rate_wait", lambda s: None)
|
||||
with pytest.raises(RuntimeError, match="rate-limiting"):
|
||||
r._download(entry)
|
||||
|
||||
|
||||
# -- the managed server, against a stand-in llama-server --------------------------------
|
||||
|
||||
FAKE = textwrap.dedent(
|
||||
"""
|
||||
import json, sys
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
args = sys.argv[1:]
|
||||
port = int(args[args.index("--port") + 1])
|
||||
alias = args[args.index("--alias") + 1]
|
||||
if "broken" in alias:
|
||||
print("E llama_model_load: error loading model", flush=True)
|
||||
sys.exit(1)
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
def _send(self, obj):
|
||||
data = json.dumps(obj).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
def do_GET(self):
|
||||
self._send({"status": "ok"} if self.path == "/health" else {"data": [{"id": alias}]})
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
||||
msg = {"role": "assistant", "content": "hello"}
|
||||
if body.get("tools") and "notools" not in alias:
|
||||
msg = {"role": "assistant", "content": None, "tool_calls": [{"id": "1", "type": "function", "function": {"name": "get_time", "arguments": "{}"}}]}
|
||||
self._send({"choices": [{"message": msg}]})
|
||||
HTTPServer(("127.0.0.1", port), H).serve_forever()
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_server(tmp_path, monkeypatch):
|
||||
script = tmp_path / "fake_llama_server.py"
|
||||
script.write_text(FAKE)
|
||||
exe = tmp_path / "llama-server"
|
||||
exe.write_text(f"#!/bin/sh\nexec {sys.executable} {script} \"$@\"\n")
|
||||
exe.chmod(0o755)
|
||||
monkeypatch.setattr(setup, "find_llama_server", lambda: str(exe))
|
||||
monkeypatch.setattr(setup, "DEFAULT_PORT", 18_900)
|
||||
return exe
|
||||
|
||||
|
||||
def _settle(r: setup.LlamaCppRunner, timeout: float = 30) -> dict:
|
||||
end = time.monotonic() + timeout
|
||||
while r.busy() and time.monotonic() < end:
|
||||
time.sleep(0.1)
|
||||
return r.state()
|
||||
|
||||
|
||||
def _alive(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _gguf(tmp_path: Path, name: str) -> str:
|
||||
p = tmp_path / f"{name}.gguf"
|
||||
p.write_bytes(b"GGUF")
|
||||
return str(p)
|
||||
|
||||
|
||||
def test_start_checks_points_the_provider_and_switches(fake_server, tmp_path):
|
||||
r = setup.runner()
|
||||
told = []
|
||||
r.on_running = lambda url, alias: told.append((url, alias))
|
||||
assert r.start({"kind": "file", "path": _gguf(tmp_path, "good")})["ok"]
|
||||
st = _settle(r)
|
||||
assert st["phase"] == "running", st["error"]
|
||||
assert st["check"] == {"answers": True, "tools": True}
|
||||
assert told == [(f"http://127.0.0.1:{st['address'].split(':')[1]}", "good")]
|
||||
first = json.loads(setup._state_path("llamacpp.pid").read_text())
|
||||
assert _alive(first["pid"]) and _alive(first["watcher"])
|
||||
assert setup._load_settings()["choice"]["path"].endswith("good.gguf")
|
||||
|
||||
# Switch: the first server goes, the new one answers but cannot call tools.
|
||||
r.start({"kind": "file", "path": _gguf(tmp_path, "notools")})
|
||||
st = _settle(r)
|
||||
assert st["phase"] == "running" and st["check"] == {"answers": True, "tools": False}
|
||||
assert not _alive(first["pid"])
|
||||
|
||||
r.stop()
|
||||
assert r.state()["phase"] == "idle"
|
||||
assert not setup._state_path("llamacpp.pid").exists()
|
||||
|
||||
|
||||
def test_a_model_that_fails_to_load_shows_the_error(fake_server, tmp_path):
|
||||
r = setup.runner()
|
||||
r.start({"kind": "file", "path": _gguf(tmp_path, "broken")})
|
||||
st = _settle(r)
|
||||
assert st["phase"] == "failed"
|
||||
assert "error loading model" in st["error"]
|
||||
|
||||
|
||||
def test_the_server_does_not_outlive_openworker(fake_server, tmp_path):
|
||||
"""The desktop app ends its server with SIGKILL: the watcher stops llama-server."""
|
||||
import subprocess
|
||||
|
||||
code = textwrap.dedent(
|
||||
f"""
|
||||
import json, os, signal, sys, time
|
||||
sys.path.insert(0, {str(Path(__file__).resolve().parents[1])!r})
|
||||
from coworker.providers import llamacpp_setup as s
|
||||
s.find_llama_server = lambda: {str(fake_server)!r}
|
||||
s.DEFAULT_PORT = 18950
|
||||
r = s.runner()
|
||||
r.start({{"kind": "file", "path": {_gguf(tmp_path, "orphan")!r}}})
|
||||
while r.busy():
|
||||
time.sleep(0.1)
|
||||
print(json.dumps(r.state()["phase"]), flush=True)
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
"""
|
||||
)
|
||||
out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=60, env=os.environ)
|
||||
assert '"running"' in out.stdout, out.stderr
|
||||
pid = json.loads(setup._state_path("llamacpp.pid").read_text())["pid"]
|
||||
for _ in range(50):
|
||||
if not _alive(pid):
|
||||
break
|
||||
time.sleep(0.2)
|
||||
assert not _alive(pid)
|
||||
|
||||
|
||||
def test_app_start_reaps_a_leftover_and_honours_autostart(fake_server, tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
|
||||
leftover = subprocess.Popen(["sleep", "60"])
|
||||
setup._state_path("llamacpp.pid").parent.mkdir(parents=True, exist_ok=True)
|
||||
setup._state_path("llamacpp.pid").write_text(json.dumps({"pid": leftover.pid}))
|
||||
monkeypatch.setattr(setup, "_is_llama_server", lambda pid: pid == leftover.pid)
|
||||
monkeypatch.setattr(setup, "supported", lambda: True)
|
||||
r = setup.runner()
|
||||
r.on_app_start()
|
||||
leftover.wait(timeout=10)
|
||||
assert r.state()["phase"] == "idle" # autostart is off by default
|
||||
|
||||
setup._save_settings({"autostart": True, "choice": {"kind": "file", "path": _gguf(tmp_path, "auto")}})
|
||||
r.on_app_start()
|
||||
assert _settle(r)["phase"] == "running"
|
||||
Reference in New Issue
Block a user