Compare commits
47
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f3c6282dae | ||
|
|
ab1d9f4c15 | ||
|
|
7880be4580 | ||
|
|
741d06445d | ||
|
|
1c3d6e59dd | ||
|
|
ae68edcfe3 | ||
|
|
355a8e4076 | ||
|
|
373ad70f4c | ||
|
|
f1b30414d5 | ||
|
|
44624631bf | ||
|
|
8b70bf4c40 | ||
|
|
cbc1054e23 | ||
|
|
356ff99030 | ||
|
|
5a3ee3c537 | ||
|
|
2b84ed921c | ||
|
|
020bd1ba0a | ||
|
|
e5078e3152 | ||
|
|
0acdf1d8c8 | ||
|
|
2ee50c69d3 | ||
|
|
97cd0d98f1 | ||
|
|
bc4824167d | ||
|
|
fe9dc06071 | ||
|
|
f13f845116 | ||
|
|
8364576e33 | ||
|
|
b10952e9c6 | ||
|
|
6f5608bed3 | ||
|
|
dc4d991373 | ||
|
|
dcdb9b25e4 | ||
|
|
a22d2918d6 | ||
|
|
17155e3ae0 | ||
|
|
5d747a91c4 | ||
|
|
503c0c0e51 | ||
|
|
73d5c896ee | ||
|
|
fee392fee1 | ||
|
|
fc0009b9ba | ||
|
|
ad4034711d | ||
|
|
fd433e046a | ||
|
|
c8089dabcd | ||
|
|
2d71e9e9bc | ||
|
|
8dd07bd517 | ||
|
|
40c3b62b30 | ||
|
|
3c5c389f18 | ||
|
|
45556b71ce | ||
|
|
caf8e2f214 | ||
|
|
7bbdabbef2 | ||
|
|
54a0f07101 | ||
|
|
77beb6a085 |
@@ -0,0 +1,56 @@
|
||||
name: Hermes Plugin Validate
|
||||
description: >-
|
||||
Validate a Hermes Agent plugin (plugin.yaml manifest schema AND
|
||||
declared-vs-actually-registered capabilities) using
|
||||
`hermes plugins validate`. Drop this into your plugin repo's CI:
|
||||
|
||||
- uses: actions/checkout@<sha>
|
||||
- uses: NousResearch/hermes-agent/.github/actions/plugin-validate@main
|
||||
with:
|
||||
path: .
|
||||
|
||||
The caller's job owns checkout; this action installs Python + hermes-agent
|
||||
(git install — a supported CI-context install route) and runs the
|
||||
validator against your plugin directory.
|
||||
|
||||
inputs:
|
||||
path:
|
||||
description: Path to the plugin directory (containing plugin.yaml).
|
||||
default: "."
|
||||
hermes-ref:
|
||||
description: hermes-agent git ref (branch/tag/sha) to install and validate with.
|
||||
default: "main"
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install hermes-agent
|
||||
shell: bash
|
||||
env:
|
||||
_HERMES_REF: ${{ inputs.hermes-ref }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# CI-context install from git; the ref lets plugin authors validate
|
||||
# against a pinned hermes release instead of main.
|
||||
pip install "git+https://github.com/NousResearch/hermes-agent@${_HERMES_REF}"
|
||||
|
||||
- name: Validate plugin
|
||||
shell: bash
|
||||
env:
|
||||
_PLUGIN_PATH: ${{ inputs.path }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# `hermes plugins validate` checks the plugin.yaml manifest schema
|
||||
# and loads the plugin in a scratch subprocess to verify that the
|
||||
# capabilities it DECLARES match what it actually registers.
|
||||
if hermes plugins validate "$_PLUGIN_PATH"; then
|
||||
echo "✅ PASS: plugin at '$_PLUGIN_PATH' validated cleanly"
|
||||
else
|
||||
echo "❌ FAIL: plugin at '$_PLUGIN_PATH' failed validation (see output above)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,140 @@
|
||||
name: Plugin Catalog CI
|
||||
|
||||
# Admission gate for plugin-catalog entries. Fires ONLY on PRs touching
|
||||
# plugin-catalog/** so it can never go red on unrelated PRs.
|
||||
#
|
||||
# Two gates:
|
||||
# structural — cheap schema check, no hermes install needed
|
||||
# pinned-source-validate — supply-chain gate: the pinned sha MUST be
|
||||
# reachable in the entry's repo, and the plugin
|
||||
# at that exact commit must pass
|
||||
# `hermes plugins validate`.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "plugin-catalog/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
structural:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install PyYAML
|
||||
uses: ./.github/actions/retry
|
||||
with:
|
||||
command: pip install pyyaml==6.0.2
|
||||
|
||||
- name: Validate catalog files (structural)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Validating the whole directory is simpler than diffing and keeps
|
||||
# the invariant that EVERYTHING in plugin-catalog/ stays valid.
|
||||
python3 scripts/validate_plugin_catalog.py plugin-catalog/
|
||||
|
||||
pinned-source-validate:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0 # need the merge-base to diff changed catalog files
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Find changed catalog entries
|
||||
id: changed
|
||||
run: |
|
||||
set -euo pipefail
|
||||
MERGE_BASE=$(git merge-base "origin/${{ github.base_ref }}" HEAD)
|
||||
# Added + modified entry files only; deletions and removed.yaml
|
||||
# have nothing to clone.
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$MERGE_BASE"...HEAD \
|
||||
-- 'plugin-catalog/*.yaml' 'plugin-catalog/*.yml' \
|
||||
| grep -v '/removed\.yaml$' || true)
|
||||
echo "Changed catalog entries:"
|
||||
echo "${CHANGED:-<none>}"
|
||||
{
|
||||
echo 'files<<__EOF__'
|
||||
echo "$CHANGED"
|
||||
echo '__EOF__'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Install hermes-agent from the PR's own checkout
|
||||
if: steps.changed.outputs.files != ''
|
||||
uses: ./.github/actions/retry
|
||||
with:
|
||||
command: pip install -e .
|
||||
|
||||
- name: Clone each entry at its pinned sha and validate
|
||||
if: steps.changed.outputs.files != ''
|
||||
env:
|
||||
CHANGED_FILES: ${{ steps.changed.outputs.files }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
FAILED=0
|
||||
while IFS= read -r entry; do
|
||||
[ -z "$entry" ] && continue
|
||||
echo "::group::validate $entry"
|
||||
|
||||
# Parse repo / sha / subdir from the entry yaml.
|
||||
eval "$(python3 - "$entry" <<'PYEOF'
|
||||
import shlex
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as fh:
|
||||
data = yaml.safe_load(fh) or {}
|
||||
print(f"REPO={shlex.quote(str(data.get('repo', '')))}")
|
||||
print(f"SHA={shlex.quote(str(data.get('sha', '')))}")
|
||||
print(f"SUBDIR={shlex.quote(str(data.get('subdir', '') or ''))}")
|
||||
PYEOF
|
||||
)"
|
||||
echo "repo=$REPO sha=$SHA subdir=$SUBDIR"
|
||||
|
||||
CLONE_DIR=$(mktemp -d)
|
||||
# Full clone (no --depth 1): the pinned sha may not be the branch tip.
|
||||
if ! git clone "$REPO" "$CLONE_DIR"; then
|
||||
echo "::error file=$entry::clone failed for $REPO"
|
||||
FAILED=1; echo "::endgroup::"; continue
|
||||
fi
|
||||
|
||||
# SUPPLY-CHAIN GATE: the pinned sha must be reachable in the repo.
|
||||
if ! git -C "$CLONE_DIR" checkout --detach "$SHA"; then
|
||||
echo "::error file=$entry::pinned sha $SHA is not reachable in $REPO"
|
||||
FAILED=1; echo "::endgroup::"; continue
|
||||
fi
|
||||
|
||||
PLUGIN_DIR="$CLONE_DIR${SUBDIR:+/$SUBDIR}"
|
||||
if [ ! -f "$PLUGIN_DIR/plugin.yaml" ]; then
|
||||
echo "::error file=$entry::no plugin.yaml at subdir '$SUBDIR' of $REPO@$SHA"
|
||||
FAILED=1; echo "::endgroup::"; continue
|
||||
fi
|
||||
|
||||
# Manifest schema + declared-vs-registered capability check.
|
||||
if hermes plugins validate "$PLUGIN_DIR"; then
|
||||
echo "✅ PASS: $entry"
|
||||
else
|
||||
echo "::error file=$entry::hermes plugins validate failed"
|
||||
FAILED=1
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done <<< "$CHANGED_FILES"
|
||||
|
||||
if [ "$FAILED" -ne 0 ]; then
|
||||
echo "❌ FAIL: one or more catalog entries failed pinned-source validation"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ PASS: all changed catalog entries validated at their pinned shas"
|
||||
@@ -173,3 +173,4 @@ apps/desktop/demo/
|
||||
# PR body is the archive. See the hermes-agent-dev skill's
|
||||
# pr-infographic-workflow reference (storage rule + lapse #8 / #COMMIT-1).
|
||||
infographic/
|
||||
native/fts5_cjk/*.so
|
||||
|
||||
+18
-1
@@ -1839,6 +1839,18 @@ def init_agent(
|
||||
}
|
||||
else:
|
||||
compression_model_thresholds = {}
|
||||
# Absolute token cap: when set, compression triggers at the lower of
|
||||
# the ratio-based threshold and this absolute count. Clamped to the
|
||||
# model's context length at apply-time so a cap above the window is
|
||||
# a no-op (ratio-based threshold wins).
|
||||
compression_threshold_tokens = _compression_cfg.get("threshold_tokens")
|
||||
if compression_threshold_tokens is not None:
|
||||
try:
|
||||
compression_threshold_tokens = int(compression_threshold_tokens)
|
||||
if compression_threshold_tokens <= 0:
|
||||
compression_threshold_tokens = None
|
||||
except (TypeError, ValueError):
|
||||
compression_threshold_tokens = None
|
||||
# In-place compaction: when True, compress_context() rewrites the message
|
||||
# list + rebuilds the system prompt WITHOUT rotating the session id (no
|
||||
# parent_session_id chain, no `name #N` renumber). See #38763 and
|
||||
@@ -2271,6 +2283,7 @@ def init_agent(
|
||||
abort_on_summary_failure=compression_abort_on_summary_failure,
|
||||
max_tokens=agent.max_tokens,
|
||||
model_thresholds=compression_model_thresholds,
|
||||
threshold_tokens_cap=compression_threshold_tokens,
|
||||
)
|
||||
_bind_session_state = getattr(agent.context_compressor, "bind_session_state", None)
|
||||
if callable(_bind_session_state):
|
||||
@@ -2482,7 +2495,11 @@ def init_agent(
|
||||
_active_threshold_pct = getattr(
|
||||
agent.context_compressor, "threshold_percent", compression_threshold
|
||||
)
|
||||
print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(_active_threshold_pct*100)}% = {agent.context_compressor.threshold_tokens:,})")
|
||||
_cap_note = ""
|
||||
_cap = getattr(agent.context_compressor, "threshold_tokens_cap", None)
|
||||
if _cap and _cap > 0:
|
||||
_cap_note = f" (capped at {_cap:,} tokens)"
|
||||
print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(_active_threshold_pct*100)}% = {agent.context_compressor.threshold_tokens:,}{_cap_note})")
|
||||
else:
|
||||
print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (auto-compression disabled)")
|
||||
# Notice with the exact opt-back-out command. Printed inline at startup
|
||||
|
||||
+505
-43
@@ -22,6 +22,7 @@ import logging
|
||||
import sqlite3
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.auxiliary_client import call_llm, _is_connection_error, aux_interrupt_protection
|
||||
@@ -40,6 +41,14 @@ from tools.todo_tool import TODO_INJECTION_HEADER
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
"""Best-effort integer coercion for telemetry fields."""
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
_SUMMARY_PERMANENT_QUOTA_MARKERS: tuple[str, ...] = (
|
||||
"insufficient_quota",
|
||||
"quota exceeded",
|
||||
@@ -342,6 +351,27 @@ _HISTORICAL_TASK_SECTION_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def _redact_compaction_text(text: Any) -> str:
|
||||
"""Redact text that crosses a compaction summary boundary.
|
||||
|
||||
Compaction summaries persist across sessions and are re-injected into
|
||||
every subsequent summarizer prompt, so this boundary uses strict mode:
|
||||
|
||||
- ``force=True`` — deliberately overrides ``security.redact_secrets:
|
||||
false``. That opt-out targets *live tool output* (e.g. working on the
|
||||
redactor itself); a summary is a persistence boundary where a leaked
|
||||
credential keeps re-entering prompts indefinitely.
|
||||
- ``redact_url_credentials=True`` — OAuth callback codes, magic-link
|
||||
tokens, and URL userinfo never need to survive summarization the way
|
||||
they must survive live navigation flows.
|
||||
"""
|
||||
return redact_sensitive_text(
|
||||
text or "",
|
||||
force=True,
|
||||
redact_url_credentials=True,
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_append(items: list[str], value: str, *, limit: int) -> None:
|
||||
value = value.strip()
|
||||
if value and value not in items and len(items) < limit:
|
||||
@@ -941,6 +971,102 @@ class ContextCompressor(ContextEngine):
|
||||
self.last_compression_rough_tokens = 0
|
||||
self.last_rough_tokens_when_real_prompt_fit = 0
|
||||
self.awaiting_real_usage_after_compression = False
|
||||
self._last_compression_telemetry = None
|
||||
self._active_compression_telemetry = None
|
||||
self._compression_telemetry_seed = None
|
||||
|
||||
def _begin_compression_telemetry(
|
||||
self,
|
||||
*,
|
||||
current_tokens: int | None,
|
||||
attempt_id: str | None = None,
|
||||
session_id: str | None = None,
|
||||
trigger_source: str | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Initialize content-free per-attempt compression telemetry."""
|
||||
seed = getattr(self, "_compression_telemetry_seed", None)
|
||||
if isinstance(seed, dict):
|
||||
attempt_id = attempt_id or seed.get("attempt_id")
|
||||
session_id = session_id or seed.get("session_id")
|
||||
trigger_source = trigger_source or seed.get("trigger_source")
|
||||
telemetry: Dict[str, Any] = {
|
||||
"event": "compression_attempt",
|
||||
"attempt_id": attempt_id or uuid.uuid4().hex,
|
||||
"session_id": session_id or "",
|
||||
"trigger_source": trigger_source or "unknown",
|
||||
"main_provider": self.provider or "",
|
||||
"main_model": self.model or "",
|
||||
"main_context_limit": _safe_int(self.context_length),
|
||||
"current_estimated_tokens": _safe_int(current_tokens),
|
||||
"effective_threshold": _safe_int(self.threshold_tokens),
|
||||
"protected_head_tokens": None,
|
||||
"protected_tail_tokens": None,
|
||||
"middle_window_tokens": None,
|
||||
"aux_prompt_tokens": None,
|
||||
"aux_output_reservation": None,
|
||||
"aux_provider": "",
|
||||
"aux_model": "",
|
||||
"effective_aux_context": None,
|
||||
"fit_margin": None,
|
||||
"chunking": False,
|
||||
"chunk_count": 0,
|
||||
"total_duration_ms": None,
|
||||
"aux_call_duration_ms": None,
|
||||
"fallback_used": False,
|
||||
"commit_status": "unknown",
|
||||
"split_status": "unknown",
|
||||
"failure_class": None,
|
||||
}
|
||||
self._active_compression_telemetry = telemetry
|
||||
self._last_compression_telemetry = telemetry
|
||||
return telemetry
|
||||
|
||||
def _record_compression_regions(
|
||||
self,
|
||||
*,
|
||||
head_messages: List[Dict[str, Any]],
|
||||
middle_messages: List[Dict[str, Any]],
|
||||
tail_messages: List[Dict[str, Any]],
|
||||
) -> None:
|
||||
telemetry = getattr(self, "_active_compression_telemetry", None)
|
||||
if not isinstance(telemetry, dict):
|
||||
return
|
||||
telemetry["protected_head_tokens"] = estimate_messages_tokens_rough(head_messages)
|
||||
telemetry["middle_window_tokens"] = estimate_messages_tokens_rough(middle_messages)
|
||||
telemetry["protected_tail_tokens"] = estimate_messages_tokens_rough(tail_messages)
|
||||
|
||||
def _record_aux_compression_call(
|
||||
self,
|
||||
*,
|
||||
prompt_messages: List[Dict[str, Any]],
|
||||
max_tokens: int | None,
|
||||
duration_ms: int,
|
||||
aux_provider: str | None = None,
|
||||
aux_model: str | None = None,
|
||||
effective_aux_context: int | None = None,
|
||||
) -> None:
|
||||
telemetry = getattr(self, "_active_compression_telemetry", None)
|
||||
if not isinstance(telemetry, dict):
|
||||
return
|
||||
telemetry["aux_prompt_tokens"] = estimate_messages_tokens_rough(prompt_messages)
|
||||
telemetry["aux_output_reservation"] = _safe_int(max_tokens)
|
||||
if aux_provider:
|
||||
telemetry["aux_provider"] = aux_provider
|
||||
if aux_model:
|
||||
telemetry["aux_model"] = aux_model
|
||||
if effective_aux_context is not None:
|
||||
telemetry["effective_aux_context"] = _safe_int(effective_aux_context)
|
||||
if (
|
||||
telemetry["effective_aux_context"] is not None
|
||||
and telemetry["aux_prompt_tokens"] is not None
|
||||
):
|
||||
telemetry["fit_margin"] = (
|
||||
telemetry["effective_aux_context"]
|
||||
- telemetry["aux_prompt_tokens"]
|
||||
- (telemetry["aux_output_reservation"] or 0)
|
||||
)
|
||||
previous = telemetry.get("aux_call_duration_ms") or 0
|
||||
telemetry["aux_call_duration_ms"] = previous + max(0, int(duration_ms))
|
||||
|
||||
def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> None:
|
||||
"""Clear all per-session compaction state at a real session boundary.
|
||||
@@ -983,6 +1109,9 @@ class ContextCompressor(ContextEngine):
|
||||
self.last_compression_rough_tokens = 0
|
||||
self.last_rough_tokens_when_real_prompt_fit = 0
|
||||
self.awaiting_real_usage_after_compression = False
|
||||
self._last_compression_telemetry = None
|
||||
self._active_compression_telemetry = None
|
||||
self._compression_telemetry_seed = None
|
||||
|
||||
def bind_session_state(self, session_db: Any = None, session_id: str = "") -> None:
|
||||
"""Bind the current session row so durable cooldowns can round-trip."""
|
||||
@@ -1231,6 +1360,11 @@ class ContextCompressor(ContextEngine):
|
||||
self.threshold_tokens = self._compute_threshold_tokens(
|
||||
context_length, self.threshold_percent, self.max_tokens,
|
||||
)
|
||||
# Re-apply the absolute token cap so it survives model switches
|
||||
# and fallback activations. The cap is a first-class config value
|
||||
# stored on the compressor instance, not a one-time post-construction
|
||||
# patch — this is why update_model() must re-apply it.
|
||||
self._apply_threshold_tokens_cap()
|
||||
# Recalculate token budgets for the new context length so the
|
||||
# compressor stays calibrated after a model switch (e.g. 200K → 32K).
|
||||
target_tokens = int(self.threshold_tokens * self.summary_target_ratio)
|
||||
@@ -1293,6 +1427,36 @@ class ContextCompressor(ContextEngine):
|
||||
return None
|
||||
return ivalue if ivalue > 0 else None
|
||||
|
||||
@staticmethod
|
||||
def _coerce_threshold_tokens_cap(value: Any) -> int | None:
|
||||
"""Normalize a threshold_tokens cap to a positive int or None.
|
||||
|
||||
None means "no absolute cap — use the ratio-based threshold only".
|
||||
Non-numeric or non-positive values are treated as None so a bad
|
||||
config value never silently caps the threshold at zero.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
ivalue = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return ivalue if ivalue > 0 else None
|
||||
|
||||
def _apply_threshold_tokens_cap(self) -> None:
|
||||
"""Apply the absolute token cap if configured.
|
||||
|
||||
After ``threshold_tokens`` is (re)computed from the ratio-based
|
||||
percent, clamp it to the cap so compression never fires later
|
||||
than the user's preferred absolute token count. The cap itself
|
||||
is clamped to the current context length so a cap larger than
|
||||
the model's window is a no-op (the ratio-based threshold wins).
|
||||
"""
|
||||
if self.threshold_tokens_cap is not None and self.threshold_tokens_cap > 0:
|
||||
_effective_cap = min(self.threshold_tokens_cap, self.context_length)
|
||||
if _effective_cap < self.threshold_tokens:
|
||||
self.threshold_tokens = _effective_cap
|
||||
|
||||
@staticmethod
|
||||
def _effective_threshold_percent(
|
||||
context_length: int, threshold_percent: float,
|
||||
@@ -1368,6 +1532,7 @@ class ContextCompressor(ContextEngine):
|
||||
abort_on_summary_failure: bool = False,
|
||||
max_tokens: int | None = None,
|
||||
model_thresholds: dict[str, float] | None = None,
|
||||
threshold_tokens_cap: Any = None,
|
||||
):
|
||||
self.model = model
|
||||
self.base_url = base_url
|
||||
@@ -1387,6 +1552,14 @@ class ContextCompressor(ContextEngine):
|
||||
model, self.model_thresholds, threshold_percent,
|
||||
)
|
||||
self.threshold_percent = self._base_threshold_percent
|
||||
# Absolute token cap from config (compression.threshold_tokens). When
|
||||
# set, the effective trigger point is min(ratio-based threshold, cap)
|
||||
# so compression never fires later than the user's preferred token
|
||||
# count regardless of which model is active. Applied in __init__ and
|
||||
# re-applied in update_model() so it survives model switches/fallbacks.
|
||||
self.threshold_tokens_cap = self._coerce_threshold_tokens_cap(
|
||||
threshold_tokens_cap,
|
||||
)
|
||||
self.protect_first_n = protect_first_n
|
||||
self.protect_last_n = protect_last_n
|
||||
self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80))
|
||||
@@ -1432,6 +1605,9 @@ class ContextCompressor(ContextEngine):
|
||||
self.threshold_tokens = self._compute_threshold_tokens(
|
||||
self.context_length, threshold_percent, self.max_tokens,
|
||||
)
|
||||
# Apply absolute token cap (compression.threshold_tokens) — takes
|
||||
# the lower of the ratio-based threshold and the cap.
|
||||
self._apply_threshold_tokens_cap()
|
||||
self.compression_count = 0
|
||||
|
||||
# Derive token budgets: ratio is relative to the threshold, not total context
|
||||
@@ -1522,6 +1698,9 @@ class ContextCompressor(ContextEngine):
|
||||
# succeeded. Silent recovery would hide the broken config.
|
||||
self._last_aux_model_failure_error: Optional[str] = None
|
||||
self._last_aux_model_failure_model: Optional[str] = None
|
||||
self._last_compression_telemetry: Optional[Dict[str, Any]] = None
|
||||
self._active_compression_telemetry: Optional[Dict[str, Any]] = None
|
||||
self._compression_telemetry_seed: Optional[Dict[str, Any]] = None
|
||||
|
||||
def update_from_response(self, usage: Dict[str, Any]):
|
||||
"""Update tracked token usage from API response."""
|
||||
@@ -1934,7 +2113,7 @@ class ContextCompressor(ContextEngine):
|
||||
elif isinstance(part, str):
|
||||
text_parts.append(part)
|
||||
content = "\n".join(text_parts)
|
||||
content = redact_sensitive_text(content or "")
|
||||
content = _redact_compaction_text(content or "")
|
||||
content = _MEDIA_DIRECTIVE_RE.sub("[media attachment]", content)
|
||||
# Strip inline reasoning blocks (<think>, <reasoning>, etc.) from
|
||||
# assistant content before it reaches the summarizer. Reasoning
|
||||
@@ -1967,7 +2146,7 @@ class ContextCompressor(ContextEngine):
|
||||
if isinstance(tc, dict):
|
||||
fn = tc.get("function", {})
|
||||
name = fn.get("name", "?")
|
||||
args = redact_sensitive_text(fn.get("arguments", ""))
|
||||
args = _redact_compaction_text(fn.get("arguments", ""))
|
||||
# Truncate long arguments but keep enough for context
|
||||
if len(args) > self._TOOL_ARGS_MAX:
|
||||
args = args[:self._TOOL_ARGS_HEAD] + "..."
|
||||
@@ -2010,7 +2189,7 @@ class ContextCompressor(ContextEngine):
|
||||
last_dropped_turns: list[str] = []
|
||||
|
||||
def _compact_fallback_turn(value: Any) -> str:
|
||||
text = redact_sensitive_text(_content_text_for_contains(value))
|
||||
text = _redact_compaction_text(_content_text_for_contains(value))
|
||||
text = re.sub(r"\bgh[pousr]_[A-Za-z0-9_]{8,}\b", "[REDACTED]", text)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
if len(text) > _FALLBACK_TURN_MAX_CHARS:
|
||||
@@ -2042,7 +2221,7 @@ class ContextCompressor(ContextEngine):
|
||||
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
for tc in msg.get("tool_calls") or []:
|
||||
name, raw_args = _extract_tool_call_name_and_args(tc)
|
||||
args = redact_sensitive_text(raw_args)
|
||||
args = _redact_compaction_text(raw_args)
|
||||
call_id = _extract_tool_call_id(tc)
|
||||
if call_id:
|
||||
call_id_to_tool[call_id] = (name, args)
|
||||
@@ -2180,7 +2359,7 @@ Continue from the most recent unfulfilled user ask and protected tail messages.
|
||||
|
||||
## Critical Context
|
||||
Summary generation was unavailable, so this is a best-effort deterministic fallback for {len(turns_to_summarize)} compacted message(s).{reason_text}"""
|
||||
summary = self._with_summary_prefix(redact_sensitive_text(body.strip()))
|
||||
summary = self._with_summary_prefix(_redact_compaction_text(body.strip()))
|
||||
if len(summary) > _FALLBACK_SUMMARY_MAX_CHARS:
|
||||
summary = summary[: _FALLBACK_SUMMARY_MAX_CHARS - 42].rstrip() + "\n...[fallback summary truncated]"
|
||||
return summary
|
||||
@@ -2209,6 +2388,10 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb
|
||||
_err_text = _err_text[:217].rstrip() + "..."
|
||||
self._last_aux_model_failure_error = _err_text
|
||||
self._last_aux_model_failure_model = self.summary_model
|
||||
telemetry = getattr(self, "_active_compression_telemetry", None)
|
||||
if isinstance(telemetry, dict):
|
||||
telemetry["fallback_used"] = True
|
||||
telemetry["failure_class"] = telemetry.get("failure_class") or "aux_model_fallback"
|
||||
self.summary_model = "" # empty = use main model
|
||||
self._clear_compression_failure_cooldown() # no cooldown — retry immediately
|
||||
|
||||
@@ -2243,6 +2426,15 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb
|
||||
)
|
||||
return None
|
||||
|
||||
# Strict-redact prompt inputs that bypass _serialize_for_summary:
|
||||
# a manual `/compress <focus>` string, and a previous summary that
|
||||
# may predate compaction redaction (resumed from a persisted
|
||||
# handoff message written before this boundary existed).
|
||||
if focus_topic:
|
||||
focus_topic = _redact_compaction_text(focus_topic)
|
||||
if self._previous_summary:
|
||||
self._previous_summary = _redact_compaction_text(self._previous_summary)
|
||||
|
||||
summary_budget = self._compute_summary_budget(turns_to_summarize)
|
||||
content_to_summarize = self._serialize_for_summary(turns_to_summarize)
|
||||
_sanitized_memory_context = sanitize_memory_context(memory_context)
|
||||
@@ -2500,13 +2692,43 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
}
|
||||
if self.summary_model:
|
||||
call_kwargs["model"] = self.summary_model
|
||||
_aux_provider = ""
|
||||
_aux_model = self.summary_model or ""
|
||||
_aux_context = None
|
||||
try:
|
||||
from agent.auxiliary_client import _resolve_task_provider_model
|
||||
|
||||
_resolved_provider, _resolved_model, _, _, _ = _resolve_task_provider_model(
|
||||
"compression",
|
||||
model=(self.summary_model or ""),
|
||||
)
|
||||
_aux_provider = _resolved_provider or ""
|
||||
_aux_model = _resolved_model or _aux_model or self.model or ""
|
||||
if _aux_model == self.model:
|
||||
_aux_context = self.context_length
|
||||
except Exception:
|
||||
pass
|
||||
# Compression is atomic: protect the in-flight summary call from a
|
||||
# mid-turn gateway interrupt. Without this, an incoming user message
|
||||
# aborts the summary and compression falls back to a degraded static
|
||||
# marker, losing the real handoff (#23975). Re-entrant: a main-model
|
||||
# retry (_generate_summary recursion) re-enters harmlessly.
|
||||
with aux_interrupt_protection():
|
||||
response = call_llm(**call_kwargs)
|
||||
_aux_call_start = time.monotonic()
|
||||
try:
|
||||
with aux_interrupt_protection():
|
||||
response = call_llm(**call_kwargs)
|
||||
finally:
|
||||
self._record_aux_compression_call(
|
||||
prompt_messages=call_kwargs["messages"],
|
||||
# Current main intentionally omits max_tokens from the aux
|
||||
# call (summary_budget is prompt-level guidance only) —
|
||||
# use .get() so the telemetry hook never breaks the call.
|
||||
max_tokens=call_kwargs.get("max_tokens"),
|
||||
duration_ms=int((time.monotonic() - _aux_call_start) * 1000),
|
||||
aux_provider=_aux_provider,
|
||||
aux_model=_aux_model,
|
||||
effective_aux_context=_aux_context,
|
||||
)
|
||||
# ``_validate_llm_response`` only guarantees ``choices[0].message``
|
||||
# exists, not that it's an object with ``.content``. Some
|
||||
# OpenAI-compatible proxies / local backends return a dict- or
|
||||
@@ -2547,7 +2769,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
content = stripped
|
||||
# Redact the summary output as well — the summarizer LLM may
|
||||
# ignore prompt instructions and echo back secrets verbatim.
|
||||
summary = redact_sensitive_text(content.strip())
|
||||
summary = _redact_compaction_text(content.strip())
|
||||
summary = self._ground_historical_task_snapshot(summary, turns_to_summarize)
|
||||
self._validate_summary_user_provenance(summary, has_user_turn)
|
||||
# Store for iterative updates on next compaction
|
||||
@@ -2859,6 +3081,68 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
"session with no user-authored turns"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _is_blank_user_turn(cls, message: Any) -> bool:
|
||||
"""Return whether *message* is an empty, non-summary user-role echo."""
|
||||
if not isinstance(message, dict) or message.get("role") != "user":
|
||||
return False
|
||||
if cls._has_compressed_summary_metadata(message):
|
||||
return False
|
||||
content = message.get("content")
|
||||
if cls._is_context_summary_content(content):
|
||||
return False
|
||||
if content is None or (isinstance(content, str) and not content.strip()):
|
||||
return True
|
||||
if not isinstance(content, list):
|
||||
return False
|
||||
if not content:
|
||||
return True
|
||||
for part in content:
|
||||
if isinstance(part, str):
|
||||
if part.strip():
|
||||
return False
|
||||
continue
|
||||
if isinstance(part, dict) and part.get("type") in {"text", "input_text"}:
|
||||
text = part.get("text")
|
||||
if isinstance(text, str) and not text.strip():
|
||||
continue
|
||||
# Images, audio, and unknown structured blocks are user input.
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _is_actionable_user_turn(cls, message: Any) -> bool:
|
||||
"""Return whether *message* contains user input worth anchoring."""
|
||||
if not isinstance(message, dict) or message.get("role") != "user":
|
||||
return False
|
||||
if cls._has_compressed_summary_metadata(message):
|
||||
return False
|
||||
content = message.get("content")
|
||||
if cls._is_context_summary_content(content):
|
||||
return False
|
||||
return not cls._is_blank_user_turn(message)
|
||||
|
||||
@classmethod
|
||||
def _blank_echo_indices_after(
|
||||
cls, messages: List[Dict[str, Any]], user_idx: int
|
||||
) -> set[int]:
|
||||
"""Return contiguous blank echoes safe to remove after a user event.
|
||||
|
||||
A blank user row is only a removable platform echo when an assistant turn
|
||||
immediately follows it. Otherwise it may be an intentional alternation
|
||||
placeholder for a transcript still being assembled.
|
||||
"""
|
||||
indices: set[int] = set()
|
||||
if user_idx < 0:
|
||||
return indices
|
||||
idx = user_idx + 1
|
||||
while idx < len(messages) and cls._is_blank_user_turn(messages[idx]):
|
||||
indices.add(idx)
|
||||
idx += 1
|
||||
if not indices or idx >= len(messages):
|
||||
return set()
|
||||
return indices if messages[idx].get("role") == "assistant" else set()
|
||||
|
||||
@classmethod
|
||||
def _derive_auto_focus_topic(
|
||||
cls,
|
||||
@@ -2873,7 +3157,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
if cls._is_synthetic_compression_user_turn(msg):
|
||||
continue
|
||||
content = msg.get("content")
|
||||
text = redact_sensitive_text(_content_text_for_contains(content).strip())
|
||||
text = _redact_compaction_text(_content_text_for_contains(content).strip())
|
||||
if not text:
|
||||
continue
|
||||
text = " ".join(text.split())
|
||||
@@ -2917,7 +3201,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
if not _is_real_user_message(msg):
|
||||
continue
|
||||
content = msg.get("content")
|
||||
text = redact_sensitive_text(_content_text_for_contains(content).strip())
|
||||
text = _redact_compaction_text(_content_text_for_contains(content).strip())
|
||||
if not text:
|
||||
continue
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
@@ -2969,6 +3253,125 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
return idx, cls._strip_summary_prefix(_content_text_for_contains(content))
|
||||
return None, ""
|
||||
|
||||
@classmethod
|
||||
def _strip_context_summary_handoff_message(
|
||||
cls,
|
||||
message: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Drop stale handoff data while preserving merged prior-tail content."""
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
|
||||
content = message.get("content")
|
||||
is_summary = (
|
||||
cls._is_context_summary_content(content)
|
||||
or cls._has_compressed_summary_metadata(message)
|
||||
)
|
||||
if not is_summary:
|
||||
return message.copy()
|
||||
|
||||
if isinstance(content, str):
|
||||
if _MERGED_SUMMARY_DELIMITER in content:
|
||||
prior = content.split(_MERGED_SUMMARY_DELIMITER, 1)[0].strip()
|
||||
if prior.startswith(_MERGED_PRIOR_CONTEXT_HEADER):
|
||||
prior = prior[len(_MERGED_PRIOR_CONTEXT_HEADER):].lstrip()
|
||||
if prior:
|
||||
unwrapped = message.copy()
|
||||
unwrapped["content"] = prior
|
||||
unwrapped.pop(COMPRESSED_SUMMARY_METADATA_KEY, None)
|
||||
return unwrapped
|
||||
else:
|
||||
marker_idx = content.find(_SUMMARY_END_MARKER)
|
||||
if marker_idx >= 0:
|
||||
remainder = content[marker_idx + len(_SUMMARY_END_MARKER):].lstrip()
|
||||
if remainder:
|
||||
unwrapped = message.copy()
|
||||
unwrapped["content"] = remainder
|
||||
unwrapped.pop(COMPRESSED_SUMMARY_METADATA_KEY, None)
|
||||
return unwrapped
|
||||
|
||||
if isinstance(content, list):
|
||||
prior_blocks: list[Any] = []
|
||||
found_delimiter = False
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
if _MERGED_SUMMARY_DELIMITER in item:
|
||||
before = item.split(_MERGED_SUMMARY_DELIMITER, 1)[0]
|
||||
if before.strip():
|
||||
prior_blocks.append(before)
|
||||
found_delimiter = True
|
||||
break
|
||||
prior_blocks.append(item)
|
||||
continue
|
||||
if isinstance(item, dict):
|
||||
text = item.get("text")
|
||||
if isinstance(text, str) and _MERGED_SUMMARY_DELIMITER in text:
|
||||
before = text.split(_MERGED_SUMMARY_DELIMITER, 1)[0]
|
||||
if before.strip():
|
||||
copied = item.copy()
|
||||
copied["text"] = before
|
||||
prior_blocks.append(copied)
|
||||
found_delimiter = True
|
||||
break
|
||||
prior_blocks.append(item.copy())
|
||||
continue
|
||||
prior_blocks.append(item)
|
||||
|
||||
if not found_delimiter:
|
||||
legacy_blocks: list[Any] = []
|
||||
found_marker = False
|
||||
for index, item in enumerate(content):
|
||||
text = item if isinstance(item, str) else item.get("text") if isinstance(item, dict) else None
|
||||
if not isinstance(text, str) or _SUMMARY_END_MARKER not in text:
|
||||
continue
|
||||
remainder = text.split(_SUMMARY_END_MARKER, 1)[1].lstrip()
|
||||
if remainder:
|
||||
if isinstance(item, dict):
|
||||
copied = item.copy()
|
||||
copied["text"] = remainder
|
||||
legacy_blocks.append(copied)
|
||||
else:
|
||||
legacy_blocks.append(remainder)
|
||||
for later in content[index + 1:]:
|
||||
legacy_blocks.append(later.copy() if isinstance(later, dict) else later)
|
||||
found_marker = True
|
||||
break
|
||||
if found_marker and legacy_blocks:
|
||||
unwrapped = message.copy()
|
||||
unwrapped["content"] = legacy_blocks
|
||||
unwrapped.pop(COMPRESSED_SUMMARY_METADATA_KEY, None)
|
||||
return unwrapped
|
||||
|
||||
if found_delimiter:
|
||||
for index, item in enumerate(prior_blocks):
|
||||
if isinstance(item, str):
|
||||
if item.lstrip().startswith(_MERGED_PRIOR_CONTEXT_HEADER):
|
||||
leading = item.lstrip()[len(_MERGED_PRIOR_CONTEXT_HEADER):].lstrip()
|
||||
if leading:
|
||||
prior_blocks[index] = leading
|
||||
else:
|
||||
prior_blocks.pop(index)
|
||||
break
|
||||
elif isinstance(item, dict) and isinstance(item.get("text"), str):
|
||||
text = item["text"]
|
||||
if text.lstrip().startswith(_MERGED_PRIOR_CONTEXT_HEADER):
|
||||
leading = text.lstrip()[len(_MERGED_PRIOR_CONTEXT_HEADER):].lstrip()
|
||||
if leading:
|
||||
copied = item.copy()
|
||||
copied["text"] = leading
|
||||
prior_blocks[index] = copied
|
||||
else:
|
||||
prior_blocks.pop(index)
|
||||
break
|
||||
|
||||
if prior_blocks:
|
||||
unwrapped = message.copy()
|
||||
unwrapped["content"] = prior_blocks
|
||||
unwrapped.pop(COMPRESSED_SUMMARY_METADATA_KEY, None)
|
||||
return unwrapped
|
||||
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tool-call / tool-result pair integrity helpers
|
||||
# ------------------------------------------------------------------
|
||||
@@ -3141,20 +3544,16 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
def _find_last_user_message_idx(
|
||||
self, messages: List[Dict[str, Any]], head_end: int
|
||||
) -> int:
|
||||
"""Return the index of the last user-role message at or after *head_end*, or -1.
|
||||
"""Return the latest actionable user turn at or after *head_end*, or -1.
|
||||
|
||||
A context-compaction handoff banner can be inserted as a ``role="user"``
|
||||
message (see the summary-role selection in ``compress``). It is internal
|
||||
continuity state, not a real user turn, so it must not be picked as the
|
||||
tail anchor — otherwise ``_ensure_last_user_message_in_tail`` protects
|
||||
the summary and rolls the genuine last user message into the next
|
||||
compaction, re-triggering the active-task loss the anchor exists to
|
||||
prevent.
|
||||
Compaction handoffs and empty platform echoes are continuity artifacts;
|
||||
neither may displace the request, correction, or completion that the tail
|
||||
anchor exists to preserve.
|
||||
"""
|
||||
for i in range(len(messages) - 1, head_end - 1, -1):
|
||||
msg = messages[i]
|
||||
if (
|
||||
msg.get("role") == "user"
|
||||
self._is_actionable_user_turn(msg)
|
||||
and not self._is_synthetic_compression_user_turn(msg)
|
||||
):
|
||||
return i
|
||||
@@ -3520,6 +3919,13 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
4. Summarize middle turns with structured LLM prompt
|
||||
5. On re-compression, iteratively update the previous summary
|
||||
|
||||
Blank platform-echo user rows trailing the latest actionable user
|
||||
turn are removed in the same cheap pre-pass phase as tool-result
|
||||
pruning — i.e. BEFORE any summary-abort early return. An aborted
|
||||
compression can therefore still hand back a modified list (echoes
|
||||
stripped, no turns summarized); this mirrors the long-standing
|
||||
Phase-1 pruning behavior, which likewise survives an abort.
|
||||
|
||||
After compression, orphaned tool_call / tool_result pairs are cleaned
|
||||
up so the API never receives mismatched IDs.
|
||||
|
||||
@@ -3553,6 +3959,8 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# static-fallback — the exact data-loss #29559 describes. Letting them
|
||||
# persist across compress() calls is safe because a successful summary
|
||||
# always clears both.
|
||||
telemetry = self._begin_compression_telemetry(current_tokens=current_tokens)
|
||||
telemetry["chunk_count"] = 0
|
||||
|
||||
# Manual /compress (force=True) bypasses the failure cooldown so the
|
||||
# user can retry immediately after an auto-compress abort. Without
|
||||
@@ -3572,6 +3980,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# returns here unchanged, and the CLI appears frozen.
|
||||
self._ineffective_compression_count += 1
|
||||
self._last_compression_savings_pct = 0.0
|
||||
telemetry["failure_class"] = "insufficient_messages"
|
||||
if not self.quiet_mode:
|
||||
logger.warning(
|
||||
"Cannot compress: only %d messages (need > %d). "
|
||||
@@ -3591,6 +4000,19 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
if pruned_count and not self.quiet_mode:
|
||||
logger.info("Pre-compression: pruned %d old tool result(s)", pruned_count)
|
||||
|
||||
latest_actionable_idx = self._find_last_user_message_idx(messages, 0)
|
||||
blank_echo_indices = self._blank_echo_indices_after(
|
||||
messages, latest_actionable_idx
|
||||
)
|
||||
if blank_echo_indices:
|
||||
messages = [
|
||||
message
|
||||
for idx, message in enumerate(messages)
|
||||
if idx not in blank_echo_indices
|
||||
]
|
||||
n_messages = len(messages)
|
||||
latest_actionable_idx = self._find_last_user_message_idx(messages, 0)
|
||||
|
||||
# Phase 2: Determine boundaries
|
||||
compress_start = self._protect_head_size(messages)
|
||||
compress_start = self._align_boundary_forward(messages, compress_start)
|
||||
@@ -3598,7 +4020,27 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# Use token-budget tail protection instead of fixed message count
|
||||
compress_end = self._find_tail_cut_by_tokens(messages, compress_start)
|
||||
|
||||
# A double role collision can merge the summary into the first tail
|
||||
# row. Keep an actionable user event out of that position by retaining
|
||||
# the genuinely older assistant/tool bridge when one exists.
|
||||
if compress_end == latest_actionable_idx:
|
||||
bridge_idx = latest_actionable_idx - 1
|
||||
if bridge_idx >= 0 and messages[bridge_idx].get("role") == "tool":
|
||||
bridge_idx = self._align_boundary_backward(
|
||||
messages, latest_actionable_idx
|
||||
)
|
||||
elif bridge_idx < 0 or messages[bridge_idx].get("role") != "assistant":
|
||||
bridge_idx = -1
|
||||
if bridge_idx > compress_start:
|
||||
compress_end = bridge_idx
|
||||
|
||||
if compress_start >= compress_end:
|
||||
self._record_compression_regions(
|
||||
head_messages=messages[:compress_start],
|
||||
middle_messages=[],
|
||||
tail_messages=messages[compress_end:],
|
||||
)
|
||||
telemetry["failure_class"] = "no_compressible_window"
|
||||
# No compressable window — the entire transcript fits within
|
||||
# the tail budget (soft_ceiling). Without recording this as
|
||||
# an ineffective compression the anti-thrashing guard in
|
||||
@@ -3660,6 +4102,13 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
else:
|
||||
self._summary_has_user_turn = real_user_present
|
||||
|
||||
self._record_compression_regions(
|
||||
head_messages=messages[:compress_start],
|
||||
middle_messages=turns_to_summarize,
|
||||
tail_messages=messages[compress_end:],
|
||||
)
|
||||
telemetry["chunk_count"] = 1 if turns_to_summarize else 0
|
||||
|
||||
if not self.quiet_mode:
|
||||
logger.info(
|
||||
"Context compression triggered (%d tokens >= %d threshold)",
|
||||
@@ -3719,6 +4168,12 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
self._last_summary_dropped_count = 0 # nothing actually dropped
|
||||
self._last_summary_fallback_used = False
|
||||
self._last_compress_aborted = True
|
||||
if self._last_summary_auth_failure:
|
||||
telemetry["failure_class"] = "summary_auth_failure"
|
||||
elif self._last_summary_network_failure:
|
||||
telemetry["failure_class"] = "summary_network_failure"
|
||||
else:
|
||||
telemetry["failure_class"] = "summary_generation_aborted"
|
||||
if not self.quiet_mode:
|
||||
if self._last_summary_auth_failure:
|
||||
logger.warning(
|
||||
@@ -3752,19 +4207,15 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# Phase 4: Assemble compressed message list
|
||||
compressed = []
|
||||
for i in range(compress_start):
|
||||
# If an earlier compaction handoff is in the protected head
|
||||
# (common after resume / in-place compaction), do not carry it
|
||||
# forward verbatim. It has already been rehydrated into
|
||||
# _previous_summary above and _generate_summary() will emit the
|
||||
# updated replacement below. Keeping both makes repeated
|
||||
# compactions accumulate old summaries and prevents the live prompt
|
||||
# from actually shrinking.
|
||||
if (
|
||||
summary_idx is not None
|
||||
and i == summary_idx
|
||||
and self._is_context_summary_content(messages[i].get("content"))
|
||||
):
|
||||
continue
|
||||
# An earlier compaction handoff in the protected head (common
|
||||
# after resume / in-place compaction) must not be carried forward
|
||||
# verbatim — it is already rehydrated into _previous_summary and
|
||||
# _generate_summary() emits the updated replacement below.
|
||||
# _strip_context_summary_handoff_message() handles both shapes:
|
||||
# standalone handoffs strip to None (dropped), merged handoffs
|
||||
# unwrap to their genuine prior-tail content (preserved). Do NOT
|
||||
# short-circuit on summary_idx here: a merged handoff carries real
|
||||
# user content that a blanket skip would silently delete.
|
||||
msg = _fresh_compaction_message_copy(messages[i])
|
||||
if i == 0 and msg.get("role") == "system":
|
||||
existing = msg.get("content")
|
||||
@@ -3774,7 +4225,9 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
existing,
|
||||
"\n\n" + _compression_note if isinstance(existing, str) and existing else _compression_note,
|
||||
)
|
||||
compressed.append(msg)
|
||||
stripped = self._strip_context_summary_handoff_message(msg)
|
||||
if stripped is not None:
|
||||
compressed.append(stripped)
|
||||
|
||||
# If LLM summary failed, insert a deterministic fallback so the model
|
||||
# gets at least locally recoverable continuity anchors instead of a
|
||||
@@ -3785,14 +4238,26 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
n_dropped = compress_end - compress_start
|
||||
self._last_summary_dropped_count = n_dropped
|
||||
self._last_summary_fallback_used = True
|
||||
telemetry["fallback_used"] = True
|
||||
telemetry["failure_class"] = telemetry.get("failure_class") or "summary_generation_failed"
|
||||
summary = self._build_static_fallback_summary(
|
||||
turns_to_summarize,
|
||||
reason=self._last_summary_error,
|
||||
)
|
||||
|
||||
tail_messages: List[Dict[str, Any]] = []
|
||||
for i in range(compress_end, n_messages):
|
||||
msg = _fresh_compaction_message_copy(messages[i])
|
||||
stripped = self._strip_context_summary_handoff_message(msg)
|
||||
if stripped is not None:
|
||||
tail_messages.append(stripped)
|
||||
|
||||
_merge_summary_into_tail = False
|
||||
last_head_role = compressed[-1].get("role", "user") if compressed else "user"
|
||||
first_tail_role = messages[compress_end].get("role", "user") if compress_end < n_messages else "user"
|
||||
# NOTE: derive the tail's leading role from tail_messages (post
|
||||
# handoff-strip), not messages[compress_end] — a stripped stale
|
||||
# handoff must not influence alternation-safe role selection.
|
||||
first_tail_role = tail_messages[0].get("role", "user") if tail_messages else None
|
||||
# When the only protected head message is the system prompt, the
|
||||
# summary becomes the first *visible* message in the API request
|
||||
# (most adapters — Anthropic, Bedrock — send the system prompt as
|
||||
@@ -3820,11 +4285,9 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# always has at least one user turn.
|
||||
if not _force_user_leading:
|
||||
_user_survives = any(
|
||||
messages[i].get("role") == "user"
|
||||
for i in range(0, compress_start)
|
||||
message.get("role") == "user" for message in compressed
|
||||
) or any(
|
||||
messages[i].get("role") == "user"
|
||||
for i in range(compress_end, n_messages)
|
||||
message.get("role") == "user" for message in tail_messages
|
||||
)
|
||||
if not _user_survives:
|
||||
_force_user_leading = True
|
||||
@@ -3836,7 +4299,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
summary_role = "assistant"
|
||||
# If the chosen role collides with the tail AND flipping wouldn't
|
||||
# collide with the head, flip it.
|
||||
if summary_role == first_tail_role:
|
||||
if first_tail_role and summary_role == first_tail_role:
|
||||
flipped = "assistant" if summary_role == "user" else "user"
|
||||
if flipped != last_head_role and not _force_user_leading:
|
||||
summary_role = flipped
|
||||
@@ -3845,7 +4308,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# (e.g. head=assistant, tail=user — neither role works).
|
||||
# Merge the summary into the first tail message instead
|
||||
# of inserting a standalone message that breaks alternation.
|
||||
_merge_summary_into_tail = True
|
||||
_merge_summary_into_tail = bool(tail_messages)
|
||||
|
||||
# When the summary lands as a standalone role="user" message,
|
||||
# weak models read the verbatim "## Active Task" quote of a past
|
||||
@@ -3867,9 +4330,8 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
),
|
||||
})
|
||||
|
||||
for i in range(compress_end, n_messages):
|
||||
msg = _fresh_compaction_message_copy(messages[i])
|
||||
if _merge_summary_into_tail and i == compress_end:
|
||||
for tail_idx, msg in enumerate(tail_messages):
|
||||
if _merge_summary_into_tail and tail_idx == 0:
|
||||
# Merge the summary into the first tail message, but place
|
||||
# the END MARKER at the very end so the model sees an
|
||||
# unambiguous boundary. Old tail content is preserved as
|
||||
|
||||
@@ -30,10 +30,12 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
import threading
|
||||
from datetime import datetime
|
||||
@@ -173,6 +175,43 @@ def _session_was_rotated_by_compression(session_db: Any, session_id: str) -> boo
|
||||
)
|
||||
|
||||
|
||||
def _emit_compression_attempt_telemetry(
|
||||
agent: Any,
|
||||
*,
|
||||
started_at: float,
|
||||
commit_status: str,
|
||||
split_status: str,
|
||||
failure_class: str | None = None,
|
||||
) -> None:
|
||||
"""Emit one content-free JSON log line for a compression attempt."""
|
||||
try:
|
||||
telemetry = getattr(agent.context_compressor, "_last_compression_telemetry", None)
|
||||
if not isinstance(telemetry, dict):
|
||||
telemetry = {}
|
||||
payload = dict(telemetry)
|
||||
payload.setdefault("event", "compression_attempt")
|
||||
payload.setdefault("attempt_id", getattr(agent, "_compression_attempt_id", "") or uuid.uuid4().hex)
|
||||
payload.setdefault("session_id", getattr(agent, "session_id", "") or "")
|
||||
payload["total_duration_ms"] = int((time.monotonic() - started_at) * 1000)
|
||||
payload["commit_status"] = commit_status
|
||||
payload["split_status"] = split_status
|
||||
if failure_class:
|
||||
payload["failure_class"] = failure_class
|
||||
payload.setdefault("chunking", False)
|
||||
payload.setdefault("chunk_count", 0)
|
||||
payload["fallback_used"] = bool(
|
||||
payload.get("fallback_used")
|
||||
or getattr(agent.context_compressor, "_last_summary_fallback_used", False)
|
||||
or getattr(agent.context_compressor, "_last_aux_model_failure_model", None)
|
||||
)
|
||||
logger.info(
|
||||
"context compression attempt telemetry: %s",
|
||||
json.dumps(payload, sort_keys=True, separators=(",", ":")),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("failed to emit compression attempt telemetry: %s", exc)
|
||||
|
||||
|
||||
def _compression_lock_holder(agent: Any) -> str:
|
||||
"""Build a unique holder id for the lock: pid:tid:agent-instance:uuid.
|
||||
|
||||
@@ -904,6 +943,19 @@ def compress_context(
|
||||
):
|
||||
raise RuntimeError("a compression notification is already pending")
|
||||
|
||||
_attempt_started_at = time.monotonic()
|
||||
_attempt_id = uuid.uuid4().hex
|
||||
_trigger_source = "manual" if force else "auto"
|
||||
try:
|
||||
agent._compression_attempt_id = _attempt_id
|
||||
setattr(agent.context_compressor, "_compression_telemetry_seed", {
|
||||
"attempt_id": _attempt_id,
|
||||
"session_id": agent.session_id or "",
|
||||
"trigger_source": _trigger_source,
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Codex app-server sessions: the codex agent owns the real thread context;
|
||||
# Hermes' summarizer would only rewrite a local mirror without shrinking
|
||||
# the actual thread (#36801). Route compaction to the app server's own
|
||||
@@ -1110,6 +1162,18 @@ def compress_context(
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
try:
|
||||
if hasattr(agent.context_compressor, "_begin_compression_telemetry"):
|
||||
agent.context_compressor._begin_compression_telemetry(current_tokens=approx_tokens)
|
||||
except Exception:
|
||||
pass
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class="lock_contended",
|
||||
)
|
||||
return messages, _existing_sp
|
||||
_lock_released = False
|
||||
|
||||
@@ -1235,7 +1299,7 @@ def compress_context(
|
||||
messages_before_compression = copy.deepcopy(messages)
|
||||
_activity_heartbeat = _CompressionActivityHeartbeat(agent).start()
|
||||
compressed = compress_fn(messages, **compress_kwargs)
|
||||
except BaseException:
|
||||
except BaseException as _compress_exc:
|
||||
# ANY exception after lock acquisition — memory hook, capability
|
||||
# inspection, engine lookup, or compress() — must release the lock so
|
||||
# the session isn't permanently blocked from future compression.
|
||||
@@ -1243,6 +1307,13 @@ def compress_context(
|
||||
_activity_heartbeat.stop("context compression failed")
|
||||
_activity_heartbeat = None
|
||||
_release_lock()
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class=f"exception:{type(_compress_exc).__name__}",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
if _activity_heartbeat is not None:
|
||||
@@ -1278,6 +1349,16 @@ def compress_context(
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class=(
|
||||
getattr(agent.context_compressor, "_last_summary_error", None)
|
||||
and "summary_generation_aborted"
|
||||
),
|
||||
)
|
||||
return messages, _existing_sp
|
||||
finally:
|
||||
_release_lock()
|
||||
@@ -1296,6 +1377,13 @@ def compress_context(
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class="no_progress",
|
||||
)
|
||||
_release_lock()
|
||||
return messages, _existing_sp
|
||||
|
||||
@@ -1377,7 +1465,9 @@ def compress_context(
|
||||
agent._cached_system_prompt = new_system_prompt
|
||||
|
||||
_session_commit_succeeded = False
|
||||
split_status = "not_applicable"
|
||||
if agent._session_db:
|
||||
split_status = "pending"
|
||||
try:
|
||||
# Trigger memory extraction on the current session before the
|
||||
# transcript is rewritten (runs in BOTH modes — the logical
|
||||
@@ -1405,6 +1495,7 @@ def compress_context(
|
||||
# WITHOUT destroying history, unlike a hard replace_messages).
|
||||
# See #38763.
|
||||
agent._session_db.archive_and_compact(agent.session_id, compressed)
|
||||
split_status = "in_place_committed"
|
||||
# Reset the flush identity set so the next turn's appends are
|
||||
# diffed against the COMPACTED transcript: the compacted dicts
|
||||
# are passed as conversation_history next turn and skipped by
|
||||
@@ -1521,6 +1612,7 @@ def compress_context(
|
||||
agent._session_db_created = True
|
||||
raise
|
||||
agent._session_db_created = True
|
||||
split_status = "rotated_committed"
|
||||
# Carry a persistent /goal onto the continuation session.
|
||||
# Compression mints a fresh child id; load_goal does a flat
|
||||
# per-session lookup with no parent walk, so without this an
|
||||
@@ -1559,6 +1651,7 @@ def compress_context(
|
||||
}
|
||||
_session_commit_succeeded = True
|
||||
except Exception as e:
|
||||
split_status = "aborted" if locals().get("old_session_id") is None and not in_place else "failed_not_indexed"
|
||||
# If the rotation rolled back to the parent (orphan-avoidance
|
||||
# above), agent.session_id is the still-indexed parent and
|
||||
# old_session_id was cleared — so this is recovery, not an
|
||||
@@ -1701,6 +1794,18 @@ def compress_context(
|
||||
agent.session_id or "none", _pre_msg_count, len(compressed),
|
||||
f"{_compressed_est:,}",
|
||||
)
|
||||
_commit_status = "committed" if split_status in {"not_applicable", "in_place_committed", "rotated_committed"} else "aborted"
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status=_commit_status,
|
||||
split_status=split_status,
|
||||
failure_class=(
|
||||
"session_split_failed"
|
||||
if split_status in {"failed_not_indexed", "aborted"}
|
||||
else None
|
||||
),
|
||||
)
|
||||
return compressed, new_system_prompt
|
||||
finally:
|
||||
# Release the lock on the OLD session_id only AFTER rotation completed
|
||||
|
||||
@@ -424,6 +424,15 @@ compression:
|
||||
# "claude-sonnet": 0.35
|
||||
# "gpt-5": 0.30
|
||||
|
||||
# Optional absolute token cap for the compression trigger (default: null = disabled).
|
||||
# When set, compression fires at the LOWER of the ratio-based threshold and this
|
||||
# absolute token count — first-fires-wins. It never fires later than this count
|
||||
# regardless of which model is active (useful when switching between models with
|
||||
# very different context windows). Clamped to the model's context length at
|
||||
# apply-time, so a cap above the window is a no-op (ratio-based threshold wins).
|
||||
# Survives model switches and fallback activations.
|
||||
# threshold_tokens: 200000
|
||||
|
||||
# Existing Codex gpt-5.5 behavior: raise Hermes' compaction trigger to 85%
|
||||
# for the ChatGPT Codex OAuth route. Set false to opt back down to threshold.
|
||||
codex_gpt55_autoraise: true
|
||||
|
||||
@@ -772,6 +772,16 @@ def load_cli_config() -> Dict[str, Any]:
|
||||
if redact is not None:
|
||||
os.environ["HERMES_REDACT_SECRETS"] = str(redact).lower()
|
||||
|
||||
# Session-search index knobs (hermes_state reads the env carriers).
|
||||
sessions_config = defaults.get("sessions", {})
|
||||
if isinstance(sessions_config, dict):
|
||||
if "cjk_fts" in sessions_config:
|
||||
os.environ["HERMES_CJK_FTS"] = str(sessions_config["cjk_fts"])
|
||||
if "search_slow_ms" in sessions_config:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(
|
||||
sessions_config["search_slow_ms"]
|
||||
)
|
||||
|
||||
return defaults
|
||||
|
||||
# Load configuration at module startup
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
knoal
|
||||
# PR #64067 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
vexclawx31
|
||||
# PR #33215 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
LevSky22
|
||||
# PR #66069 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
87degrees
|
||||
# PR #52923 salvage (slack ping/pong staleness)
|
||||
@@ -0,0 +1,2 @@
|
||||
kaiyisg
|
||||
# PR #24848 salvage
|
||||
@@ -0,0 +1 @@
|
||||
DanielMaly
|
||||
@@ -0,0 +1,2 @@
|
||||
MrAbsaroka
|
||||
# PR #40064 salvage (slack dedup TTL)
|
||||
@@ -0,0 +1 @@
|
||||
AndrewMoryakov
|
||||
@@ -1481,6 +1481,14 @@ def _bridge_max_turns_from_config(home: "Path") -> None:
|
||||
agent_cfg = cfg.get("agent", {})
|
||||
if isinstance(agent_cfg, dict) and "max_turns" in agent_cfg:
|
||||
os.environ["HERMES_MAX_ITERATIONS"] = str(agent_cfg["max_turns"])
|
||||
# config-authoritative knobs for the session-search index (config.yaml
|
||||
# sessions.* wins over stale env; env stays the cross-process carrier).
|
||||
sessions_cfg = cfg.get("sessions", {})
|
||||
if isinstance(sessions_cfg, dict):
|
||||
if "cjk_fts" in sessions_cfg:
|
||||
os.environ["HERMES_CJK_FTS"] = str(sessions_cfg["cjk_fts"])
|
||||
if "search_slow_ms" in sessions_cfg:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(sessions_cfg["search_slow_ms"])
|
||||
|
||||
|
||||
def _current_max_iterations() -> int:
|
||||
@@ -1763,6 +1771,16 @@ if _config_path.exists():
|
||||
os.environ["HERMES_AUTO_CONTINUE_FRESHNESS"] = str(
|
||||
_agent_cfg["gateway_auto_continue_freshness"]
|
||||
)
|
||||
# config-authoritative knobs for the session-search index; same
|
||||
# bridge semantics as the agent settings above.
|
||||
_sessions_cfg = _cfg.get("sessions", {})
|
||||
if _sessions_cfg and isinstance(_sessions_cfg, dict):
|
||||
if "cjk_fts" in _sessions_cfg:
|
||||
os.environ["HERMES_CJK_FTS"] = str(_sessions_cfg["cjk_fts"])
|
||||
if "search_slow_ms" in _sessions_cfg:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(
|
||||
_sessions_cfg["search_slow_ms"]
|
||||
)
|
||||
_display_cfg = _cfg.get("display", {})
|
||||
if _display_cfg and isinstance(_display_cfg, dict):
|
||||
if "busy_input_mode" in _display_cfg:
|
||||
@@ -11818,6 +11836,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
# (mirrors the same field's treatment in
|
||||
# build_session_context_prompt via _format_untrusted_prompt_value).
|
||||
_safe_user_name = neutralize_untrusted_inline_text(source.user_name)
|
||||
# On Slack, expose the current author's verifiable user ID next to
|
||||
# the display name (#17916): "mention me again" requests need a
|
||||
# trusted `<@U...>` target for the CURRENT speaker — display names
|
||||
# are ambiguous and historical mentions may point at someone else.
|
||||
# The user_id comes from the Slack event envelope (not
|
||||
# user-editable text), so it does not need neutralization.
|
||||
if source.platform == Platform.SLACK and source.user_id:
|
||||
_safe_user_name = (
|
||||
f"{_safe_user_name} | Slack user <@{source.user_id}>"
|
||||
)
|
||||
message_text = f"[{_safe_user_name}] {message_text}"
|
||||
|
||||
# Prepend channel context from history backfill (if any). This
|
||||
@@ -17644,6 +17672,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
("compression", "enabled"),
|
||||
("compression", "threshold"),
|
||||
("compression", "model_thresholds"),
|
||||
("compression", "threshold_tokens"),
|
||||
("compression", "codex_gpt55_autoraise"),
|
||||
("compression", "codex_app_server_auto"),
|
||||
("compression", "target_ratio"),
|
||||
|
||||
+51
-1
@@ -17,7 +17,7 @@ import threading
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -526,6 +526,13 @@ def build_session_context_prompt(
|
||||
"current message's Slack block/attachment payload when available, but "
|
||||
"you still cannot call Slack APIs yourself."
|
||||
)
|
||||
if context.shared_multi_user_session:
|
||||
lines.append(
|
||||
"In shared Slack threads, use the current turn's sender prefix "
|
||||
"as the only verified current-author mention target. Do not "
|
||||
"guess or reuse `<@U...>` mentions from names, memory, or prior "
|
||||
"conversation history."
|
||||
)
|
||||
elif context.source.platform == Platform.DISCORD:
|
||||
# Inject the Discord IDs block only when the agent actually has
|
||||
# Discord tools loaded this session — i.e. the user opted into
|
||||
@@ -691,6 +698,11 @@ class SessionEntry:
|
||||
display_name: Optional[str] = None
|
||||
platform: Optional[Platform] = None
|
||||
chat_type: str = "dm"
|
||||
|
||||
# Lightweight persisted key/value state scoped to this session entry
|
||||
# (e.g. Slack thread-context watermarks). Survives gateway restarts via
|
||||
# the routing index; must stay small and JSON-serializable.
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# Token tracking
|
||||
input_tokens: int = 0
|
||||
@@ -760,6 +772,7 @@ class SessionEntry:
|
||||
"display_name": self.display_name,
|
||||
"platform": self.platform.value if self.platform else None,
|
||||
"chat_type": self.chat_type,
|
||||
"metadata": self.metadata,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"cache_read_tokens": self.cache_read_tokens,
|
||||
@@ -839,6 +852,7 @@ class SessionEntry:
|
||||
display_name=data.get("display_name"),
|
||||
platform=platform,
|
||||
chat_type=data.get("chat_type", "dm"),
|
||||
metadata=dict(data.get("metadata") or {}),
|
||||
input_tokens=data.get("input_tokens", 0),
|
||||
output_tokens=data.get("output_tokens", 0),
|
||||
cache_read_tokens=data.get("cache_read_tokens", 0),
|
||||
@@ -2149,6 +2163,42 @@ class SessionStore:
|
||||
display_name=entry.display_name,
|
||||
)
|
||||
|
||||
def get_session_metadata(
|
||||
self,
|
||||
session_key: str,
|
||||
key: str,
|
||||
default: Any = None,
|
||||
) -> Any:
|
||||
"""Return a metadata value stored on a live session entry."""
|
||||
with self._lock:
|
||||
self._ensure_loaded_locked()
|
||||
entry = self._entries.get(session_key)
|
||||
if entry is None:
|
||||
return default
|
||||
return entry.metadata.get(key, default)
|
||||
|
||||
def set_session_metadata(
|
||||
self,
|
||||
session_key: str,
|
||||
key: str,
|
||||
value: Any,
|
||||
) -> bool:
|
||||
"""Persist a metadata value on a live session entry.
|
||||
|
||||
Values must be small and JSON-serializable — they are written into
|
||||
the routing index (state.db gateway_routing table + the legacy
|
||||
sessions.json mirror) so they survive gateway restarts.
|
||||
"""
|
||||
with self._lock:
|
||||
self._ensure_loaded_locked()
|
||||
entry = self._entries.get(session_key)
|
||||
if entry is None:
|
||||
return False
|
||||
entry.metadata[key] = value
|
||||
entry.updated_at = _now()
|
||||
self._save()
|
||||
return True
|
||||
|
||||
def set_model_override(
|
||||
self, session_key: str, override: Optional[Dict[str, Any]]
|
||||
) -> None:
|
||||
|
||||
@@ -1469,6 +1469,10 @@ DEFAULT_CONFIG = {
|
||||
# floored at 0.75 (raise-only) so compaction
|
||||
# doesn't fire with half the window still free;
|
||||
# set this above 0.75 to override the floor.
|
||||
"threshold_tokens": None, # absolute token cap — when set, compression
|
||||
# triggers at the lower of the ratio-based
|
||||
# threshold and this token count. Clamped to
|
||||
# the model's context length at apply-time.
|
||||
"target_ratio": 0.20, # fraction of threshold to preserve as recent tail
|
||||
"protect_last_n": 20, # minimum recent messages to keep uncompressed
|
||||
"max_attempts": 3, # compression retry rounds before a turn gives up
|
||||
@@ -3249,6 +3253,20 @@ DEFAULT_CONFIG = {
|
||||
# enforcement is a copy/gating change, not new migration code.
|
||||
# "off": suppress the notice entirely.
|
||||
"fts_optimize_notice": "advise",
|
||||
# CJK-bigram search index (messages_fts_cjk, cjk_unicode61 loadable
|
||||
# tokenizer). When the extension is built (native/fts5_cjk/build.sh →
|
||||
# ~/.hermes/lib/libfts5_cjk.so), 1-2 char CJK terms (일본, 项目, ...)
|
||||
# get index-speed exact matching instead of LIKE full-table scans.
|
||||
# True (default): use the index when the extension is present; the
|
||||
# setting is inert when it isn't. False: never load the extension or
|
||||
# serve the cjk index. Bridged to HERMES_CJK_FTS (internal carrier).
|
||||
"cjk_fts": True,
|
||||
# Slow session-search log threshold in milliseconds: searches at or
|
||||
# above it log one INFO line with the routing path taken (fts_cjk /
|
||||
# fts5 / trigram / like_scan) so latency regressions stay
|
||||
# attributable per query shape. 0 logs every search. Bridged to
|
||||
# HERMES_SEARCH_SLOW_MS (internal carrier).
|
||||
"search_slow_ms": 1000,
|
||||
},
|
||||
|
||||
# Contextual first-touch onboarding hints (see agent/onboarding.py).
|
||||
@@ -8548,6 +8566,14 @@ def show_config():
|
||||
print(f" Enabled: {'yes' if enabled else 'no'}")
|
||||
if enabled:
|
||||
print(f" Threshold: {compression.get('threshold', 0.50) * 100:.0f}%")
|
||||
_tt = compression.get('threshold_tokens')
|
||||
if _tt is not None:
|
||||
try:
|
||||
_tt = int(_tt)
|
||||
if _tt > 0:
|
||||
print(f" Token cap: {_tt:,} tokens (takes lower of ratio vs absolute)")
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
print(f" Target ratio: {compression.get('target_ratio', 0.20) * 100:.0f}% of threshold preserved")
|
||||
print(f" Protect last: {compression.get('protect_last_n', 20)} messages")
|
||||
print(f" Protect first: {compression.get('protect_first_n', 3)} non-system head messages")
|
||||
|
||||
@@ -6406,6 +6406,10 @@ def _print_fts_optimize_available_notice() -> None:
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1"
|
||||
).fetchone()
|
||||
or db._conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key IN "
|
||||
"('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1"
|
||||
).fetchone()
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
@@ -0,0 +1,416 @@
|
||||
"""Plugin catalog — curated, Nous-approved Hermes plugins shipped with the repo.
|
||||
|
||||
Mirrors the ``optional-mcps/`` MCP-catalog pattern (see
|
||||
:mod:`hermes_cli.mcp_catalog`): each catalog entry is a single YAML file under
|
||||
the in-tree ``plugin-catalog/`` directory, pinned to an exact 40-character
|
||||
commit SHA. Users discover entries via ``hermes plugins catalog`` /
|
||||
``hermes plugins search`` and install them with
|
||||
``hermes plugins install <name>``, which clones the pinned commit.
|
||||
|
||||
Catalog policy (see plugin-catalog/README.md for the full admission policy):
|
||||
- Entries are added only by merging a PR into hermes-agent — presence in the
|
||||
``plugin-catalog/`` directory is the human-merged approval gate.
|
||||
- Every entry pins an exact 40-hex commit SHA. SHA bumps are new PRs,
|
||||
re-reviewed as diffs. The pinned release should be at least 2 weeks old at
|
||||
pin time, mirroring the optional-mcps supply-chain rules.
|
||||
- ``plugin-catalog/removed.yaml`` is the blocklist: entries pulled from the
|
||||
catalog for security or policy reasons are recorded there so installs of
|
||||
the same name/repo are refused with the recorded reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CATALOG_TIERS = ("official", "community")
|
||||
|
||||
_SHA_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
_NAME_RE = re.compile(r"^[a-z0-9_-]{1,64}$")
|
||||
|
||||
|
||||
# ─── Data classes ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemovedEntry:
|
||||
name: str
|
||||
repo: str = ""
|
||||
reason: str = ""
|
||||
date: str = "" # ISO date string
|
||||
|
||||
|
||||
@dataclass
|
||||
class CatalogCapabilities:
|
||||
provides_tools: List[str] = field(default_factory=list)
|
||||
provides_hooks: List[str] = field(default_factory=list)
|
||||
provides_middleware: List[str] = field(default_factory=list)
|
||||
requires_env: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PluginCatalogEntry:
|
||||
name: str # catalog key, [a-z0-9_-]{1,64}
|
||||
repo: str # https:// git URL
|
||||
sha: str # 40-hex pinned commit — MANDATORY, validated
|
||||
description: str
|
||||
maintainer: str
|
||||
tier: str = "community" # one of CATALOG_TIERS
|
||||
requires_hermes: str = "" # e.g. ">=0.19" (optional)
|
||||
subdir: str = "" # optional path within the repo
|
||||
docs_url: str = ""
|
||||
platforms: List[str] = field(default_factory=list) # empty = all OSes
|
||||
capabilities: CatalogCapabilities = field(default_factory=CatalogCapabilities)
|
||||
|
||||
|
||||
# ─── Directory resolution ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def get_catalog_dir() -> Path:
|
||||
"""Return the ``plugin-catalog/`` directory shipped with this checkout.
|
||||
|
||||
``HERMES_PLUGIN_CATALOG_DIR`` overrides the location for tests only —
|
||||
read via ``os.getenv`` at call time so monkeypatched values take effect.
|
||||
"""
|
||||
override = os.getenv("HERMES_PLUGIN_CATALOG_DIR", "").strip()
|
||||
if override:
|
||||
return Path(override)
|
||||
return Path(__file__).resolve().parent.parent / "plugin-catalog"
|
||||
|
||||
|
||||
# ─── Loading / validation ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _str_list(raw: Any) -> List[str]:
|
||||
"""Coerce a YAML value into a list of strings (drop non-strings)."""
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
return [str(item) for item in raw if isinstance(item, (str, int, float))]
|
||||
|
||||
|
||||
def _parse_entry(path: Path) -> Optional[PluginCatalogEntry]:
|
||||
"""Parse and validate one catalog YAML file.
|
||||
|
||||
Returns ``None`` (after logging a warning) on any validation failure —
|
||||
the loader never raises for a bad entry.
|
||||
"""
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
except Exception as exc:
|
||||
logger.warning("Plugin catalog: failed to read %s: %s", path, exc)
|
||||
return None
|
||||
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("Plugin catalog: %s: entry must be a mapping", path)
|
||||
return None
|
||||
|
||||
name = str(data.get("name") or "")
|
||||
if not _NAME_RE.match(name):
|
||||
logger.warning(
|
||||
"Plugin catalog: %s: invalid name %r (must match [a-z0-9_-]{1,64})",
|
||||
path, name,
|
||||
)
|
||||
return None
|
||||
|
||||
repo = str(data.get("repo") or "")
|
||||
if not repo.startswith("https://"):
|
||||
logger.warning(
|
||||
"Plugin catalog: %s: repo must be an https:// URL (got %r)",
|
||||
path, repo,
|
||||
)
|
||||
return None
|
||||
|
||||
sha = str(data.get("sha") or "").strip().lower()
|
||||
if not _SHA_RE.match(sha):
|
||||
logger.warning(
|
||||
"Plugin catalog: %s: sha must be a full 40-character hex commit "
|
||||
"SHA (got %r)", path, data.get("sha"),
|
||||
)
|
||||
return None
|
||||
|
||||
tier = str(data.get("tier") or "community")
|
||||
if tier not in CATALOG_TIERS:
|
||||
logger.warning(
|
||||
"Plugin catalog: %s: tier must be one of %s (got %r)",
|
||||
path, "/".join(CATALOG_TIERS), tier,
|
||||
)
|
||||
return None
|
||||
|
||||
caps_raw = data.get("capabilities") or {}
|
||||
if not isinstance(caps_raw, dict):
|
||||
caps_raw = {}
|
||||
capabilities = CatalogCapabilities(
|
||||
provides_tools=_str_list(caps_raw.get("provides_tools")),
|
||||
provides_hooks=_str_list(caps_raw.get("provides_hooks")),
|
||||
provides_middleware=_str_list(caps_raw.get("provides_middleware")),
|
||||
requires_env=_str_list(caps_raw.get("requires_env")),
|
||||
)
|
||||
|
||||
return PluginCatalogEntry(
|
||||
name=name,
|
||||
repo=repo,
|
||||
sha=sha,
|
||||
description=str(data.get("description") or "").strip(),
|
||||
maintainer=str(data.get("maintainer") or "").strip(),
|
||||
tier=tier,
|
||||
requires_hermes=str(data.get("requires_hermes") or "").strip(),
|
||||
subdir=str(data.get("subdir") or "").strip(),
|
||||
docs_url=str(data.get("docs_url") or "").strip(),
|
||||
platforms=_str_list(data.get("platforms")),
|
||||
capabilities=capabilities,
|
||||
)
|
||||
|
||||
|
||||
def load_catalog() -> List[PluginCatalogEntry]:
|
||||
"""Return all valid catalog entries, sorted by name.
|
||||
|
||||
Parses every ``*.yaml`` in the catalog dir except ``removed.yaml``.
|
||||
Invalid entries are skipped with a logged warning; this function never
|
||||
raises for a malformed entry.
|
||||
"""
|
||||
return _load_entries_from_dir(get_catalog_dir())
|
||||
|
||||
|
||||
def _load_entries_from_dir(root: Path) -> List[PluginCatalogEntry]:
|
||||
"""Parse all catalog entry files in *root* (skipping ``removed.yaml``)."""
|
||||
if not root.is_dir():
|
||||
return []
|
||||
entries: List[PluginCatalogEntry] = []
|
||||
for path in sorted(root.glob("*.yaml")):
|
||||
if path.name == "removed.yaml":
|
||||
continue
|
||||
entry = _parse_entry(path)
|
||||
if entry is not None:
|
||||
entries.append(entry)
|
||||
return entries
|
||||
|
||||
|
||||
def get_catalog_entry(name: str) -> Optional[PluginCatalogEntry]:
|
||||
"""Look up a single catalog entry by name."""
|
||||
for entry in load_catalog():
|
||||
if entry.name == name:
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def search_catalog(query: str) -> List[PluginCatalogEntry]:
|
||||
"""Case-insensitive substring search over name, description, and
|
||||
declared tools. An empty query returns the whole catalog."""
|
||||
return filter_entries(load_catalog(), query)
|
||||
|
||||
|
||||
def filter_entries(
|
||||
entries: List[PluginCatalogEntry], query: str
|
||||
) -> List[PluginCatalogEntry]:
|
||||
"""Filter *entries* with :func:`search_catalog` semantics.
|
||||
|
||||
Lets callers that already hold a (possibly live-fetched) entry list
|
||||
apply the same matching rules without re-loading the catalog.
|
||||
"""
|
||||
q = (query or "").strip().lower()
|
||||
if not q:
|
||||
return entries
|
||||
results: List[PluginCatalogEntry] = []
|
||||
for entry in entries:
|
||||
haystacks = [entry.name, entry.description]
|
||||
haystacks.extend(entry.capabilities.provides_tools)
|
||||
if any(q in h.lower() for h in haystacks):
|
||||
results.append(entry)
|
||||
return results
|
||||
|
||||
|
||||
# ─── Removed / blocklist ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _normalize_repo(url: str) -> str:
|
||||
"""Normalize a repo URL for comparison (.git suffix and trailing slash
|
||||
stripped, lowercased)."""
|
||||
return url.strip().rstrip("/").removesuffix(".git").lower()
|
||||
|
||||
|
||||
def load_removed_list() -> List[RemovedEntry]:
|
||||
"""Load ``plugin-catalog/removed.yaml`` (the ``removed:`` list).
|
||||
|
||||
Missing or malformed files yield an empty list — never raises.
|
||||
"""
|
||||
path = get_catalog_dir() / "removed.yaml"
|
||||
if not path.is_file():
|
||||
return []
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
except Exception as exc:
|
||||
logger.warning("Plugin catalog: failed to read %s: %s", path, exc)
|
||||
return []
|
||||
raw_list = data.get("removed") if isinstance(data, dict) else None
|
||||
if not isinstance(raw_list, list):
|
||||
return []
|
||||
removed: List[RemovedEntry] = []
|
||||
for raw in raw_list:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
name = str(raw.get("name") or "")
|
||||
if not name:
|
||||
continue
|
||||
removed.append(
|
||||
RemovedEntry(
|
||||
name=name,
|
||||
repo=str(raw.get("repo") or ""),
|
||||
reason=str(raw.get("reason") or ""),
|
||||
date=str(raw.get("date") or ""),
|
||||
)
|
||||
)
|
||||
return removed
|
||||
|
||||
|
||||
def find_removed(name_or_repo: str) -> Optional[RemovedEntry]:
|
||||
"""Match *name_or_repo* against the removed blocklist.
|
||||
|
||||
Matches by exact catalog name OR by repo URL (normalized — ``.git``
|
||||
suffix and trailing slashes are ignored).
|
||||
"""
|
||||
if not name_or_repo:
|
||||
return None
|
||||
candidate = name_or_repo.strip()
|
||||
candidate_repo = _normalize_repo(candidate)
|
||||
for entry in load_removed_list():
|
||||
if candidate == entry.name:
|
||||
return entry
|
||||
if entry.repo and candidate_repo == _normalize_repo(entry.repo):
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
# ─── Live index ──────────────────────────────────────────────────────────────
|
||||
|
||||
# GitHub contents API for the in-repo catalog dir. Unauthenticated (60 req/hr
|
||||
# rate limit) — fine for interactive use, and any failure falls back to the
|
||||
# in-tree catalog silently.
|
||||
_LIVE_INDEX_URL = (
|
||||
"https://api.github.com/repos/NousResearch/hermes-agent/contents/"
|
||||
"plugin-catalog?ref=main"
|
||||
)
|
||||
_LIVE_TTL_SECONDS = 6 * 60 * 60 # 6h
|
||||
_REQUEST_TIMEOUT = 5.0
|
||||
|
||||
|
||||
def _live_cache_dir() -> Path:
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
return get_hermes_home() / "cache" / "plugin-catalog"
|
||||
|
||||
|
||||
def fetch_live_catalog(*, force: bool = False) -> Optional[Path]:
|
||||
"""Refresh the catalog cache from the GitHub repo; return the cache dir.
|
||||
|
||||
Lists ``plugin-catalog/*.yaml`` via the GitHub contents API, raw-fetches
|
||||
each file, and stores them under ``<hermes_home>/cache/plugin-catalog/``
|
||||
with a 6-hour TTL (repeat searches don't re-hit the API). Returns the
|
||||
cache directory on success (or fresh cache), or ``None`` on ANY network
|
||||
or parse failure — callers then fall back to the in-tree catalog.
|
||||
"""
|
||||
cache = _live_cache_dir()
|
||||
marker = cache / ".fetched"
|
||||
if not force and marker.is_file():
|
||||
try:
|
||||
age = time.time() - marker.stat().st_mtime
|
||||
except OSError:
|
||||
age = _LIVE_TTL_SECONDS + 1
|
||||
if age < _LIVE_TTL_SECONDS:
|
||||
return cache
|
||||
|
||||
try:
|
||||
import httpx
|
||||
|
||||
resp = httpx.get(
|
||||
_LIVE_INDEX_URL,
|
||||
timeout=_REQUEST_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
headers={"Accept": "application/vnd.github+json"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
listing = resp.json()
|
||||
if not isinstance(listing, list):
|
||||
raise ValueError("unexpected contents-API payload")
|
||||
|
||||
fetched: dict[str, str] = {}
|
||||
for item in listing:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
fname = str(item.get("name") or "")
|
||||
url = str(item.get("download_url") or "")
|
||||
if not fname.endswith(".yaml") or not url:
|
||||
continue
|
||||
file_resp = httpx.get(
|
||||
url, timeout=_REQUEST_TIMEOUT, follow_redirects=True
|
||||
)
|
||||
file_resp.raise_for_status()
|
||||
fetched[fname] = file_resp.text
|
||||
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
# Replace stale cached entries wholesale so removed files disappear.
|
||||
for old in cache.glob("*.yaml"):
|
||||
if old.name not in fetched:
|
||||
old.unlink(missing_ok=True)
|
||||
for fname, text in fetched.items():
|
||||
(cache / fname).write_text(text, encoding="utf-8")
|
||||
marker.touch()
|
||||
return cache
|
||||
except Exception as exc:
|
||||
logger.debug("Plugin catalog: live index fetch failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def load_catalog_live() -> List[PluginCatalogEntry]:
|
||||
"""Return catalog entries, preferring a live-fetched (or cached) index.
|
||||
|
||||
Falls back silently to the in-tree catalog when the network is
|
||||
unavailable or the fetch fails.
|
||||
"""
|
||||
cache = fetch_live_catalog()
|
||||
if cache is not None and any(
|
||||
p.name != "removed.yaml" for p in cache.glob("*.yaml")
|
||||
):
|
||||
return _load_entries_from_dir(cache)
|
||||
return load_catalog()
|
||||
|
||||
|
||||
# ─── Human summaries ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def entry_capability_summary(entry: PluginCatalogEntry) -> str:
|
||||
"""One-paragraph human summary of what an entry declares, shown at
|
||||
install prompts so the user knows what they're granting."""
|
||||
caps = entry.capabilities
|
||||
parts: List[str] = []
|
||||
if caps.provides_tools:
|
||||
parts.append(f"registers tool(s): {', '.join(caps.provides_tools)}")
|
||||
if caps.provides_hooks:
|
||||
parts.append(f"hook(s): {', '.join(caps.provides_hooks)}")
|
||||
if caps.provides_middleware:
|
||||
parts.append(f"middleware: {', '.join(caps.provides_middleware)}")
|
||||
if caps.requires_env:
|
||||
parts.append(f"requires env var(s): {', '.join(caps.requires_env)}")
|
||||
if not parts:
|
||||
capability_text = "declares no tools, hooks, middleware, or env vars"
|
||||
else:
|
||||
capability_text = "; ".join(parts)
|
||||
bits = [
|
||||
f"{entry.name} ({entry.tier}, maintained by {entry.maintainer})",
|
||||
]
|
||||
if entry.description:
|
||||
bits.append(entry.description)
|
||||
bits.append(f"This plugin {capability_text}.")
|
||||
if entry.platforms:
|
||||
bits.append(f"Platforms: {', '.join(entry.platforms)}.")
|
||||
if entry.requires_hermes:
|
||||
bits.append(f"Requires Hermes {entry.requires_hermes}.")
|
||||
return " ".join(bits)
|
||||
@@ -0,0 +1,434 @@
|
||||
"""``hermes plugins validate`` — admission checks for a plugin directory.
|
||||
|
||||
This is the command the plugin-catalog admission CI (and the
|
||||
``.github/actions/plugin-validate`` composite action) runs against a
|
||||
candidate plugin. It performs static manifest checks plus a
|
||||
subprocess-isolated capability probe: the plugin is imported and its
|
||||
``register(ctx)`` called against a minimal recording stub context in a
|
||||
scratch child process (with a throwaway ``HERMES_HOME``), so a crashing or
|
||||
malicious plugin cannot take down the CLI, and the *actually registered*
|
||||
tools/hooks/middleware are compared against the manifest's declared
|
||||
``provides_*`` lists.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
_UPPER_SNAKE_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
|
||||
_CONFIG_TYPES = {"str", "bool", "int"}
|
||||
_PROBE_TIMEOUT = 30
|
||||
_PROBE_SENTINEL = "HERMES_VALIDATE_JSON:"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationReport:
|
||||
"""Result of validating one plugin directory."""
|
||||
|
||||
checks: List[Tuple[str, bool, str]] = field(default_factory=list)
|
||||
warnings: List[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def failures(self) -> List[str]:
|
||||
return [detail or name for name, ok, detail in self.checks if not ok]
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return all(ok for _name, ok, _detail in self.checks)
|
||||
|
||||
@property
|
||||
def exit_code(self) -> int:
|
||||
return 0 if self.ok else 1
|
||||
|
||||
def add(self, name: str, ok: bool, detail: str = "") -> None:
|
||||
self.checks.append((name, ok, detail))
|
||||
|
||||
def warn(self, message: str) -> None:
|
||||
self.warnings.append(message)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"ok": self.ok,
|
||||
"checks": [
|
||||
{"name": name, "ok": ok, "detail": detail}
|
||||
for name, ok, detail in self.checks
|
||||
],
|
||||
"warnings": list(self.warnings),
|
||||
}
|
||||
|
||||
|
||||
# ─── Static checks ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _requires_hermes_spec_valid(spec: str) -> bool:
|
||||
"""Strictly validate a ``requires_hermes`` spec.
|
||||
|
||||
Unlike :func:`hermes_cli.plugins._version_satisfies` (permissive at load
|
||||
time), validation REJECTS clauses whose version segment doesn't parse —
|
||||
a typo'd spec should fail admission, not silently gate nothing.
|
||||
"""
|
||||
from hermes_cli.plugins import _VERSION_COMPARATOR_RE, _version_tuple
|
||||
|
||||
for clause in spec.split(","):
|
||||
clause = clause.strip()
|
||||
if not clause:
|
||||
continue
|
||||
m = _VERSION_COMPARATOR_RE.match(clause)
|
||||
target = m.group(2) if m else clause
|
||||
if _version_tuple(target) is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _check_manifest_fields(report: ValidationReport, manifest: dict) -> None:
|
||||
missing = [
|
||||
f for f in ("name", "version", "description") if not manifest.get(f)
|
||||
]
|
||||
if missing:
|
||||
report.add(
|
||||
"manifest fields",
|
||||
False,
|
||||
f"plugin.yaml missing required field(s): {', '.join(missing)}",
|
||||
)
|
||||
else:
|
||||
report.add("manifest fields", True, "name, version, description present")
|
||||
|
||||
|
||||
def _check_requires_hermes(report: ValidationReport, manifest: dict) -> None:
|
||||
spec = str(manifest.get("requires_hermes") or "").strip()
|
||||
if not spec:
|
||||
report.add("requires_hermes", True, "not declared")
|
||||
return
|
||||
if _requires_hermes_spec_valid(spec):
|
||||
report.add("requires_hermes", True, f"spec {spec!r} parses")
|
||||
else:
|
||||
report.add(
|
||||
"requires_hermes",
|
||||
False,
|
||||
f"requires_hermes spec {spec!r} does not parse "
|
||||
"(expected e.g. \">=0.19\" or \">=0.19, <1.0\")",
|
||||
)
|
||||
|
||||
|
||||
def _check_config_spec(report: ValidationReport, manifest: dict) -> None:
|
||||
raw = manifest.get("config")
|
||||
if raw in (None, [], {}):
|
||||
report.add("config spec", True, "not declared")
|
||||
return
|
||||
problems: List[str] = []
|
||||
if not isinstance(raw, list):
|
||||
problems.append("config: must be a list of mappings")
|
||||
else:
|
||||
for i, item in enumerate(raw):
|
||||
if not isinstance(item, dict) or not item.get("key"):
|
||||
problems.append(f"config[{i}]: must be a mapping with a 'key'")
|
||||
continue
|
||||
typ = item.get("type")
|
||||
if typ is not None and str(typ) not in _CONFIG_TYPES:
|
||||
problems.append(
|
||||
f"config[{i}] ({item['key']}): type must be one of "
|
||||
f"{'/'.join(sorted(_CONFIG_TYPES))}"
|
||||
)
|
||||
secret = item.get("secret")
|
||||
if secret is not None and not isinstance(secret, bool):
|
||||
problems.append(
|
||||
f"config[{i}] ({item['key']}): secret must be a boolean"
|
||||
)
|
||||
if problems:
|
||||
report.add("config spec", False, "; ".join(problems))
|
||||
else:
|
||||
report.add("config spec", True, "shape valid")
|
||||
|
||||
|
||||
def _check_requires_env(report: ValidationReport, manifest: dict) -> None:
|
||||
raw = manifest.get("requires_env") or []
|
||||
problems: List[str] = []
|
||||
if not isinstance(raw, list):
|
||||
problems.append("requires_env: must be a list")
|
||||
raw = []
|
||||
for i, entry in enumerate(raw):
|
||||
if isinstance(entry, str):
|
||||
name = entry
|
||||
elif isinstance(entry, dict):
|
||||
name = str(entry.get("name") or "")
|
||||
else:
|
||||
problems.append(f"requires_env[{i}]: must be a string or mapping")
|
||||
continue
|
||||
if not _UPPER_SNAKE_RE.match(name):
|
||||
problems.append(
|
||||
f"requires_env[{i}]: {name!r} is not UPPER_SNAKE_CASE"
|
||||
)
|
||||
if problems:
|
||||
report.add("requires_env", False, "; ".join(problems))
|
||||
else:
|
||||
report.add("requires_env", True, "all entries UPPER_SNAKE")
|
||||
|
||||
|
||||
# ─── Capability probe (subprocess-isolated) ──────────────────────────────────
|
||||
|
||||
# Self-contained harness run in a scratch child process. Imports the plugin
|
||||
# module using the same file-location mechanics PluginManager uses, calls
|
||||
# register() against a recording stub ctx, and prints a sentinel-prefixed
|
||||
# JSON line of what was actually registered. Deliberately imports NOTHING
|
||||
# from hermes so a hostile plugin only sees a bare interpreter.
|
||||
_PROBE_SCRIPT = r"""
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
|
||||
plugin_dir = sys.argv[1]
|
||||
sentinel = sys.argv[2]
|
||||
|
||||
recorded = {"tools": [], "hooks": [], "middleware": [], "commands": []}
|
||||
|
||||
|
||||
class RecordingContext:
|
||||
plugin_config = {}
|
||||
profile_name = "default"
|
||||
|
||||
def register_tool(self, name, *args, **kwargs):
|
||||
recorded["tools"].append(str(name))
|
||||
|
||||
def register_hook(self, hook_name, callback):
|
||||
recorded["hooks"].append(str(hook_name))
|
||||
|
||||
def register_middleware(self, kind, callback):
|
||||
recorded["middleware"].append(str(kind))
|
||||
|
||||
def register_command(self, name, *args, **kwargs):
|
||||
recorded["commands"].append(str(name))
|
||||
|
||||
def register_cli_command(self, name, *args, **kwargs):
|
||||
recorded["commands"].append(str(name))
|
||||
|
||||
def __getattr__(self, _name):
|
||||
# Any other registration surface (platforms, providers, skills,
|
||||
# context engines, ...) is accepted as a no-op — the probe only
|
||||
# audits the declared-capability categories.
|
||||
def _noop(*args, **kwargs):
|
||||
return None
|
||||
|
||||
return _noop
|
||||
|
||||
|
||||
def emit(payload):
|
||||
print(sentinel + json.dumps(payload))
|
||||
|
||||
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"hermes_validate_probe_plugin",
|
||||
plugin_dir + "/__init__.py",
|
||||
submodule_search_locations=[plugin_dir],
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
module.__path__ = [plugin_dir]
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
except Exception as exc:
|
||||
emit({"error": "import failed: %s" % exc})
|
||||
sys.exit(0)
|
||||
|
||||
register = getattr(module, "register", None)
|
||||
if register is None:
|
||||
emit({"error": "no register() function"})
|
||||
sys.exit(0)
|
||||
|
||||
try:
|
||||
register(RecordingContext())
|
||||
except Exception as exc:
|
||||
emit({"error": "register() raised: %s" % exc})
|
||||
sys.exit(0)
|
||||
|
||||
emit(recorded)
|
||||
"""
|
||||
|
||||
|
||||
def _run_capability_probe(plugin_dir: Path) -> Tuple[Optional[dict], str]:
|
||||
"""Run the recording probe in a scratch subprocess.
|
||||
|
||||
Returns ``(recorded, error)`` — exactly one is meaningful: *recorded*
|
||||
is the ``{tools, hooks, middleware, commands}`` dict on success, and
|
||||
*error* is a human-readable failure description otherwise.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory(prefix="hermes-validate-") as scratch:
|
||||
env = dict(os.environ)
|
||||
env["HERMES_HOME"] = scratch
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
_PROBE_SCRIPT,
|
||||
str(plugin_dir),
|
||||
_PROBE_SENTINEL,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_PROBE_TIMEOUT,
|
||||
env=env,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return None, f"capability probe timed out after {_PROBE_TIMEOUT}s"
|
||||
|
||||
payload: Optional[dict] = None
|
||||
for line in (result.stdout or "").splitlines():
|
||||
if line.startswith(_PROBE_SENTINEL):
|
||||
try:
|
||||
payload = json.loads(line[len(_PROBE_SENTINEL):])
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
|
||||
if payload is None:
|
||||
err = (result.stderr or "").strip()
|
||||
return None, (
|
||||
"capability probe produced no result "
|
||||
f"(exit {result.returncode})" + (f": {err}" if err else "")
|
||||
)
|
||||
if "error" in payload:
|
||||
return None, str(payload["error"])
|
||||
return payload, ""
|
||||
|
||||
|
||||
def _declared_list(manifest: dict, key: str) -> List[str]:
|
||||
raw = manifest.get(key) or []
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
return [str(item) for item in raw if isinstance(item, str)]
|
||||
|
||||
|
||||
def _check_capabilities(
|
||||
report: ValidationReport, manifest: dict, plugin_dir: Path
|
||||
) -> Optional[dict]:
|
||||
"""Probe actual registrations and diff against declared capabilities.
|
||||
|
||||
Returns the recorded dict (for the built-in collision check) or None
|
||||
when the probe failed / was skipped.
|
||||
"""
|
||||
if not (plugin_dir / "__init__.py").is_file():
|
||||
report.warn(
|
||||
"no __init__.py — capability probe skipped (manifest-only plugin)"
|
||||
)
|
||||
report.add("capability probe", True, "skipped (no __init__.py)")
|
||||
return None
|
||||
|
||||
recorded, error = _run_capability_probe(plugin_dir)
|
||||
if recorded is None:
|
||||
report.add("capability probe", False, error)
|
||||
return None
|
||||
report.add("capability probe", True, "register() ran in isolation")
|
||||
|
||||
for kind, manifest_key in (
|
||||
("tools", "provides_tools"),
|
||||
("hooks", "provides_hooks"),
|
||||
("middleware", "provides_middleware"),
|
||||
):
|
||||
declared = set(_declared_list(manifest, manifest_key))
|
||||
actual = set(recorded.get(kind) or [])
|
||||
undeclared = sorted(actual - declared)
|
||||
unregistered = sorted(declared - actual)
|
||||
if undeclared:
|
||||
report.add(
|
||||
f"declared {kind}",
|
||||
False,
|
||||
f"undeclared {kind} registered (not in {manifest_key}): "
|
||||
f"{', '.join(undeclared)}",
|
||||
)
|
||||
else:
|
||||
report.add(f"declared {kind}", True, "matches registrations")
|
||||
if unregistered:
|
||||
report.warn(
|
||||
f"{manifest_key} declares {', '.join(unregistered)} "
|
||||
f"but register() did not register them"
|
||||
)
|
||||
return recorded
|
||||
|
||||
|
||||
def _builtin_tool_names() -> List[str]:
|
||||
"""Return the built-in tool registry names (discovery-timing safe).
|
||||
|
||||
``tools.registry`` starts empty — built-in tool modules self-register on
|
||||
import, so we must run ``discover_builtin_tools()`` first (idempotent;
|
||||
see the AGENTS.md discover_plugins timing pitfall).
|
||||
"""
|
||||
try:
|
||||
from tools.registry import discover_builtin_tools, registry
|
||||
|
||||
discover_builtin_tools()
|
||||
return list(registry.get_all_tool_names())
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _check_builtin_collisions(
|
||||
report: ValidationReport, manifest: dict, recorded: Optional[dict]
|
||||
) -> None:
|
||||
candidate_tools = set(_declared_list(manifest, "provides_tools"))
|
||||
if recorded:
|
||||
candidate_tools.update(recorded.get("tools") or [])
|
||||
if not candidate_tools:
|
||||
report.add("built-in tool collisions", True, "no tools to check")
|
||||
return
|
||||
builtin = set(_builtin_tool_names())
|
||||
collisions = sorted(candidate_tools & builtin)
|
||||
if collisions:
|
||||
report.add(
|
||||
"built-in tool collisions",
|
||||
False,
|
||||
"tool name(s) collide with built-in tools: "
|
||||
f"{', '.join(collisions)}",
|
||||
)
|
||||
else:
|
||||
report.add("built-in tool collisions", True, "no collisions")
|
||||
|
||||
|
||||
# ─── Entry point ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def validate_plugin_dir(plugin_dir: Path) -> ValidationReport:
|
||||
"""Run every admission check against *plugin_dir* and return the report."""
|
||||
report = ValidationReport()
|
||||
plugin_dir = Path(plugin_dir)
|
||||
|
||||
if not plugin_dir.is_dir():
|
||||
report.add(
|
||||
"plugin directory", False, f"{plugin_dir} is not a directory"
|
||||
)
|
||||
return report
|
||||
|
||||
manifest_file = plugin_dir / "plugin.yaml"
|
||||
if not manifest_file.is_file():
|
||||
manifest_file = plugin_dir / "plugin.yml"
|
||||
if not manifest_file.is_file():
|
||||
report.add("manifest", False, "no plugin.yaml in the plugin directory")
|
||||
return report
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
manifest = yaml.safe_load(
|
||||
manifest_file.read_text(encoding="utf-8")
|
||||
)
|
||||
except Exception as exc:
|
||||
report.add("manifest", False, f"plugin.yaml failed to parse: {exc}")
|
||||
return report
|
||||
if not isinstance(manifest, dict):
|
||||
report.add("manifest", False, "plugin.yaml must be a mapping")
|
||||
return report
|
||||
report.add("manifest", True, "plugin.yaml parses")
|
||||
|
||||
_check_manifest_fields(report, manifest)
|
||||
_check_requires_hermes(report, manifest)
|
||||
_check_config_spec(report, manifest)
|
||||
_check_requires_env(report, manifest)
|
||||
recorded = _check_capabilities(report, manifest, plugin_dir)
|
||||
_check_builtin_collisions(report, manifest, recorded)
|
||||
return report
|
||||
@@ -39,6 +39,7 @@ import importlib.util
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
@@ -79,6 +80,93 @@ class PluginToolOverrideError(PermissionError):
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hermes version gate (manifest ``requires_hermes``)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VERSION_COMPARATOR_RE = re.compile(r"^\s*(>=|<=|==|!=|>|<)\s*(.+?)\s*$")
|
||||
|
||||
|
||||
def _running_hermes_version() -> str:
|
||||
"""Return the running Hermes version string.
|
||||
|
||||
Prefers installed package metadata (matches ``hermes_cli/main.py``'s
|
||||
version reporting), falling back to ``hermes_cli.__version__`` for
|
||||
source checkouts, then ``"0.0.0"`` as a last resort.
|
||||
"""
|
||||
try:
|
||||
return importlib.metadata.version("hermes-agent")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from hermes_cli import __version__
|
||||
return __version__
|
||||
except Exception:
|
||||
return "0.0.0"
|
||||
|
||||
|
||||
def _version_tuple(v: str) -> Optional[tuple]:
|
||||
"""Parse ``major.minor.patch`` into a comparable tuple.
|
||||
|
||||
Leading ``v`` and pre-release/build metadata (``-rc1``, ``+abc``) are
|
||||
stripped; missing segments default to 0. Returns ``None`` when any
|
||||
segment is non-numeric.
|
||||
"""
|
||||
s = str(v).strip().lstrip("v")
|
||||
s = re.split(r"[-+]", s, 1)[0]
|
||||
parts = s.split(".")
|
||||
while len(parts) < 3:
|
||||
parts.append("0")
|
||||
try:
|
||||
return tuple(int(p) for p in parts[:3])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _version_satisfies(spec: str, current: str) -> bool:
|
||||
"""Return True when *current* satisfies *spec*.
|
||||
|
||||
*spec* supports ``>=``, ``>``, ``<=``, ``<``, ``==``, ``!=`` and
|
||||
comma-separated combinations (all must hold). A bare version is treated
|
||||
as ``>=``. Non-numeric version segments fall back to permissive True
|
||||
(with a debug log) — no new dependency, so no full PEP 440 handling.
|
||||
"""
|
||||
if not spec or not spec.strip():
|
||||
return True
|
||||
cur = _version_tuple(current)
|
||||
if cur is None:
|
||||
logger.debug(
|
||||
"requires_hermes: unparseable running version %r — allowing", current,
|
||||
)
|
||||
return True
|
||||
for clause in spec.split(","):
|
||||
clause = clause.strip()
|
||||
if not clause:
|
||||
continue
|
||||
m = _VERSION_COMPARATOR_RE.match(clause)
|
||||
if m:
|
||||
op, target = m.group(1), m.group(2)
|
||||
else:
|
||||
op, target = ">=", clause
|
||||
tgt = _version_tuple(target)
|
||||
if tgt is None:
|
||||
logger.debug(
|
||||
"requires_hermes: unparseable version spec %r — allowing", clause,
|
||||
)
|
||||
continue
|
||||
ok = {
|
||||
">=": cur >= tgt,
|
||||
"<=": cur <= tgt,
|
||||
"==": cur == tgt,
|
||||
"!=": cur != tgt,
|
||||
">": cur > tgt,
|
||||
"<": cur < tgt,
|
||||
}[op]
|
||||
if not ok:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin developer debug logging
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -312,6 +400,16 @@ class PluginManifest:
|
||||
# category plugin at ``plugins/image_gen/openai/`` the key is
|
||||
# ``image_gen/openai``. When empty, falls back to ``name``.
|
||||
key: str = ""
|
||||
# Minimum/exact Hermes version requirement, e.g. ``">=0.19"``. Empty =
|
||||
# no requirement. Checked at load time; unsatisfied plugins are recorded
|
||||
# with an error and skipped (no register() call, no traceback).
|
||||
requires_hermes: str = ""
|
||||
# Declared config keys from the manifest's ``config:`` section — a list
|
||||
# of ``{key, prompt, type (str|bool|int), default, secret (bool)}``
|
||||
# dicts. secret=true values are prompted into ~/.hermes/.env; secret
|
||||
# =false values live under ``plugins.entries.<plugin_id>.<key>`` in
|
||||
# config.yaml. Exposed to plugins via ``ctx.plugin_config``.
|
||||
config_spec: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -386,6 +484,40 @@ class PluginContext:
|
||||
except Exception:
|
||||
return "default"
|
||||
|
||||
# -- declared plugin config ---------------------------------------------
|
||||
|
||||
@property
|
||||
def plugin_config(self) -> Dict[str, Any]:
|
||||
"""Return this plugin's effective config values.
|
||||
|
||||
Built from the manifest's ``config:`` spec defaults, overlaid with
|
||||
whatever the operator set under ``plugins.entries.<plugin_id>`` in
|
||||
config.yaml (config.yaml wins on key collision). Secret keys
|
||||
(``secret: true``) are stored in ``~/.hermes/.env`` instead and are
|
||||
NOT surfaced here — read them via ``os.environ``.
|
||||
"""
|
||||
merged: Dict[str, Any] = {}
|
||||
for spec in self.manifest.config_spec or []:
|
||||
key = spec.get("key")
|
||||
if not key:
|
||||
continue
|
||||
if spec.get("secret"):
|
||||
continue # secrets live in .env, never in config.yaml
|
||||
if "default" in spec:
|
||||
merged[key] = spec.get("default")
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
cfg = load_config() or {}
|
||||
except Exception:
|
||||
cfg = {}
|
||||
plugin_id = self.manifest.key or self.manifest.name
|
||||
entries = (cfg.get("plugins") or {}).get("entries") or {}
|
||||
entry = entries.get(plugin_id) or {}
|
||||
if isinstance(entry, dict):
|
||||
for key, value in entry.items():
|
||||
merged[key] = value
|
||||
return merged
|
||||
|
||||
# -- tool registration --------------------------------------------------
|
||||
|
||||
def register_tool(
|
||||
@@ -1632,6 +1764,23 @@ class PluginManager:
|
||||
"Parsed manifest: key=%s name=%s kind=%s source=%s path=%s",
|
||||
key, name, kind, source, plugin_dir,
|
||||
)
|
||||
raw_config = data.get("config", [])
|
||||
config_spec: List[Dict[str, Any]] = []
|
||||
if isinstance(raw_config, list):
|
||||
for item in raw_config:
|
||||
if isinstance(item, dict) and item.get("key"):
|
||||
config_spec.append(dict(item))
|
||||
else:
|
||||
logger.warning(
|
||||
"Plugin %s: ignoring invalid config entry %r "
|
||||
"(must be a mapping with a 'key')", key, item,
|
||||
)
|
||||
elif raw_config:
|
||||
logger.warning(
|
||||
"Plugin %s: 'config' must be a list of mappings; ignoring",
|
||||
key,
|
||||
)
|
||||
|
||||
return PluginManifest(
|
||||
name=name,
|
||||
version=str(data.get("version", "")),
|
||||
@@ -1644,6 +1793,8 @@ class PluginManager:
|
||||
path=str(plugin_dir),
|
||||
kind=kind,
|
||||
key=key,
|
||||
requires_hermes=str(data.get("requires_hermes") or "").strip(),
|
||||
config_spec=config_spec,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
@@ -1748,6 +1899,24 @@ class PluginManager:
|
||||
def _load_plugin(self, manifest: PluginManifest) -> None:
|
||||
"""Import a plugin module and call its ``register(ctx)`` function."""
|
||||
loaded = LoadedPlugin(manifest=manifest)
|
||||
|
||||
# requires_hermes gate — skip cleanly (no import, no traceback) when
|
||||
# the running Hermes version doesn't satisfy the manifest spec.
|
||||
if manifest.requires_hermes:
|
||||
current = _running_hermes_version()
|
||||
if not _version_satisfies(manifest.requires_hermes, current):
|
||||
loaded.enabled = False
|
||||
loaded.error = (
|
||||
f"requires hermes {manifest.requires_hermes}, "
|
||||
f"running {current}"
|
||||
)
|
||||
self._plugins[manifest.key or manifest.name] = loaded
|
||||
logger.warning(
|
||||
"Plugin '%s' skipped: %s",
|
||||
manifest.key or manifest.name, loaded.error,
|
||||
)
|
||||
return
|
||||
|
||||
logger.debug(
|
||||
"Loading plugin '%s' (source=%s, kind=%s, path=%s)",
|
||||
manifest.key or manifest.name, manifest.source, manifest.kind, manifest.path,
|
||||
|
||||
+724
-18
@@ -446,19 +446,61 @@ def _require_installed_plugin(name: str, plugins_dir: Path, console) -> Path:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _install_plugin_core(identifier: str, *, force: bool) -> tuple[Path, dict, str]:
|
||||
def _raise_removed(removed) -> None:
|
||||
"""Raise PluginOperationError describing a blocklisted plugin."""
|
||||
detail = removed.reason or "no reason recorded"
|
||||
if removed.date:
|
||||
detail += f" (removed {removed.date})"
|
||||
raise PluginOperationError(
|
||||
f"Plugin '{removed.name}' was removed from the Hermes plugin "
|
||||
f"catalog and is blocked from installation: {detail}"
|
||||
)
|
||||
|
||||
|
||||
def _install_plugin_core(
|
||||
identifier: str,
|
||||
*,
|
||||
force: bool,
|
||||
ref: Optional[str] = None,
|
||||
skip_removed_check: bool = False,
|
||||
) -> tuple[Path, dict, str]:
|
||||
"""Clone Git plugin into ``~/.hermes/plugins``.
|
||||
|
||||
``ref`` — optional git commit SHA (or tag) checked out after clone.
|
||||
When given, the clone is full-depth (no ``--depth 1``) so any commit is
|
||||
reachable.
|
||||
|
||||
Unless ``skip_removed_check`` is set, the identifier and the resolved
|
||||
repo URL are checked against the plugin catalog's removed blocklist
|
||||
(``plugin-catalog/removed.yaml``); a hit raises ``PluginOperationError``
|
||||
with the recorded reason and date.
|
||||
|
||||
Returns ``(target_dir, installed_manifest, canonical_name)``.
|
||||
Raises ``PluginOperationError`` on failure.
|
||||
"""
|
||||
import tempfile
|
||||
|
||||
if not skip_removed_check:
|
||||
from hermes_cli.plugin_catalog import find_removed
|
||||
|
||||
# Check the raw identifier first (catches catalog names before URL
|
||||
# resolution), then the resolved repo URL below.
|
||||
removed = find_removed(identifier)
|
||||
if removed is not None:
|
||||
_raise_removed(removed)
|
||||
|
||||
try:
|
||||
git_url, subdir = _resolve_git_url(identifier)
|
||||
except ValueError as e:
|
||||
raise PluginOperationError(str(e)) from e
|
||||
|
||||
if not skip_removed_check:
|
||||
from hermes_cli.plugin_catalog import find_removed
|
||||
|
||||
removed = find_removed(git_url)
|
||||
if removed is not None:
|
||||
_raise_removed(removed)
|
||||
|
||||
plugins_dir = _plugins_dir()
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
@@ -468,9 +510,15 @@ def _install_plugin_core(identifier: str, *, force: bool) -> tuple[Path, dict, s
|
||||
if not git_exe:
|
||||
raise PluginOperationError("git is not installed or not in PATH.")
|
||||
|
||||
clone_cmd = [git_exe, "clone"]
|
||||
if ref is None:
|
||||
# Fast path — only the tip is needed.
|
||||
clone_cmd += ["--depth", "1"]
|
||||
clone_cmd += [git_url, str(tmp_clone)]
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[git_exe, "clone", "--depth", "1", git_url, str(tmp_clone)],
|
||||
clone_cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
@@ -488,6 +536,24 @@ def _install_plugin_core(identifier: str, *, force: bool) -> tuple[Path, dict, s
|
||||
err = (result.stderr or result.stdout or "").strip()
|
||||
raise PluginOperationError(f"Git clone failed:\n{err}")
|
||||
|
||||
if ref is not None:
|
||||
try:
|
||||
checkout = subprocess.run(
|
||||
[git_exe, "-C", str(tmp_clone), "checkout", ref],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
raise PluginOperationError(
|
||||
f"Git checkout of ref '{ref}' timed out after 60 seconds.",
|
||||
) from e
|
||||
if checkout.returncode != 0:
|
||||
err = (checkout.stderr or checkout.stdout or "").strip()
|
||||
raise PluginOperationError(
|
||||
f"Git checkout of ref '{ref}' failed:\n{err}"
|
||||
)
|
||||
|
||||
# Resolve the directory within the clone that holds the plugin.
|
||||
if subdir:
|
||||
tmp_target = _resolve_subdir_within(tmp_clone, subdir)
|
||||
@@ -547,12 +613,98 @@ def _install_plugin_core(identifier: str, *, force: bool) -> tuple[Path, dict, s
|
||||
return target, installed_manifest, installed_name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Catalog integration helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CATALOG_SIDECAR = ".hermes-catalog.json"
|
||||
|
||||
|
||||
def _looks_like_catalog_name(identifier: str) -> bool:
|
||||
"""True when *identifier* could be a catalog entry name (not a URL/shorthand)."""
|
||||
if not identifier or "/" in identifier or "\\" in identifier:
|
||||
return False
|
||||
if identifier.startswith(("https://", "http://", "git@", "ssh://", "file://")):
|
||||
return False
|
||||
from hermes_cli.plugin_catalog import _NAME_RE
|
||||
|
||||
return bool(_NAME_RE.match(identifier))
|
||||
|
||||
|
||||
def _get_live_catalog_entry(name: str):
|
||||
"""Look up *name* in the live-refreshed catalog (falls back in-tree)."""
|
||||
from hermes_cli.plugin_catalog import load_catalog_live
|
||||
|
||||
for entry in load_catalog_live():
|
||||
if entry.name == name:
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def _catalog_install_identifier(entry) -> str:
|
||||
"""Build the ``_install_plugin_core`` identifier for a catalog entry.
|
||||
|
||||
Uses the explicit ``#subdir`` fragment form understood by
|
||||
:func:`_resolve_git_url` when the entry lives in a repo subdirectory.
|
||||
"""
|
||||
if entry.subdir:
|
||||
return f"{entry.repo}#{entry.subdir}"
|
||||
return entry.repo
|
||||
|
||||
|
||||
def _write_catalog_sidecar(target: Path, entry) -> None:
|
||||
"""Record catalog provenance in ``.hermes-catalog.json`` inside *target*.
|
||||
|
||||
The sidecar is how ``update``/``list``/``doctor`` know the plugin came
|
||||
from the catalog (and at which pin).
|
||||
"""
|
||||
import datetime
|
||||
|
||||
sidecar = {
|
||||
"catalog_name": entry.name,
|
||||
"repo": entry.repo,
|
||||
"sha": entry.sha,
|
||||
"installed_at": datetime.datetime.now(datetime.timezone.utc)
|
||||
.isoformat(timespec="seconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"tier": entry.tier,
|
||||
}
|
||||
try:
|
||||
(target / _CATALOG_SIDECAR).write_text(
|
||||
json.dumps(sidecar, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
except OSError as exc:
|
||||
logger.warning("Failed to write catalog sidecar in %s: %s", target, exc)
|
||||
|
||||
|
||||
def _read_catalog_sidecar(plugin_dir: Path) -> Optional[dict]:
|
||||
"""Return the parsed catalog provenance sidecar, or None."""
|
||||
path = plugin_dir / _CATALOG_SIDECAR
|
||||
if not path.is_file():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
# Unreadable / corrupt sidecar — treat as a non-catalog install.
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def cmd_install(
|
||||
identifier: str,
|
||||
force: bool = False,
|
||||
enable: Optional[bool] = None,
|
||||
allow_removed: bool = False,
|
||||
) -> None:
|
||||
"""Install a plugin from a Git URL or owner/repo shorthand.
|
||||
"""Install a plugin from the catalog, a Git URL, or owner/repo shorthand.
|
||||
|
||||
When *identifier* matches a catalog entry name (and is not a URL or
|
||||
``owner/repo`` shorthand), the install resolves to the entry's pinned
|
||||
commit SHA and records catalog provenance in a ``.hermes-catalog.json``
|
||||
sidecar. Raw git URLs keep the direct flow but are flagged as custom
|
||||
(unreviewed) sources.
|
||||
|
||||
``allow_removed=True`` bypasses the removed-blocklist check (loudly).
|
||||
|
||||
After install, prompt "Enable now? [y/N]" unless *enable* is provided
|
||||
(True = auto-enable without prompting, False = install disabled).
|
||||
@@ -561,6 +713,51 @@ def cmd_install(
|
||||
|
||||
console = Console()
|
||||
|
||||
entry = None
|
||||
if _looks_like_catalog_name(identifier):
|
||||
from hermes_cli.plugin_catalog import (
|
||||
entry_capability_summary,
|
||||
find_removed,
|
||||
)
|
||||
|
||||
entry = _get_live_catalog_entry(identifier)
|
||||
if entry is None:
|
||||
console.print(
|
||||
f"[red]Error:[/red] '{identifier}' is not in the Hermes "
|
||||
"plugin catalog and is not a Git URL or owner/repo "
|
||||
"shorthand.\n"
|
||||
"Browse available entries with `hermes plugins search`."
|
||||
)
|
||||
sys.exit(1)
|
||||
if not allow_removed:
|
||||
removed = find_removed(entry.name) or find_removed(entry.repo)
|
||||
if removed is not None:
|
||||
try:
|
||||
_raise_removed(removed)
|
||||
except PluginOperationError as e:
|
||||
console.print(f"[red]Error:[/red] {e}")
|
||||
sys.exit(1)
|
||||
console.print(
|
||||
f"[bold]{entry.name}[/bold] "
|
||||
f"[cyan]\\[{entry.tier}][/cyan] "
|
||||
f"[dim]pinned @ {entry.sha[:8]}[/dim]"
|
||||
)
|
||||
console.print(entry_capability_summary(entry))
|
||||
identifier = _catalog_install_identifier(entry)
|
||||
else:
|
||||
console.print(
|
||||
"[yellow]Warning:[/yellow] custom (unreviewed) source — "
|
||||
"not from the Hermes catalog."
|
||||
)
|
||||
|
||||
if allow_removed:
|
||||
console.print(
|
||||
"[bold red]WARNING:[/bold red] [red]--allow-removed set — "
|
||||
"skipping the removed-plugin blocklist check. This plugin may "
|
||||
"have been removed from the catalog for security reasons. "
|
||||
"Proceed at your own risk.[/red]"
|
||||
)
|
||||
|
||||
try:
|
||||
git_url, _subdir = _resolve_git_url(identifier)
|
||||
except ValueError as e:
|
||||
@@ -582,11 +779,16 @@ def cmd_install(
|
||||
target, installed_manifest, installed_name = _install_plugin_core(
|
||||
identifier,
|
||||
force=force,
|
||||
ref=entry.sha if entry is not None else None,
|
||||
skip_removed_check=allow_removed,
|
||||
)
|
||||
except PluginOperationError as e:
|
||||
console.print(f"[red]Error:[/red] {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if entry is not None:
|
||||
_write_catalog_sidecar(target, entry)
|
||||
|
||||
if not (target / "plugin.yaml").exists() and not (target / "plugin.yml").exists() and not (
|
||||
target / "__init__.py"
|
||||
).exists():
|
||||
@@ -634,7 +836,13 @@ def cmd_install(
|
||||
|
||||
|
||||
def cmd_update(name: str) -> None:
|
||||
"""Update an installed plugin by pulling latest from its git remote."""
|
||||
"""Update an installed plugin.
|
||||
|
||||
Catalog installs (``.hermes-catalog.json`` sidecar present) are compared
|
||||
against the current catalog pin: if the pinned SHA changed, the plugin is
|
||||
force-reinstalled at the new pin (enabled state preserved). Plain git
|
||||
installs keep the existing ``git pull`` behavior.
|
||||
"""
|
||||
from rich.console import Console
|
||||
|
||||
console = Console()
|
||||
@@ -646,6 +854,11 @@ def cmd_update(name: str) -> None:
|
||||
console.print(f"[red]Error:[/red] {e}")
|
||||
sys.exit(1)
|
||||
|
||||
sidecar = _read_catalog_sidecar(target)
|
||||
if sidecar is not None and sidecar.get("catalog_name"):
|
||||
_update_catalog_plugin(name, target, sidecar, console)
|
||||
return
|
||||
|
||||
if not (target / ".git").exists():
|
||||
console.print(
|
||||
f"[red]Error:[/red] Plugin '{name}' was not installed from git "
|
||||
@@ -673,6 +886,55 @@ def cmd_update(name: str) -> None:
|
||||
console.print(f"[dim]{out}[/dim]")
|
||||
|
||||
|
||||
def _update_catalog_plugin(name: str, target: Path, sidecar: dict, console) -> None:
|
||||
"""Re-pin a catalog-installed plugin to the current catalog SHA."""
|
||||
catalog_name = str(sidecar.get("catalog_name") or name)
|
||||
entry = _get_live_catalog_entry(catalog_name)
|
||||
if entry is None:
|
||||
console.print(
|
||||
f"[red]Error:[/red] Plugin '{catalog_name}' is no longer in the "
|
||||
"catalog — it may have been removed. Check "
|
||||
"`hermes plugins doctor` and the removed blocklist."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
installed_sha = str(sidecar.get("sha") or "").strip().lower()
|
||||
if installed_sha == entry.sha:
|
||||
console.print(
|
||||
f"[green]✓[/green] Plugin [bold]{catalog_name}[/bold] is "
|
||||
f"already at catalog pin ({entry.sha[:8]})."
|
||||
)
|
||||
return
|
||||
|
||||
console.print(
|
||||
f"[dim]Updating {catalog_name} to catalog pin:[/dim] "
|
||||
f"{installed_sha[:8] or '(unknown)'} → {entry.sha[:8]}"
|
||||
)
|
||||
|
||||
# Preserve enabled state across the force reinstall.
|
||||
was_enabled = _get_enabled_set()
|
||||
|
||||
try:
|
||||
new_target, _manifest, _installed_name = _install_plugin_core(
|
||||
_catalog_install_identifier(entry),
|
||||
force=True,
|
||||
ref=entry.sha,
|
||||
)
|
||||
except PluginOperationError as e:
|
||||
console.print(f"[red]Error:[/red] {e}")
|
||||
sys.exit(1)
|
||||
|
||||
_write_catalog_sidecar(new_target, entry)
|
||||
# Restore the pre-update enabled/disabled state verbatim (the reinstall
|
||||
# itself never touches it, but be explicit in case core ever does).
|
||||
_save_enabled_set(was_enabled)
|
||||
|
||||
console.print(
|
||||
f"[green]✓[/green] Plugin [bold]{catalog_name}[/bold] updated to "
|
||||
f"{entry.sha[:8]}."
|
||||
)
|
||||
|
||||
|
||||
def cmd_remove(name: str) -> None:
|
||||
"""Remove an installed plugin by name."""
|
||||
from rich.console import Console
|
||||
@@ -1122,6 +1384,44 @@ def _filter_plugin_entries(entries: list, args: Any, enabled: set, disabled: set
|
||||
return filtered
|
||||
|
||||
|
||||
def _catalog_annotation(dir_path) -> Optional[str]:
|
||||
"""Return ``catalog:<tier>@<shaShort>`` for a catalog install, else None."""
|
||||
if not dir_path:
|
||||
return None
|
||||
try:
|
||||
sidecar = _read_catalog_sidecar(Path(dir_path))
|
||||
except Exception:
|
||||
return None
|
||||
if not sidecar or not sidecar.get("catalog_name"):
|
||||
return None
|
||||
tier = str(sidecar.get("tier") or "community")
|
||||
sha = str(sidecar.get("sha") or "")
|
||||
return f"catalog:{tier}@{sha[:8]}"
|
||||
|
||||
|
||||
def _removed_annotation(name: str, dir_path) -> Optional[str]:
|
||||
"""Return the removed-blocklist reason when *name* matches, else None."""
|
||||
try:
|
||||
from hermes_cli.plugin_catalog import find_removed
|
||||
except Exception:
|
||||
return None
|
||||
candidates = [name]
|
||||
if dir_path:
|
||||
try:
|
||||
sidecar = _read_catalog_sidecar(Path(dir_path))
|
||||
except Exception:
|
||||
sidecar = None
|
||||
if sidecar:
|
||||
candidates.extend(
|
||||
str(v) for v in (sidecar.get("catalog_name"), sidecar.get("repo")) if v
|
||||
)
|
||||
for candidate in candidates:
|
||||
removed = find_removed(candidate)
|
||||
if removed is not None:
|
||||
return removed.reason or "no reason recorded"
|
||||
return None
|
||||
|
||||
|
||||
def cmd_list(args: Any | None = None) -> None:
|
||||
"""List all plugins (bundled + user) with enabled/disabled state."""
|
||||
from rich.console import Console
|
||||
@@ -1139,16 +1439,22 @@ def cmd_list(args: Any | None = None) -> None:
|
||||
entries = _filter_plugin_entries(entries, args, enabled, disabled)
|
||||
|
||||
if getattr(args, "json", False):
|
||||
payload = [
|
||||
{
|
||||
payload = []
|
||||
for name, version, description, source, _dir, key in entries:
|
||||
row = {
|
||||
"name": name,
|
||||
"status": _plugin_status(name, enabled, disabled, key=key),
|
||||
"version": str(version),
|
||||
"description": description,
|
||||
"source": source,
|
||||
}
|
||||
for name, version, description, source, _dir, key in entries
|
||||
]
|
||||
catalog = _catalog_annotation(_dir)
|
||||
if catalog:
|
||||
row["catalog"] = catalog
|
||||
removed_reason = _removed_annotation(name, _dir)
|
||||
if removed_reason is not None:
|
||||
row["removed"] = removed_reason
|
||||
payload.append(row)
|
||||
print(json.dumps(payload, indent=2))
|
||||
return
|
||||
|
||||
@@ -1169,6 +1475,7 @@ def cmd_list(args: Any | None = None) -> None:
|
||||
table.add_column("Description")
|
||||
table.add_column("Source", style="dim")
|
||||
|
||||
removed_lines: list[str] = []
|
||||
for name, version, description, source, _dir, key in entries:
|
||||
status_name = _plugin_status(name, enabled, disabled, key=key)
|
||||
if status_name == "disabled":
|
||||
@@ -1177,10 +1484,20 @@ def cmd_list(args: Any | None = None) -> None:
|
||||
status = "[green]enabled[/green]"
|
||||
else:
|
||||
status = "[yellow]not enabled[/yellow]"
|
||||
table.add_row(name, status, str(version), description, source)
|
||||
catalog = _catalog_annotation(_dir)
|
||||
source_label = f"{source} [cyan]{catalog}[/cyan]" if catalog else source
|
||||
table.add_row(name, status, str(version), description, source_label)
|
||||
removed_reason = _removed_annotation(name, _dir)
|
||||
if removed_reason is not None:
|
||||
removed_lines.append(
|
||||
f"[red bold]✗ {name} — REMOVED from catalog: "
|
||||
f"{removed_reason}[/red bold]"
|
||||
)
|
||||
|
||||
console.print()
|
||||
console.print(table)
|
||||
for line in removed_lines:
|
||||
console.print(line)
|
||||
console.print()
|
||||
console.print("[dim]Compact view:[/dim] hermes plugins list --plain --no-bundled")
|
||||
console.print("[dim]Interactive toggle:[/dim] hermes plugins")
|
||||
@@ -1188,6 +1505,337 @@ def cmd_list(args: Any | None = None) -> None:
|
||||
console.print("[dim]Plugins are opt-in by default — only 'enabled' plugins load.[/dim]")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Catalog commands — search / browse / info / validate / doctor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _entry_capability_counts(entry) -> str:
|
||||
"""Compact capability summary like ``2 tools, 1 hook`` for table rows."""
|
||||
caps = entry.capabilities
|
||||
parts: list[str] = []
|
||||
for count, singular in (
|
||||
(len(caps.provides_tools), "tool"),
|
||||
(len(caps.provides_hooks), "hook"),
|
||||
(len(caps.provides_middleware), "middleware"),
|
||||
):
|
||||
if count:
|
||||
plural = "" if count == 1 or singular == "middleware" else "s"
|
||||
parts.append(f"{count} {singular}{plural}")
|
||||
if caps.requires_env:
|
||||
parts.append(f"{len(caps.requires_env)} env")
|
||||
return ", ".join(parts) or "—"
|
||||
|
||||
|
||||
def _render_catalog_entries(entries, console) -> None:
|
||||
"""Render catalog entries as the shared search/browse Rich table."""
|
||||
from rich.table import Table
|
||||
|
||||
table = Table(title="Hermes Plugin Catalog", show_lines=False)
|
||||
table.add_column("Name", style="bold")
|
||||
table.add_column("Tier")
|
||||
table.add_column("Description")
|
||||
table.add_column("Pinned", style="dim")
|
||||
table.add_column("Capabilities", style="dim")
|
||||
|
||||
for entry in entries:
|
||||
tier = (
|
||||
"[cyan]official[/cyan]"
|
||||
if entry.tier == "official"
|
||||
else "[magenta]community[/magenta]"
|
||||
)
|
||||
description = entry.description
|
||||
if len(description) > 60:
|
||||
description = description[:57] + "..."
|
||||
table.add_row(
|
||||
entry.name,
|
||||
tier,
|
||||
description,
|
||||
entry.sha[:8],
|
||||
_entry_capability_counts(entry),
|
||||
)
|
||||
|
||||
console.print()
|
||||
console.print(table)
|
||||
console.print()
|
||||
console.print("[dim]Details:[/dim] hermes plugins info <name>")
|
||||
console.print("[dim]Install:[/dim] hermes plugins install <name>")
|
||||
|
||||
|
||||
def cmd_search(query: str = "") -> None:
|
||||
"""Search the plugin catalog (live index when reachable)."""
|
||||
from rich.console import Console
|
||||
|
||||
from hermes_cli.plugin_catalog import filter_entries, load_catalog_live
|
||||
|
||||
console = Console()
|
||||
entries = filter_entries(load_catalog_live(), query)
|
||||
if not entries:
|
||||
if query:
|
||||
console.print(
|
||||
f"[dim]No catalog entries match '{query}'. "
|
||||
"Browse everything with `hermes plugins browse`.[/dim]"
|
||||
)
|
||||
else:
|
||||
console.print("[dim]No catalog entries available.[/dim]")
|
||||
return
|
||||
_render_catalog_entries(entries, console)
|
||||
|
||||
|
||||
def cmd_browse() -> None:
|
||||
"""List every plugin catalog entry."""
|
||||
cmd_search("")
|
||||
|
||||
|
||||
def cmd_info(name: str) -> None:
|
||||
"""Show the full catalog entry for *name*."""
|
||||
from rich.console import Console
|
||||
|
||||
from hermes_cli.plugin_catalog import find_removed
|
||||
|
||||
console = Console()
|
||||
entry = _get_live_catalog_entry(name)
|
||||
if entry is None:
|
||||
console.print(
|
||||
f"[red]Error:[/red] '{name}' is not in the plugin catalog. "
|
||||
"Browse entries with `hermes plugins search`."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
caps = entry.capabilities
|
||||
console.print()
|
||||
console.print(f"[bold]{entry.name}[/bold] [cyan]\\[{entry.tier}][/cyan]")
|
||||
if entry.description:
|
||||
console.print(entry.description)
|
||||
console.print()
|
||||
console.print(f"[dim]Repo:[/dim] {entry.repo}")
|
||||
if entry.subdir:
|
||||
console.print(f"[dim]Subdir:[/dim] {entry.subdir}")
|
||||
console.print(f"[dim]Pinned SHA:[/dim] {entry.sha}")
|
||||
console.print(f"[dim]Maintainer:[/dim] {entry.maintainer}")
|
||||
if entry.requires_hermes:
|
||||
console.print(f"[dim]Requires:[/dim] hermes {entry.requires_hermes}")
|
||||
if entry.platforms:
|
||||
console.print(f"[dim]Platforms:[/dim] {', '.join(entry.platforms)}")
|
||||
if entry.docs_url:
|
||||
console.print(f"[dim]Docs:[/dim] {entry.docs_url}")
|
||||
console.print()
|
||||
console.print(f"[dim]Tools:[/dim] {', '.join(caps.provides_tools) or '(none)'}")
|
||||
console.print(f"[dim]Hooks:[/dim] {', '.join(caps.provides_hooks) or '(none)'}")
|
||||
console.print(f"[dim]Middleware:[/dim] {', '.join(caps.provides_middleware) or '(none)'}")
|
||||
console.print(f"[dim]Env vars:[/dim] {', '.join(caps.requires_env) or '(none)'}")
|
||||
console.print()
|
||||
|
||||
removed = find_removed(entry.name) or find_removed(entry.repo)
|
||||
if removed is not None:
|
||||
detail = removed.reason or "no reason recorded"
|
||||
if removed.date:
|
||||
detail += f" (removed {removed.date})"
|
||||
console.print(
|
||||
f"[red bold]✗ REMOVED from catalog: {detail}[/red bold]"
|
||||
)
|
||||
console.print()
|
||||
|
||||
console.print(f"[dim]Install:[/dim] hermes plugins install {entry.name}")
|
||||
console.print()
|
||||
|
||||
|
||||
def cmd_validate(path: str, as_json: bool = False) -> None:
|
||||
"""Validate a plugin directory for catalog admission. Exits 0/1."""
|
||||
from rich.console import Console
|
||||
|
||||
from hermes_cli.plugin_validate import validate_plugin_dir
|
||||
|
||||
report = validate_plugin_dir(Path(path))
|
||||
|
||||
if as_json:
|
||||
print(json.dumps(report.to_dict(), indent=2))
|
||||
sys.exit(report.exit_code)
|
||||
|
||||
console = Console()
|
||||
console.print()
|
||||
for name, ok, detail in report.checks:
|
||||
mark = "[green]✓[/green]" if ok else "[red]✗[/red]"
|
||||
line = f"{mark} {name}"
|
||||
if detail:
|
||||
line += f" [dim]— {detail}[/dim]"
|
||||
console.print(line)
|
||||
for warning in report.warnings:
|
||||
console.print(f"[yellow]⚠ {warning}[/yellow]")
|
||||
console.print()
|
||||
if report.ok:
|
||||
console.print("[green bold]Validation passed.[/green bold]")
|
||||
else:
|
||||
console.print("[red bold]Validation failed.[/red bold]")
|
||||
sys.exit(report.exit_code)
|
||||
|
||||
|
||||
def _runtime_load_errors() -> dict[str, str]:
|
||||
"""Return ``{plugin_key: error}`` from a fresh PluginManager scan.
|
||||
|
||||
Uses the idempotent ``discover_plugins()`` path so we don't need a full
|
||||
agent boot; failures are non-fatal (doctor still reports what it can).
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.plugins import discover_plugins, get_plugin_manager
|
||||
|
||||
discover_plugins()
|
||||
manager = get_plugin_manager()
|
||||
return {
|
||||
key: loaded.error
|
||||
for key, loaded in manager._plugins.items()
|
||||
if loaded.error
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.debug("doctor: runtime plugin scan failed: %s", exc)
|
||||
return {}
|
||||
|
||||
|
||||
def _doctor_plugin_report(name: str, dir_path: Path, key: str,
|
||||
enabled: set, disabled: set,
|
||||
load_errors: dict[str, str]) -> dict:
|
||||
"""Collect doctor facts for one installed plugin directory."""
|
||||
from hermes_cli.plugins import _running_hermes_version, _version_satisfies
|
||||
|
||||
manifest = _read_manifest(dir_path)
|
||||
facts: dict[str, Any] = {
|
||||
"name": name,
|
||||
"manifest_ok": bool(manifest.get("name")),
|
||||
"status": _plugin_status(name, enabled, disabled, key=key),
|
||||
"load_error": load_errors.get(key) or load_errors.get(name) or "",
|
||||
"missing_env": _missing_requires_env_names(manifest),
|
||||
"requires_hermes": "",
|
||||
"catalog": "",
|
||||
"pin": "",
|
||||
"removed": "",
|
||||
}
|
||||
|
||||
spec = str(manifest.get("requires_hermes") or "").strip()
|
||||
if spec:
|
||||
current = _running_hermes_version()
|
||||
if _version_satisfies(spec, current):
|
||||
facts["requires_hermes"] = f"{spec} ✓"
|
||||
else:
|
||||
facts["requires_hermes"] = f"{spec} ✗ (running {current})"
|
||||
|
||||
sidecar = _read_catalog_sidecar(dir_path)
|
||||
if sidecar and sidecar.get("catalog_name"):
|
||||
tier = str(sidecar.get("tier") or "community")
|
||||
sha = str(sidecar.get("sha") or "")
|
||||
facts["catalog"] = f"catalog:{tier}@{sha[:8]}"
|
||||
entry = _get_live_catalog_entry(str(sidecar["catalog_name"]))
|
||||
if entry is None:
|
||||
facts["pin"] = "entry gone from catalog"
|
||||
elif entry.sha != sha:
|
||||
facts["pin"] = (
|
||||
f"behind catalog pin ({sha[:8]} → {entry.sha[:8]}) — "
|
||||
"run `hermes plugins update`"
|
||||
)
|
||||
else:
|
||||
facts["pin"] = "at catalog pin"
|
||||
|
||||
removed_reason = _removed_annotation(name, dir_path)
|
||||
if removed_reason is not None:
|
||||
facts["removed"] = removed_reason
|
||||
return facts
|
||||
|
||||
|
||||
def cmd_doctor(name: Optional[str] = None) -> None:
|
||||
"""Diagnose installed plugins (or a single one when *name* given)."""
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
plugins_dir = _plugins_dir()
|
||||
installed = [
|
||||
(d.name, d) for d in sorted(plugins_dir.iterdir()) if d.is_dir()
|
||||
]
|
||||
if name is not None:
|
||||
installed = [(n, d) for n, d in installed if n == name]
|
||||
if not installed:
|
||||
console.print(
|
||||
f"[red]Error:[/red] Plugin '{name}' not found in {plugins_dir}."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
if not installed:
|
||||
console.print("[dim]No plugins installed under[/dim] "
|
||||
f"{plugins_dir}")
|
||||
return
|
||||
|
||||
enabled = _get_enabled_set()
|
||||
disabled = _get_disabled_set()
|
||||
load_errors = _runtime_load_errors()
|
||||
|
||||
reports = [
|
||||
_doctor_plugin_report(n, d, n, enabled, disabled, load_errors)
|
||||
for n, d in installed
|
||||
]
|
||||
|
||||
if name is not None:
|
||||
facts = reports[0]
|
||||
console.print()
|
||||
console.print(f"[bold]{facts['name']}[/bold]")
|
||||
console.print(
|
||||
f"[dim]Manifest:[/dim] "
|
||||
+ ("[green]ok[/green]" if facts["manifest_ok"]
|
||||
else "[red]missing/invalid plugin.yaml[/red]")
|
||||
)
|
||||
console.print(f"[dim]Status:[/dim] {facts['status']}")
|
||||
if facts["load_error"]:
|
||||
console.print(f"[dim]Load error:[/dim] [red]{facts['load_error']}[/red]")
|
||||
if facts["missing_env"]:
|
||||
console.print(
|
||||
f"[dim]Missing env:[/dim] [yellow]{', '.join(facts['missing_env'])}[/yellow]"
|
||||
)
|
||||
if facts["requires_hermes"]:
|
||||
console.print(f"[dim]Requires:[/dim] hermes {facts['requires_hermes']}")
|
||||
if facts["catalog"]:
|
||||
console.print(f"[dim]Provenance:[/dim] {facts['catalog']}")
|
||||
console.print(f"[dim]Pin:[/dim] {facts['pin']}")
|
||||
if facts["removed"]:
|
||||
console.print(
|
||||
f"[red bold]✗ REMOVED from catalog: {facts['removed']}[/red bold]"
|
||||
)
|
||||
console.print()
|
||||
return
|
||||
|
||||
table = Table(title="Plugin Doctor", show_lines=False)
|
||||
table.add_column("Name", style="bold")
|
||||
table.add_column("Manifest")
|
||||
table.add_column("Status")
|
||||
table.add_column("Issues")
|
||||
table.add_column("Catalog", style="dim")
|
||||
|
||||
for facts in reports:
|
||||
issues: list[str] = []
|
||||
if facts["load_error"]:
|
||||
issues.append(f"[red]{facts['load_error']}[/red]")
|
||||
if facts["missing_env"]:
|
||||
issues.append(
|
||||
f"[yellow]missing env: {', '.join(facts['missing_env'])}[/yellow]"
|
||||
)
|
||||
if facts["requires_hermes"] and "✗" in facts["requires_hermes"]:
|
||||
issues.append(f"[red]requires hermes {facts['requires_hermes']}[/red]")
|
||||
if facts["removed"]:
|
||||
issues.append(
|
||||
f"[red bold]REMOVED from catalog: {facts['removed']}[/red bold]"
|
||||
)
|
||||
if facts["pin"] and "behind" in facts["pin"]:
|
||||
issues.append(f"[yellow]{facts['pin']}[/yellow]")
|
||||
table.add_row(
|
||||
facts["name"],
|
||||
"[green]ok[/green]" if facts["manifest_ok"] else "[red]bad[/red]",
|
||||
facts["status"],
|
||||
"\n".join(issues) or "[dim]—[/dim]",
|
||||
facts["catalog"] or "[dim]—[/dim]",
|
||||
)
|
||||
|
||||
console.print()
|
||||
console.print(table)
|
||||
console.print()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider plugin discovery helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1759,31 +2407,78 @@ def _run_composite_fallback(plugin_keys, plugin_labels, plugin_selected,
|
||||
print()
|
||||
|
||||
|
||||
# Public aliases for the sidecar helpers defined above — the dashboard
|
||||
# (web_server.py) imports these names; the CLI paths use the private ones.
|
||||
write_catalog_sidecar = _write_catalog_sidecar
|
||||
read_catalog_sidecar = _read_catalog_sidecar
|
||||
|
||||
|
||||
def dashboard_install_plugin(
|
||||
identifier: str,
|
||||
*,
|
||||
force: bool,
|
||||
enable: bool,
|
||||
catalog_name: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Non-interactive install for the web dashboard. Returns a JSON-serializable dict."""
|
||||
"""Non-interactive install for the web dashboard. Returns a JSON-serializable dict.
|
||||
|
||||
When *catalog_name* is given the identifier is resolved from the plugin
|
||||
catalog and the pinned commit SHA is checked out (``ref=``). Removed
|
||||
(blocklisted) plugins are refused with the recorded reason — the
|
||||
dashboard deliberately has no bypass flag (CLI-only decision).
|
||||
"""
|
||||
warnings: list[str] = []
|
||||
try:
|
||||
git_url, _subdir = _resolve_git_url(identifier)
|
||||
if git_url.startswith(("http://", "file://")):
|
||||
warnings.append(
|
||||
"Insecure URL scheme; prefer https:// or git@ for production installs.",
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
entry = None
|
||||
ref: Optional[str] = None
|
||||
|
||||
if catalog_name:
|
||||
from hermes_cli.plugin_catalog import find_removed, get_catalog_entry
|
||||
|
||||
removed = find_removed(catalog_name)
|
||||
if removed is not None:
|
||||
detail = removed.reason or "no reason recorded"
|
||||
if removed.date:
|
||||
detail += f" (removed {removed.date})"
|
||||
return {
|
||||
"ok": False,
|
||||
"error": (
|
||||
f"Plugin '{removed.name}' was removed from the Hermes "
|
||||
f"plugin catalog and is blocked from installation: {detail}"
|
||||
),
|
||||
}
|
||||
entry = get_catalog_entry(catalog_name)
|
||||
if entry is None:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": f"'{catalog_name}' is not in the Hermes plugin catalog.",
|
||||
}
|
||||
identifier = f"{entry.repo}#{entry.subdir}" if entry.subdir else entry.repo
|
||||
ref = entry.sha
|
||||
else:
|
||||
try:
|
||||
git_url, _subdir = _resolve_git_url(identifier)
|
||||
if git_url.startswith(("http://", "file://")):
|
||||
warnings.append(
|
||||
"Insecure URL scheme; prefer https:// or git@ for production installs.",
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
target, installed_manifest, installed_name = _install_plugin_core(
|
||||
identifier,
|
||||
force=force,
|
||||
ref=ref,
|
||||
)
|
||||
except PluginOperationError as exc:
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
if entry is not None:
|
||||
try:
|
||||
write_catalog_sidecar(target, entry)
|
||||
except OSError as exc:
|
||||
warnings.append(f"Could not record catalog provenance: {exc}")
|
||||
|
||||
missing_env = _missing_requires_env_names(installed_manifest)
|
||||
if enable:
|
||||
en = _get_enabled_set()
|
||||
@@ -2017,7 +2712,18 @@ def plugins_command(args) -> None:
|
||||
args.identifier,
|
||||
force=getattr(args, "force", False),
|
||||
enable=enable_arg,
|
||||
allow_removed=getattr(args, "allow_removed", False),
|
||||
)
|
||||
elif action == "search":
|
||||
cmd_search(getattr(args, "query", "") or "")
|
||||
elif action == "browse":
|
||||
cmd_browse()
|
||||
elif action == "info":
|
||||
cmd_info(args.name)
|
||||
elif action == "validate":
|
||||
cmd_validate(args.path, as_json=getattr(args, "json", False))
|
||||
elif action == "doctor":
|
||||
cmd_doctor(getattr(args, "name", None))
|
||||
elif action == "update":
|
||||
cmd_update(args.name)
|
||||
elif action in {"remove", "rm", "uninstall"}:
|
||||
|
||||
@@ -42,6 +42,51 @@ def build_plugins_parser(subparsers, *, cmd_plugins: Callable) -> None:
|
||||
action="store_true",
|
||||
help="Install disabled (skip confirmation prompt); enable later with `hermes plugins enable <name>`",
|
||||
)
|
||||
plugins_install.add_argument(
|
||||
"--allow-removed",
|
||||
action="store_true",
|
||||
help="DANGEROUS: bypass the catalog removed-plugin blocklist check",
|
||||
)
|
||||
|
||||
plugins_search = plugins_subparsers.add_parser(
|
||||
"search", help="Search the Hermes plugin catalog"
|
||||
)
|
||||
plugins_search.add_argument(
|
||||
"query",
|
||||
nargs="?",
|
||||
default="",
|
||||
help="Substring to match against entry names, descriptions, and tools",
|
||||
)
|
||||
|
||||
plugins_subparsers.add_parser(
|
||||
"browse", help="Browse every plugin catalog entry"
|
||||
)
|
||||
|
||||
plugins_info = plugins_subparsers.add_parser(
|
||||
"info", help="Show full catalog details for an entry"
|
||||
)
|
||||
plugins_info.add_argument("name", help="Catalog entry name")
|
||||
|
||||
plugins_validate = plugins_subparsers.add_parser(
|
||||
"validate",
|
||||
help="Validate a plugin directory for catalog admission (CI gate)",
|
||||
)
|
||||
plugins_validate.add_argument("path", help="Path to the plugin directory")
|
||||
plugins_validate.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="Print machine-readable JSON (for CI)",
|
||||
)
|
||||
|
||||
plugins_doctor = plugins_subparsers.add_parser(
|
||||
"doctor", help="Diagnose installed plugins"
|
||||
)
|
||||
plugins_doctor.add_argument(
|
||||
"name",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="Plugin name to inspect in detail (default: all installed)",
|
||||
)
|
||||
|
||||
plugins_update = plugins_subparsers.add_parser(
|
||||
"update", help="Pull latest changes for an installed plugin"
|
||||
|
||||
+135
-10
@@ -18849,12 +18849,27 @@ class _AgentPluginInstallBody(BaseModel):
|
||||
identifier: str
|
||||
force: bool = False
|
||||
enable: bool = True
|
||||
catalog_name: Optional[str] = None
|
||||
|
||||
|
||||
def _strip_dashboard_manifest(p: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {k: v for k, v in p.items() if not k.startswith("_")}
|
||||
|
||||
|
||||
def _plugin_runtime_status(aliases: set, enabled_set: set, disabled_set: set) -> str:
|
||||
"""Map a plugin's name aliases onto enabled/disabled/inactive.
|
||||
|
||||
Both the path-derived key (nested category plugins) and the bare
|
||||
manifest name count for enabled/disabled state, matching the runtime
|
||||
loader's back-compat lookup.
|
||||
"""
|
||||
if aliases & disabled_set:
|
||||
return "disabled"
|
||||
if aliases & enabled_set:
|
||||
return "enabled"
|
||||
return "inactive"
|
||||
|
||||
|
||||
def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
"""Agent discovery + dashboard manifests + optional provider picker metadata."""
|
||||
from hermes_cli.plugins_cmd import (
|
||||
@@ -18866,6 +18881,7 @@ def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
_get_enabled_set,
|
||||
_read_manifest as _read_plugin_manifest_at,
|
||||
)
|
||||
from hermes_cli.plugin_catalog import find_removed
|
||||
|
||||
dashboard_list = _get_dashboard_plugins()
|
||||
dash_by_name = {str(p["name"]): p for p in dashboard_list}
|
||||
@@ -18881,18 +18897,10 @@ def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
rows: List[Dict[str, Any]] = []
|
||||
|
||||
for name, version, description, source, dir_str, key in _discover_all_plugins():
|
||||
# Both the path-derived key (nested category plugins) and the bare
|
||||
# manifest name count for enabled/disabled state, matching the runtime
|
||||
# loader's back-compat lookup.
|
||||
aliases = {name}
|
||||
if key:
|
||||
aliases.add(key)
|
||||
if aliases & disabled_set:
|
||||
runtime_status = "disabled"
|
||||
elif aliases & enabled_set:
|
||||
runtime_status = "enabled"
|
||||
else:
|
||||
runtime_status = "inactive"
|
||||
runtime_status = _plugin_runtime_status(aliases, enabled_set, disabled_set)
|
||||
|
||||
dir_path = Path(dir_str)
|
||||
dm = dash_by_name.get(name)
|
||||
@@ -18926,6 +18934,14 @@ def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
removed_reason = None
|
||||
try:
|
||||
removed = find_removed(name)
|
||||
if removed is not None:
|
||||
removed_reason = removed.reason or "removed from the plugin catalog"
|
||||
except Exception:
|
||||
removed_reason = None
|
||||
|
||||
rows.append({
|
||||
"name": name,
|
||||
"version": version or "",
|
||||
@@ -18940,6 +18956,7 @@ def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
"auth_required": auth_required,
|
||||
"auth_command": auth_command,
|
||||
"user_hidden": name in hidden_plugins,
|
||||
"removed_reason": removed_reason,
|
||||
})
|
||||
|
||||
agent_names = {r["name"] for r in rows}
|
||||
@@ -18981,15 +18998,123 @@ async def get_plugins_hub(request: Request):
|
||||
raise HTTPException(status_code=500, detail="Failed to build plugins hub.") from exc
|
||||
|
||||
|
||||
def _plugins_catalog_payload() -> Dict[str, Any]:
|
||||
"""Catalog entries merged with installed-state for the dashboard.
|
||||
|
||||
Each entry carries the static catalog metadata plus:
|
||||
|
||||
* ``installed`` — a plugin with the same name is discoverable locally.
|
||||
* ``installed_sha`` — pinned SHA recorded in the plugin's
|
||||
``.hermes-catalog.json`` sidecar at install time (``None`` when the
|
||||
sidecar is absent, e.g. a pre-catalog raw-git install).
|
||||
* ``update_available`` — sidecar SHA differs from the catalog pin.
|
||||
* ``runtime_status`` — enabled/disabled/inactive for installed entries,
|
||||
``None`` otherwise.
|
||||
"""
|
||||
from hermes_cli.plugin_catalog import (
|
||||
entry_capability_summary,
|
||||
load_catalog,
|
||||
load_removed_list,
|
||||
)
|
||||
from hermes_cli.plugins_cmd import (
|
||||
_discover_all_plugins,
|
||||
_get_disabled_set,
|
||||
_get_enabled_set,
|
||||
read_catalog_sidecar,
|
||||
)
|
||||
|
||||
disabled_set = _get_disabled_set()
|
||||
enabled_set = _get_enabled_set()
|
||||
|
||||
installed: Dict[str, Dict[str, Any]] = {}
|
||||
for name, _version, _description, _source, dir_str, key in _discover_all_plugins():
|
||||
aliases = {name}
|
||||
if key:
|
||||
aliases.add(key)
|
||||
info = {
|
||||
"dir": dir_str,
|
||||
"runtime_status": _plugin_runtime_status(aliases, enabled_set, disabled_set),
|
||||
}
|
||||
for alias in aliases:
|
||||
installed[alias] = info
|
||||
|
||||
entries: List[Dict[str, Any]] = []
|
||||
for entry in load_catalog():
|
||||
caps = entry.capabilities
|
||||
local = installed.get(entry.name)
|
||||
installed_sha = None
|
||||
if local is not None:
|
||||
sidecar = read_catalog_sidecar(Path(local["dir"]))
|
||||
if sidecar:
|
||||
raw_sha = sidecar.get("sha")
|
||||
installed_sha = str(raw_sha) if raw_sha else None
|
||||
entries.append({
|
||||
"name": entry.name,
|
||||
"description": entry.description,
|
||||
"repo": entry.repo,
|
||||
"sha": entry.sha,
|
||||
"sha_short": entry.sha[:7],
|
||||
"tier": entry.tier,
|
||||
"maintainer": entry.maintainer,
|
||||
"requires_hermes": entry.requires_hermes,
|
||||
"platforms": entry.platforms,
|
||||
"capabilities": {
|
||||
"provides_tools": caps.provides_tools,
|
||||
"provides_hooks": caps.provides_hooks,
|
||||
"provides_middleware": caps.provides_middleware,
|
||||
"requires_env": caps.requires_env,
|
||||
},
|
||||
"docs_url": entry.docs_url,
|
||||
"capability_summary": entry_capability_summary(entry),
|
||||
"installed": local is not None,
|
||||
"installed_sha": installed_sha,
|
||||
"update_available": bool(installed_sha) and installed_sha != entry.sha,
|
||||
"runtime_status": local["runtime_status"] if local is not None else None,
|
||||
})
|
||||
|
||||
removed = [
|
||||
{"name": r.name, "repo": r.repo, "reason": r.reason, "date": r.date}
|
||||
for r in load_removed_list()
|
||||
]
|
||||
|
||||
return {
|
||||
"entries": entries,
|
||||
"removed": removed,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/dashboard/plugins/catalog")
|
||||
async def get_plugins_catalog(request: Request):
|
||||
"""Curated plugin catalog merged with installed-state (session protected)."""
|
||||
_require_token(request)
|
||||
try:
|
||||
return _plugins_catalog_payload()
|
||||
except Exception as exc:
|
||||
_log.warning("plugins/catalog failed: %s", exc)
|
||||
raise HTTPException(
|
||||
status_code=500, detail="Failed to build plugins catalog."
|
||||
) from exc
|
||||
|
||||
|
||||
@app.post("/api/dashboard/agent-plugins/install")
|
||||
async def post_agent_plugin_install(request: Request, body: _AgentPluginInstallBody):
|
||||
_require_token(request)
|
||||
from hermes_cli.plugins_cmd import dashboard_install_plugin
|
||||
|
||||
catalog_name = (body.catalog_name or "").strip()
|
||||
identifier = body.identifier.strip()
|
||||
if not identifier and not catalog_name:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Provide an identifier or a catalog_name.",
|
||||
)
|
||||
|
||||
result = dashboard_install_plugin(
|
||||
body.identifier.strip(),
|
||||
identifier,
|
||||
force=body.force,
|
||||
enable=body.enable,
|
||||
catalog_name=catalog_name or None,
|
||||
)
|
||||
if not result.get("ok"):
|
||||
raise HTTPException(
|
||||
|
||||
+665
-11
@@ -17,6 +17,7 @@ Key design decisions:
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sqlite3
|
||||
@@ -582,6 +583,13 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
"""
|
||||
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
||||
try:
|
||||
# Best-effort tokenizer load: a DB carrying the messages_fts_cjk
|
||||
# index needs the cjk_unicode61 extension before any statement can
|
||||
# touch that table — including the trigger-driven write probe below.
|
||||
# Without it, this probe sees the DB exactly as a tokenizer-less
|
||||
# SessionDB open would (which drops the cjk triggers to keep writes
|
||||
# working), so tokenizer absence must never classify as corruption.
|
||||
load_fts5_cjk_extension(conn)
|
||||
conn.execute("PRAGMA journal_mode").fetchone()
|
||||
rows = conn.execute("PRAGMA integrity_check").fetchall()
|
||||
problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"]
|
||||
@@ -603,7 +611,7 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
# Catch the full sqlite3 exception hierarchy (not just
|
||||
# OperationalError) so the malformed-shadow-table class is reported
|
||||
# rather than letting it crash the caller.
|
||||
for fts_table in ("messages_fts", "messages_fts_trigram"):
|
||||
for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"):
|
||||
try:
|
||||
# No-op queries against the actual FTS5 APIs the search
|
||||
# tools use. The trigram table is included because it backs
|
||||
@@ -674,6 +682,12 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
msg = str(exc).lower()
|
||||
if "no such table" in msg or "no such column" in msg:
|
||||
return None
|
||||
if "no such tokenizer: cjk_unicode61" in msg:
|
||||
# This probe process couldn't load the cjk extension while
|
||||
# the DB carries the cjk index — capability gap, not
|
||||
# corruption. A tokenizer-capable SessionDB serves it fine;
|
||||
# a tokenizer-less one self-heals by dropping the triggers.
|
||||
return None
|
||||
return str(exc)
|
||||
return None
|
||||
except sqlite3.DatabaseError as exc:
|
||||
@@ -737,13 +751,19 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
||||
try:
|
||||
for table_name in ("messages_fts", "messages_fts_trigram"):
|
||||
# The cjk index can only be rebuilt with its tokenizer loaded;
|
||||
# best-effort (a tokenizer-less host skips it at the probe below).
|
||||
load_fts5_cjk_extension(conn)
|
||||
for table_name in (
|
||||
"messages_fts", "messages_fts_trigram", "messages_fts_cjk"
|
||||
):
|
||||
try:
|
||||
conn.execute(
|
||||
f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')"
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
# Table absent (FTS disabled / trigram off) — skip it.
|
||||
# Table absent (FTS disabled / trigram off / cjk not
|
||||
# present or tokenizer unavailable) — skip it.
|
||||
continue
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -1149,6 +1169,151 @@ BEGIN
|
||||
END;
|
||||
"""
|
||||
|
||||
# ── CJK-bigram FTS index (replaces the trigram index when available) ────
|
||||
#
|
||||
# The trigram tokenizer needs >=3 chars per query term, so 1-2 char CJK
|
||||
# terms (ubiquitous in Korean/Chinese: 일본, 구글, 项目, ...) fall through
|
||||
# to a LIKE full-table scan — measured 3-6s CPU per query on multi-GB
|
||||
# installs and the dominant base cost of session_search on CJK workloads.
|
||||
#
|
||||
# ``cjk_unicode61`` (native/fts5_cjk/, a ~250-line loadable FTS5 tokenizer
|
||||
# with no dependencies) wraps unicode61: maximal CJK runs are re-emitted as
|
||||
# overlapping character bigrams (Lucene CJKAnalyzer semantics), everything
|
||||
# else passes through unchanged. FTS5 phrase semantics turn a query term's
|
||||
# consecutive bigrams into exact substring matching down to 2 chars at
|
||||
# index speed. Contributed by Soju06 (PR #65544).
|
||||
#
|
||||
# Same v23 storage discipline as the trigram table it replaces:
|
||||
# external-content over a tool-row-excluding view (zero inline text
|
||||
# copies; tool rows stay searchable via ``messages_fts``), triggers gated
|
||||
# on a DEDICATED marker pair (``fts_cjk_rebuild_high_water`` /
|
||||
# ``fts_cjk_rebuild_progress``) so a cjk-only backfill — e.g. the
|
||||
# trigram→cjk upgrade on an already-optimized DB — never gates the
|
||||
# complete ``messages_fts`` index's triggers.
|
||||
#
|
||||
# The table exists ONLY when the loadable tokenizer is available
|
||||
# (``~/.hermes/lib/libfts5_cjk.so``, built by ``native/fts5_cjk/build.sh``).
|
||||
# A process that cannot load it self-heals by dropping the cjk triggers
|
||||
# (message writes keep working; the index goes stale and is rebuilt by the
|
||||
# next ``hermes sessions optimize-storage`` on a capable host).
|
||||
#
|
||||
# Split DDL: the table/view part is safe to ensure any time; the triggers
|
||||
# are created ONLY while the index is complete-or-marker-gated. A stale
|
||||
# index (trigger gap of unknown extent) must keep its triggers DROPPED —
|
||||
# an external-content 'delete' op for a rowid the index never held is the
|
||||
# canonical FTS5 index-corruption hazard the v23 marker gating exists to
|
||||
# prevent.
|
||||
FTS_CJK_TABLE_SQL = """
|
||||
CREATE VIEW IF NOT EXISTS messages_fts_cjk_src AS
|
||||
SELECT id, role, content, tool_name, tool_calls
|
||||
FROM messages
|
||||
WHERE role <> 'tool';
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_cjk USING fts5(
|
||||
content,
|
||||
tool_name,
|
||||
tool_calls,
|
||||
content='messages_fts_cjk_src',
|
||||
content_rowid='id',
|
||||
tokenize='cjk_unicode61'
|
||||
);
|
||||
"""
|
||||
|
||||
FTS_CJK_TRIGGER_SQL = """
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_insert AFTER INSERT ON messages
|
||||
WHEN new.role <> 'tool'
|
||||
AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
|
||||
VALUES (new.id, new.content, new.tool_name, new.tool_calls);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_delete AFTER DELETE ON messages
|
||||
WHEN old.role <> 'tool'
|
||||
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
|
||||
VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_update AFTER UPDATE ON messages
|
||||
WHEN (old.content IS NOT new.content
|
||||
OR old.tool_name IS NOT new.tool_name
|
||||
OR old.tool_calls IS NOT new.tool_calls
|
||||
OR old.role IS NOT new.role)
|
||||
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
|
||||
SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls
|
||||
WHERE old.role <> 'tool';
|
||||
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
|
||||
SELECT new.id, new.content, new.tool_name, new.tool_calls
|
||||
WHERE new.role <> 'tool';
|
||||
END;
|
||||
"""
|
||||
|
||||
_FTS_CJK_TRIGGERS = (
|
||||
"messages_fts_cjk_insert",
|
||||
"messages_fts_cjk_delete",
|
||||
"messages_fts_cjk_update",
|
||||
)
|
||||
|
||||
# state_meta breadcrumb set when a tokenizer-less process had to drop the
|
||||
# cjk triggers to keep message writes alive: rows written from that moment
|
||||
# on are missing from the cjk index, so it must not serve reads until
|
||||
# `hermes sessions optimize-storage` rebuilds it on a capable host.
|
||||
FTS_CJK_STALE_KEY = "fts_cjk_stale"
|
||||
|
||||
|
||||
def fts5_cjk_so_path() -> Path:
|
||||
"""Location of the cjk_unicode61 loadable extension."""
|
||||
env = os.getenv("HERMES_FTS5_CJK_SO")
|
||||
if env:
|
||||
return Path(env).expanduser()
|
||||
return get_hermes_home() / "lib" / "libfts5_cjk.so"
|
||||
|
||||
|
||||
def _cjk_fts_config_enabled() -> bool:
|
||||
"""config.yaml ``sessions.cjk_fts`` (default on), via its env bridge."""
|
||||
return os.getenv("HERMES_CJK_FTS", "1").strip().lower() not in (
|
||||
"0", "false", "off", "no",
|
||||
)
|
||||
|
||||
|
||||
def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:
|
||||
"""Best-effort load of the cjk_unicode61 tokenizer into ``conn``.
|
||||
|
||||
Returns False (never raises) when the .so is absent, the feature is
|
||||
disabled via ``sessions.cjk_fts``, or this Python build has extension
|
||||
loading compiled out — every caller treats False as "behave exactly as
|
||||
before the cjk index existed".
|
||||
"""
|
||||
if not _cjk_fts_config_enabled():
|
||||
return False
|
||||
path = fts5_cjk_so_path()
|
||||
if not path.exists():
|
||||
return False
|
||||
try:
|
||||
conn.enable_load_extension(True)
|
||||
try:
|
||||
conn.load_extension(str(path))
|
||||
finally:
|
||||
conn.enable_load_extension(False)
|
||||
return True
|
||||
except Exception:
|
||||
logger.warning("fts5_cjk extension load failed (%s)", path, exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
# ── Legacy (v22 / inline-content) FTS DDL ──────────────────────────────
|
||||
# Used ONLY to keep an existing pre-v23 install's search working and its
|
||||
@@ -1269,6 +1434,12 @@ class SessionDB:
|
||||
self._fts_runtime_rebuild_attempted = False
|
||||
self._fts_enabled = False
|
||||
self._trigram_available = False
|
||||
# CJK-bigram index (cjk_unicode61 loadable tokenizer). _fts_cjk_loaded:
|
||||
# extension present on the writer connection; _fts_cjk_available: the
|
||||
# messages_fts_cjk table is queryable AND not marked stale. Set during
|
||||
# _init_schema / _probe_fts_cjk.
|
||||
self._fts_cjk_loaded = False
|
||||
self._fts_cjk_available = False
|
||||
self._fts_unavailable_warned = False
|
||||
self._conn = None
|
||||
try:
|
||||
@@ -1309,6 +1480,7 @@ class SessionDB:
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
apply_wal_with_fallback(self._conn, db_label="state.db")
|
||||
self._conn.execute("PRAGMA foreign_keys=ON")
|
||||
self._fts_cjk_loaded = load_fts5_cjk_extension(self._conn)
|
||||
self._init_schema()
|
||||
|
||||
try:
|
||||
@@ -1372,12 +1544,25 @@ class SessionDB:
|
||||
# Scope to trigram specifically to avoid masking unrelated tokenizer errors.
|
||||
if "no such tokenizer: trigram" in err:
|
||||
return True
|
||||
# The cjk_unicode61 tokenizer is a loadable extension — a process
|
||||
# that couldn't load it sees the same capability-error shape.
|
||||
if "no such tokenizer: cjk_unicode61" in err:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _is_trigram_unavailable_error(exc: sqlite3.OperationalError) -> bool:
|
||||
"""True when only the trigram tokenizer is missing (FTS5 itself works)."""
|
||||
return "no such tokenizer: trigram" in str(exc).lower()
|
||||
"""True when only an optional tokenizer is missing (FTS5 itself works).
|
||||
|
||||
Covers the built-in trigram tokenizer (needs SQLite >= 3.34) and the
|
||||
loadable cjk_unicode61 tokenizer — both mean "this one index can't be
|
||||
served here", never "disable FTS".
|
||||
"""
|
||||
err = str(exc).lower()
|
||||
return (
|
||||
"no such tokenizer: trigram" in err
|
||||
or "no such tokenizer: cjk_unicode61" in err
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _db_has_legacy_inline_fts(cursor: sqlite3.Cursor) -> bool:
|
||||
@@ -1444,6 +1629,124 @@ class SessionDB:
|
||||
self._warn_fts5_unavailable(exc)
|
||||
return False
|
||||
|
||||
def _ensure_fts_cjk_schema(self, cursor) -> None:
|
||||
"""Create / repair / self-heal the CJK-bigram index surface.
|
||||
|
||||
``cursor`` may be a Cursor or a Connection (both expose execute /
|
||||
executescript). Called only for v23-shape DBs with the base FTS
|
||||
surface healthy. Sets ``self._fts_cjk_available``. Never raises;
|
||||
every failure mode degrades to "no cjk index" (trigram/LIKE routing
|
||||
keeps working).
|
||||
|
||||
Cases:
|
||||
tokenizer loaded, table absent → create. Empty DB: index is
|
||||
complete by construction (triggers cover everything). Populated
|
||||
DB: set the cjk backfill markers so the id-gated triggers stay
|
||||
correct and `optimize-storage` can backfill; the index is NOT
|
||||
served until the backfill completes.
|
||||
tokenizer loaded, table present → ensure triggers (recreates any
|
||||
dropped by a tokenizer-less process), honour the stale
|
||||
breadcrumb (serve only when absent and no backfill pending).
|
||||
tokenizer NOT loaded, table present with live triggers → drop the
|
||||
cjk triggers so message INSERTs don't fail at trigger time,
|
||||
and leave the stale breadcrumb (#self-heal). The table itself
|
||||
stays for a later capable open to rebuild.
|
||||
"""
|
||||
cjk_present = bool(cursor.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name = 'messages_fts_cjk'"
|
||||
).fetchone())
|
||||
|
||||
if not self._fts_cjk_loaded:
|
||||
if cjk_present:
|
||||
live = [
|
||||
r[0] for r in cursor.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'trigger' "
|
||||
f"AND name IN ({','.join('?' for _ in _FTS_CJK_TRIGGERS)})",
|
||||
_FTS_CJK_TRIGGERS,
|
||||
).fetchall()
|
||||
]
|
||||
if live:
|
||||
# Self-heal: this process cannot tokenize, so every
|
||||
# message INSERT would die inside the cjk trigger.
|
||||
# Breadcrumb FIRST (crash between the two statements is
|
||||
# merely conservative), then drop.
|
||||
logger.warning(
|
||||
"messages_fts_cjk triggers present but the "
|
||||
"cjk_unicode61 tokenizer is unavailable (%s) — "
|
||||
"dropping the cjk triggers so message writes keep "
|
||||
"working. CJK search falls back to trigram/LIKE; "
|
||||
"run `hermes sessions optimize-storage` on a host "
|
||||
"with the extension to rebuild.",
|
||||
fts5_cjk_so_path(),
|
||||
)
|
||||
cursor.execute(
|
||||
"INSERT INTO state_meta (key, value) VALUES (?, '1') "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = '1'",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
)
|
||||
for trig in live:
|
||||
cursor.execute(f"DROP TRIGGER IF EXISTS {trig}")
|
||||
self._fts_cjk_available = False
|
||||
return
|
||||
|
||||
try:
|
||||
cursor.executescript(FTS_CJK_TABLE_SQL)
|
||||
if not cjk_present:
|
||||
# Freshly created. An empty DB's index is complete by
|
||||
# construction (triggers will cover every future row); a
|
||||
# populated DB (e.g. a v23 install predating the cjk index)
|
||||
# gets the dedicated marker pair so the id-gated triggers
|
||||
# keep NEW rows indexed while old rows await the
|
||||
# `optimize-storage` backfill. Either way any old stale
|
||||
# breadcrumb refers to a table that no longer exists.
|
||||
cursor.execute(
|
||||
"DELETE FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
)
|
||||
n_msgs = cursor.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
if n_msgs > 0:
|
||||
hw = cursor.execute(
|
||||
"SELECT COALESCE(MAX(id), 0) FROM messages"
|
||||
).fetchone()[0]
|
||||
for k, v in (
|
||||
("fts_cjk_rebuild_high_water", str(hw)),
|
||||
("fts_cjk_rebuild_progress", "0"),
|
||||
):
|
||||
cursor.execute(
|
||||
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(k, v),
|
||||
)
|
||||
stale = cursor.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
).fetchone()
|
||||
if stale:
|
||||
# A tokenizer-less process dropped the triggers at some
|
||||
# unknown point — the index has a gap of unknown extent.
|
||||
# Do NOT reinstall triggers (an external-content 'delete'
|
||||
# for an unindexed rowid corrupts the index); the next
|
||||
# `optimize-storage` run rebuilds from scratch.
|
||||
self._fts_cjk_available = False
|
||||
return
|
||||
cursor.executescript(FTS_CJK_TRIGGER_SQL)
|
||||
backfill_pending = cursor.execute(
|
||||
"SELECT 1 FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_high_water' LIMIT 1"
|
||||
).fetchone()
|
||||
self._fts_cjk_available = not backfill_pending
|
||||
except sqlite3.OperationalError:
|
||||
# Includes "no such tokenizer: cjk_unicode61" if the extension
|
||||
# loaded but registration failed — degrade to trigram/LIKE.
|
||||
logger.warning(
|
||||
"messages_fts_cjk ensure failed; CJK search stays on "
|
||||
"trigram/LIKE", exc_info=True,
|
||||
)
|
||||
self._fts_cjk_available = False
|
||||
|
||||
@staticmethod
|
||||
def _drop_fts_triggers(cursor: sqlite3.Cursor) -> None:
|
||||
for trigger in _FTS_TRIGGERS:
|
||||
@@ -1968,6 +2271,136 @@ class SessionDB:
|
||||
return False
|
||||
return bool(more)
|
||||
|
||||
# ── CJK-bigram index backfill (dedicated marker pair) ──
|
||||
#
|
||||
# Same chunk engine as the main deferred rebuild, but on the
|
||||
# ``fts_cjk_rebuild_*`` markers so a cjk-only backfill (the common case:
|
||||
# an already-optimized v23 DB gaining the cjk index) never gates the
|
||||
# complete ``messages_fts`` / trigram triggers.
|
||||
|
||||
def fts_cjk_rebuild_status(self) -> Optional[Dict[str, Any]]:
|
||||
"""CJK-index backfill progress, or None when none is pending."""
|
||||
high_water = self.get_meta("fts_cjk_rebuild_high_water")
|
||||
if high_water is None:
|
||||
return None
|
||||
progress = int(self.get_meta("fts_cjk_rebuild_progress") or 0)
|
||||
total = int(high_water)
|
||||
if total <= 0:
|
||||
return None
|
||||
pct = min(100, int(100 * progress / total))
|
||||
return {"pending": True, "total": total, "indexed": progress, "percent": pct}
|
||||
|
||||
def fts_cjk_rebuild_step(self) -> bool:
|
||||
"""Backfill one chunk of the CJK index. True while work remains."""
|
||||
if not self._fts_enabled or not self._fts_cjk_loaded:
|
||||
return False
|
||||
high_water_raw = self.get_meta("fts_cjk_rebuild_high_water")
|
||||
if high_water_raw is None:
|
||||
return False
|
||||
high_water = int(high_water_raw)
|
||||
chunk = self._FTS_REBUILD_CHUNK_ROWS
|
||||
|
||||
def _do(conn):
|
||||
row = conn.execute(
|
||||
"SELECT value FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_progress'"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False # finished (or cleared) by another process
|
||||
progress = int(row[0])
|
||||
if progress >= high_water:
|
||||
return False
|
||||
upper = min(progress + chunk, high_water)
|
||||
conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) "
|
||||
"SELECT id, content, tool_name, tool_calls FROM messages "
|
||||
"WHERE id > ? AND id <= ? AND role <> 'tool'",
|
||||
(progress, upper),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE state_meta SET value = ? "
|
||||
"WHERE key = 'fts_cjk_rebuild_progress'",
|
||||
(str(upper),),
|
||||
)
|
||||
return upper < high_water
|
||||
|
||||
try:
|
||||
more = self._execute_write(_do)
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.debug("CJK FTS rebuild chunk failed (will retry): %s", exc)
|
||||
return True
|
||||
if more is False:
|
||||
status = self.fts_cjk_rebuild_status()
|
||||
if status is not None and status["indexed"] >= status["total"]:
|
||||
self._fts_cjk_rebuild_finish()
|
||||
return False
|
||||
return bool(more)
|
||||
|
||||
def _fts_cjk_rebuild_finish(self) -> None:
|
||||
"""Boundary sweep + clear the cjk markers; index becomes servable."""
|
||||
def _do(conn):
|
||||
hw_row = conn.execute(
|
||||
"SELECT value FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_high_water'"
|
||||
).fetchone()
|
||||
if hw_row is not None:
|
||||
hw = int(hw_row[0])
|
||||
lo, hi = hw - 1000, hw + 1000
|
||||
conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) "
|
||||
"SELECT m.id, m.content, m.tool_name, m.tool_calls "
|
||||
"FROM messages m "
|
||||
"WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' "
|
||||
"AND NOT EXISTS (SELECT 1 FROM messages_fts_cjk_docsize d WHERE d.id = m.id)",
|
||||
(lo, hi),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM state_meta WHERE key IN "
|
||||
"('fts_cjk_rebuild_high_water', 'fts_cjk_rebuild_progress')"
|
||||
)
|
||||
self._execute_write(_do)
|
||||
self._fts_cjk_available = True
|
||||
logger.info("CJK FTS index backfill complete — serving CJK search.")
|
||||
|
||||
def _fts_cjk_reset_if_stale(self) -> None:
|
||||
"""Rebuild path for a stale cjk index (triggers were dropped).
|
||||
|
||||
The gap's extent is unknown, so the only safe recovery is a from-
|
||||
scratch rebuild: drop the table + triggers, clear the breadcrumb,
|
||||
recreate via ``_ensure_fts_cjk_schema`` (which sets fresh backfill
|
||||
markers on a populated DB). Called from ``optimize_fts_storage`` on
|
||||
a tokenizer-capable host; no-op when not stale.
|
||||
"""
|
||||
if not self._fts_cjk_loaded:
|
||||
return
|
||||
|
||||
def _do(conn):
|
||||
stale = conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
).fetchone()
|
||||
if not stale:
|
||||
return False
|
||||
for trig in _FTS_CJK_TRIGGERS:
|
||||
conn.execute(f"DROP TRIGGER IF EXISTS {trig}")
|
||||
conn.execute("DROP TABLE IF EXISTS messages_fts_cjk")
|
||||
conn.execute("DROP VIEW IF EXISTS messages_fts_cjk_src")
|
||||
conn.execute(
|
||||
"DELETE FROM state_meta WHERE key IN "
|
||||
f"('{FTS_CJK_STALE_KEY}', 'fts_cjk_rebuild_high_water', "
|
||||
"'fts_cjk_rebuild_progress')"
|
||||
)
|
||||
return True
|
||||
was_stale = self._execute_write(_do)
|
||||
if was_stale:
|
||||
# Recreate outside the write transaction — _ensure_fts_cjk_schema
|
||||
# uses executescript(), which implicitly commits any pending
|
||||
# transaction and must not run inside _execute_write's BEGIN
|
||||
# IMMEDIATE. Sets fresh backfill markers on a populated DB.
|
||||
with self._lock:
|
||||
self._ensure_fts_cjk_schema(self._conn)
|
||||
self._conn.commit()
|
||||
|
||||
# ── Opt-in v23 FTS storage optimization (`hermes sessions optimize-storage`) ──
|
||||
#
|
||||
# This is the ONLY path that migrates an existing legacy (v22 inline) DB
|
||||
@@ -1983,8 +2416,10 @@ class SessionDB:
|
||||
is a legacy inline-FTS install that can be optimized to the v23
|
||||
external-content schema, or a previous optimize run was interrupted
|
||||
(legacy vtables already demoted, but backfill markers and/or trash
|
||||
tables remain) and re-running would resume it. False for fresh and
|
||||
fully-optimized installs (and when FTS5 is unavailable)."""
|
||||
tables remain) and re-running would resume it, or the CJK-bigram
|
||||
index needs a backfill/rebuild on this tokenizer-capable host.
|
||||
False for fresh and fully-optimized installs (and when FTS5 is
|
||||
unavailable)."""
|
||||
if not self._fts_enabled or self.read_only:
|
||||
return False
|
||||
with self._lock:
|
||||
@@ -2000,6 +2435,14 @@ class SessionDB:
|
||||
"WHERE key = 'fts_rebuild_high_water' LIMIT 1"
|
||||
).fetchone():
|
||||
return True
|
||||
# CJK-bigram index work — only offerable when THIS process can
|
||||
# tokenize: a pending backfill (markers set at creation on a
|
||||
# populated DB) or a stale index awaiting a from-scratch rebuild.
|
||||
if self._fts_cjk_loaded and self._conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key IN "
|
||||
f"('fts_cjk_rebuild_high_water', '{FTS_CJK_STALE_KEY}') LIMIT 1"
|
||||
).fetchone():
|
||||
return True
|
||||
return self._has_fts_trash(self._conn)
|
||||
|
||||
def _has_fts_trash(self, conn) -> bool:
|
||||
@@ -2089,10 +2532,24 @@ class SessionDB:
|
||||
if legacy and not pending:
|
||||
self._demote_legacy_fts_to_trash()
|
||||
|
||||
# A stale CJK index (triggers dropped by a tokenizer-less process)
|
||||
# can only be recovered from scratch — reset it now so the cjk
|
||||
# backfill phase below rebuilds it. No-op without the tokenizer.
|
||||
self._fts_cjk_reset_if_stale()
|
||||
# An optimized v23 DB gaining the cjk index for the first time (no
|
||||
# legacy work left, tokenizer newly installed): ensure the table +
|
||||
# markers exist so the backfill phase has work to claim.
|
||||
if self._fts_cjk_loaded:
|
||||
with self._lock:
|
||||
self._ensure_fts_cjk_schema(self._conn)
|
||||
self._conn.commit()
|
||||
|
||||
def _emit(phase: str) -> None:
|
||||
if progress_cb is None:
|
||||
return
|
||||
st = self.fts_rebuild_status()
|
||||
if st is None:
|
||||
st = self.fts_cjk_rebuild_status()
|
||||
progress_cb({
|
||||
"phase": phase,
|
||||
"percent": st["percent"] if st else 100,
|
||||
@@ -2125,6 +2582,15 @@ class SessionDB:
|
||||
_pause(time.monotonic() - _t0)
|
||||
_emit("backfill")
|
||||
|
||||
# Phase 1b: backfill the CJK-bigram index (its own marker pair; a
|
||||
# no-op when the tokenizer isn't loadable or nothing is pending).
|
||||
while True:
|
||||
_t0 = time.monotonic()
|
||||
if not self.fts_cjk_rebuild_step():
|
||||
break
|
||||
_emit("backfill")
|
||||
_pause(time.monotonic() - _t0)
|
||||
|
||||
# Phase 2: tear down the demoted legacy shadow tables in chunks.
|
||||
_emit("teardown")
|
||||
while True:
|
||||
@@ -2687,6 +3153,9 @@ class SessionDB:
|
||||
cursor,
|
||||
include_trigram=trigram_enabled,
|
||||
)
|
||||
# CJK-bigram index (cjk_unicode61). Strictly additive to
|
||||
# the surfaces above and gated on the loadable tokenizer:
|
||||
self._ensure_fts_cjk_schema(cursor)
|
||||
|
||||
self._conn.commit()
|
||||
|
||||
@@ -6290,6 +6759,24 @@ class SessionDB:
|
||||
"""Count CJK characters in text."""
|
||||
return sum(1 for ch in text if cls._is_cjk_codepoint(ord(ch)))
|
||||
|
||||
@classmethod
|
||||
def _has_lone_cjk_run(cls, query: str) -> bool:
|
||||
"""True when any maximal CJK run in the query is a single char.
|
||||
|
||||
The cjk-bigram index stores bigrams for runs >=2 chars and unigrams
|
||||
only for isolated chars, so a 1-char CJK term can't match inside
|
||||
longer runs there — those queries keep the LIKE substring route.
|
||||
"""
|
||||
run = 0
|
||||
for ch in query:
|
||||
if cls._is_cjk_codepoint(ord(ch)):
|
||||
run += 1
|
||||
else:
|
||||
if run == 1:
|
||||
return True
|
||||
run = 0
|
||||
return run == 1
|
||||
|
||||
def search_messages(
|
||||
self,
|
||||
query: str,
|
||||
@@ -6300,6 +6787,76 @@ class SessionDB:
|
||||
offset: int = 0,
|
||||
sort: str = None,
|
||||
include_inactive: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Instrumented wrapper around :meth:`_search_messages_impl`.
|
||||
|
||||
Logs one line per slow search with the routing path taken, so
|
||||
production latency stays attributable per query shape (the 2026-07
|
||||
session_search investigation needed trace archaeology to discover
|
||||
the LIKE full scans; this makes the next regression a grep).
|
||||
Threshold: HERMES_SEARCH_SLOW_MS (default 1000; 0 logs every call).
|
||||
"""
|
||||
started = time.time()
|
||||
rows = None
|
||||
try:
|
||||
rows = self._search_messages_impl(
|
||||
query,
|
||||
source_filter=source_filter,
|
||||
exclude_sources=exclude_sources,
|
||||
role_filter=role_filter,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
sort=sort,
|
||||
include_inactive=include_inactive,
|
||||
)
|
||||
return rows
|
||||
finally:
|
||||
try:
|
||||
threshold = float(os.getenv("HERMES_SEARCH_SLOW_MS", "1000"))
|
||||
except (TypeError, ValueError):
|
||||
threshold = 1000.0
|
||||
elapsed_ms = (time.time() - started) * 1000.0
|
||||
if elapsed_ms >= threshold:
|
||||
logger.info(
|
||||
"slow session search: path=%s elapsed=%.0fms rows=%s query=%r",
|
||||
self._describe_search_path(query),
|
||||
elapsed_ms,
|
||||
len(rows) if rows is not None else "err",
|
||||
query[:200],
|
||||
)
|
||||
|
||||
def _describe_search_path(self, query: str) -> str:
|
||||
"""Best-effort name of the routing path a query takes (log-only)."""
|
||||
try:
|
||||
sanitized = self._sanitize_fts5_query(query or "")
|
||||
if not sanitized:
|
||||
return "empty"
|
||||
if not self._contains_cjk(sanitized):
|
||||
return "fts5"
|
||||
raw = sanitized.strip('"').strip()
|
||||
if self._fts_cjk_available and not self._has_lone_cjk_run(raw):
|
||||
return "fts_cjk"
|
||||
tokens = [
|
||||
t for t in raw.split()
|
||||
if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t)
|
||||
]
|
||||
short = any(self._count_cjk(t) < 3 for t in tokens)
|
||||
if self._count_cjk(raw) >= 3 and not short and self._trigram_available:
|
||||
return "trigram"
|
||||
return "like_scan"
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
def _search_messages_impl(
|
||||
self,
|
||||
query: str,
|
||||
source_filter: List[str] = None,
|
||||
exclude_sources: List[str] = None,
|
||||
role_filter: List[str] = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
sort: str = None,
|
||||
include_inactive: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Full-text search across session messages using FTS5.
|
||||
@@ -6437,8 +6994,104 @@ class SessionDB:
|
||||
# query explicitly filtering on role='tool' must therefore use
|
||||
# the LIKE fallback, which scans the base table directly.
|
||||
_wants_tool_rows = bool(role_filter) and "tool" in role_filter
|
||||
|
||||
# ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ──────
|
||||
# When the bigram index is available it serves EVERY CJK query
|
||||
# shape the legacy code split between trigram (>=3 chars/token)
|
||||
# and LIKE full scans (1-2 char tokens) — the whole point of the
|
||||
# index (PR #65544). Exceptions stay on the legacy routes:
|
||||
# - role_filter=['tool'] queries (tool rows aren't in the cjk
|
||||
# index, same exclusion as trigram),
|
||||
# - queries containing a LONE 1-char CJK run: the index stores
|
||||
# bigrams for runs >=2, so a single-char term can only match
|
||||
# isolated chars — LIKE substring semantics are broader.
|
||||
if (
|
||||
cjk_count >= 3
|
||||
self._fts_cjk_available
|
||||
and not _wants_tool_rows
|
||||
and not self._has_lone_cjk_run(raw_query)
|
||||
):
|
||||
tokens = raw_query.split()
|
||||
parts = []
|
||||
for tok in tokens:
|
||||
if tok.upper() in {"AND", "OR", "NOT"}:
|
||||
parts.append(tok)
|
||||
else:
|
||||
parts.append('"' + tok.replace('"', '""') + '"')
|
||||
cjk_query = " ".join(parts)
|
||||
cjk_where = ["messages_fts_cjk MATCH ?"]
|
||||
cjk_params: list = [cjk_query]
|
||||
if not include_inactive:
|
||||
cjk_where.append("(m.active = 1 OR m.compacted = 1)")
|
||||
if source_filter is not None:
|
||||
cjk_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})")
|
||||
cjk_params.extend(source_filter)
|
||||
if exclude_sources is not None:
|
||||
cjk_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})")
|
||||
cjk_params.extend(exclude_sources)
|
||||
if role_filter:
|
||||
cjk_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})")
|
||||
cjk_params.extend(role_filter)
|
||||
cjk_sql = f"""
|
||||
SELECT
|
||||
m.id,
|
||||
m.session_id,
|
||||
m.role,
|
||||
snippet(messages_fts_cjk, -1, '>>>', '<<<', '...', 40) AS snippet,
|
||||
m.content,
|
||||
m.timestamp,
|
||||
m.tool_name,
|
||||
s.source,
|
||||
s.model,
|
||||
s.started_at AS session_started
|
||||
FROM messages_fts_cjk
|
||||
JOIN messages m ON m.id = messages_fts_cjk.rowid
|
||||
JOIN sessions s ON s.id = m.session_id
|
||||
WHERE {' AND '.join(cjk_where)}
|
||||
{order_by_sql}
|
||||
LIMIT ? OFFSET ?
|
||||
"""
|
||||
cjk_params.extend([limit, offset])
|
||||
try:
|
||||
with self._lock:
|
||||
cjk_cursor = self._conn.execute(cjk_sql, cjk_params)
|
||||
matches = [dict(row) for row in cjk_cursor.fetchall()]
|
||||
_trigram_succeeded = True
|
||||
except sqlite3.OperationalError:
|
||||
# Tokenizer missing on this connection / query syntax —
|
||||
# the trigram + LIKE routes below still answer.
|
||||
logger.debug(
|
||||
"messages_fts_cjk query failed; falling back to "
|
||||
"trigram/LIKE", exc_info=True,
|
||||
)
|
||||
except sqlite3.DatabaseError as exc:
|
||||
# Same corruption class as the other FTS reads: rebuild
|
||||
# in place once and retry; on refusal/failure fall back.
|
||||
if self._try_runtime_fts_rebuild(exc):
|
||||
try:
|
||||
with self._lock:
|
||||
cjk_cursor = self._conn.execute(
|
||||
cjk_sql, cjk_params
|
||||
)
|
||||
matches = [
|
||||
dict(row) for row in cjk_cursor.fetchall()
|
||||
]
|
||||
_trigram_succeeded = True
|
||||
except sqlite3.DatabaseError:
|
||||
logger.warning(
|
||||
"CJK-bigram FTS search still failing after "
|
||||
"in-place rebuild; falling back to "
|
||||
"trigram/LIKE."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"CJK-bigram FTS search hit a corruption error "
|
||||
"(%s) and no in-place rebuild was possible; "
|
||||
"falling back to trigram/LIKE.", exc,
|
||||
)
|
||||
|
||||
if (
|
||||
not _trigram_succeeded
|
||||
and cjk_count >= 3
|
||||
and not _any_short_cjk
|
||||
and self._trigram_available
|
||||
and not _wants_tool_rows
|
||||
@@ -8534,9 +9187,10 @@ class SessionDB:
|
||||
# ── Space reclamation ──
|
||||
|
||||
# FTS5 virtual tables whose b-tree segments we merge on optimize. The
|
||||
# trigram table is created lazily / may be disabled, so we probe before
|
||||
# touching it (see optimize_fts).
|
||||
_FTS_TABLES = ("messages_fts", "messages_fts_trigram")
|
||||
# trigram table is created lazily / may be disabled, and the cjk-bigram
|
||||
# table only exists (and is only queryable) when the loadable tokenizer
|
||||
# is present — so we probe each before touching it (see optimize_fts).
|
||||
_FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
|
||||
|
||||
def _fts_table_exists(self, name: str) -> bool:
|
||||
"""True if an FTS5 virtual table is queryable in this DB."""
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# fts5_cjk — cjk_unicode61 FTS5 tokenizer
|
||||
|
||||
unicode61 + CJK character bigrams (Lucene CJKAnalyzer semantics). Fixes
|
||||
1-2 char Korean/Chinese/Japanese terms falling through to LIKE full-table
|
||||
scans in session search.
|
||||
|
||||
Build & install to `~/.hermes/lib/`:
|
||||
|
||||
./build.sh
|
||||
|
||||
Uses the system `sqlite3ext.h` when available, else the vendored copy in
|
||||
`vendor/` — no libsqlite3-dev required.
|
||||
|
||||
Once the extension is installed, the next `SessionDB` open creates the
|
||||
`messages_fts_cjk` index (external-content, tool rows excluded — same v23
|
||||
storage discipline as the other indexes). On a populated database, run
|
||||
|
||||
hermes sessions optimize-storage
|
||||
|
||||
to backfill it; new messages are indexed live either way. Set
|
||||
`sessions.cjk_fts: false` in `~/.hermes/config.yaml` to disable. Override
|
||||
the .so location with `HERMES_FTS5_CJK_SO`.
|
||||
|
||||
Contributed by Soju06 (PR #65544).
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# Build libfts5_cjk.so and install to ~/.hermes/lib/ (or $1).
|
||||
#
|
||||
# Uses the system sqlite3ext.h when present, else the vendored copy in
|
||||
# vendor/ (public-domain SQLite amalgamation headers) so the build works
|
||||
# without libsqlite3-dev installed.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
CFLAGS_EXTRA=""
|
||||
if ! echo '#include <sqlite3ext.h>' | gcc -E -xc - >/dev/null 2>&1; then
|
||||
CFLAGS_EXTRA="-Ivendor"
|
||||
fi
|
||||
|
||||
gcc -shared -fPIC -O2 -Wall -Wextra $CFLAGS_EXTRA fts5_cjk.c -o libfts5_cjk.so
|
||||
dest="${1:-$HOME/.hermes/lib}"
|
||||
mkdir -p "$dest"
|
||||
install -m 0644 libfts5_cjk.so "$dest/libfts5_cjk.so"
|
||||
echo "installed: $dest/libfts5_cjk.so"
|
||||
@@ -0,0 +1,252 @@
|
||||
/*
|
||||
** fts5_cjk.c — "cjk_unicode61" FTS5 tokenizer: unicode61 + CJK bigrams.
|
||||
**
|
||||
** Why: SQLite's unicode61 tokenizer treats a CJK run as ONE token
|
||||
** ("웅기가말했다" indexes as a single 6-char token), so a 2-char Korean
|
||||
** query can never match inside it. The stock trigram tokenizer fixes
|
||||
** substring search but needs >=3 chars per query term — 2-char Korean
|
||||
** words (일본, 구글, 우리, ...) fall through to a full-table LIKE scan,
|
||||
** measured at 3-6s per query on a 6.8GB messages table and the #1 driver
|
||||
** of hermes session_search latency.
|
||||
**
|
||||
** What: wrap unicode61. Every token it emits is re-examined; maximal CJK
|
||||
** runs inside the token are re-emitted as overlapping character BIGRAMS
|
||||
** (Lucene CJKAnalyzer semantics), non-CJK segments pass through unchanged.
|
||||
** A lone CJK char (run length 1) is emitted as a unigram. Because FTS5
|
||||
** turns consecutive tokens emitted from one query term into a phrase,
|
||||
** a query word like 캘린더 → [캘린][린더] gets exact substring semantics
|
||||
** with index-speed lookups, down to 2-char terms.
|
||||
**
|
||||
** Build: gcc -shared -fPIC -O2 fts5_cjk.c -o libfts5_cjk.so
|
||||
** Load: conn.load_extension(path) # default entrypoint sqlite3_ftscjk_init
|
||||
** Use: CREATE VIRTUAL TABLE t USING fts5(c, tokenize='cjk_unicode61');
|
||||
** Extra args pass through to unicode61:
|
||||
** tokenize='cjk_unicode61 remove_diacritics 2'
|
||||
*/
|
||||
#include <sqlite3ext.h>
|
||||
SQLITE_EXTENSION_INIT1
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/* ── CJK classification ──────────────────────────────────────────────── */
|
||||
|
||||
static int cjk_is_cjk(unsigned int cp) {
|
||||
return (cp >= 0xAC00 && cp <= 0xD7A3) /* Hangul syllables */
|
||||
|| (cp >= 0x1100 && cp <= 0x11FF) /* Hangul Jamo */
|
||||
|| (cp >= 0x3130 && cp <= 0x318F) /* Hangul compat Jamo */
|
||||
|| (cp >= 0xA960 && cp <= 0xA97F) /* Hangul Jamo ext-A */
|
||||
|| (cp >= 0xD7B0 && cp <= 0xD7FF) /* Hangul Jamo ext-B */
|
||||
|| (cp >= 0x4E00 && cp <= 0x9FFF) /* CJK unified ideographs */
|
||||
|| (cp >= 0x3400 && cp <= 0x4DBF) /* CJK ext A */
|
||||
|| (cp >= 0xF900 && cp <= 0xFAFF) /* CJK compat ideographs */
|
||||
|| (cp >= 0x20000 && cp <= 0x2FA1F) /* CJK ext B..F, compat sup*/
|
||||
|| (cp >= 0x3040 && cp <= 0x309F) /* Hiragana */
|
||||
|| (cp >= 0x30A0 && cp <= 0x30FF) /* Katakana */
|
||||
|| (cp >= 0x31F0 && cp <= 0x31FF); /* Katakana phonetic ext */
|
||||
}
|
||||
|
||||
/* Decode one UTF-8 codepoint at p (n bytes available). Returns byte len
|
||||
** consumed (>=1); stores codepoint in *pCp. Invalid bytes decode as
|
||||
** themselves so segmentation still terminates. */
|
||||
static int cjk_utf8_decode(const unsigned char *p, int n, unsigned int *pCp) {
|
||||
unsigned int c = p[0];
|
||||
if (c < 0x80) { *pCp = c; return 1; }
|
||||
if ((c & 0xE0) == 0xC0 && n >= 2) {
|
||||
*pCp = ((c & 0x1F) << 6) | (p[1] & 0x3F);
|
||||
return 2;
|
||||
}
|
||||
if ((c & 0xF0) == 0xE0 && n >= 3) {
|
||||
*pCp = ((c & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F);
|
||||
return 3;
|
||||
}
|
||||
if ((c & 0xF8) == 0xF0 && n >= 4) {
|
||||
*pCp = ((c & 0x07) << 18) | ((p[1] & 0x3F) << 12) |
|
||||
((p[2] & 0x3F) << 6) | (p[3] & 0x3F);
|
||||
return 4;
|
||||
}
|
||||
*pCp = c;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ── tokenizer plumbing ──────────────────────────────────────────────── */
|
||||
|
||||
typedef struct CjkTokenizer CjkTokenizer;
|
||||
struct CjkTokenizer {
|
||||
fts5_tokenizer inner; /* unicode61 methods */
|
||||
Fts5Tokenizer *pInner; /* unicode61 instance */
|
||||
};
|
||||
|
||||
typedef struct CjkCallbackCtx CjkCallbackCtx;
|
||||
struct CjkCallbackCtx {
|
||||
void *pOuterCtx;
|
||||
int (*xOuterToken)(void*, int, const char*, int, int, int);
|
||||
};
|
||||
|
||||
/* Re-emit one unicode61 token, splitting CJK runs into bigrams.
|
||||
**
|
||||
** Offsets: unicode61 reports [iStart,iEnd) into the ORIGINAL text. For
|
||||
** CJK bytes unicode61's folding is the identity, and ASCII case folding
|
||||
** preserves byte length, so mapping sub-token offsets by byte position
|
||||
** within the token is exact for CJK and correct-length for ASCII. For
|
||||
** rare length-changing folds (accented latin) the highlight offsets can
|
||||
** drift by a few bytes inside that token; matching is unaffected. Every
|
||||
** emitted offset is clamped to [iStart,iEnd).
|
||||
*/
|
||||
static int cjk_emit(CjkCallbackCtx *p, int tflags,
|
||||
const char *pToken, int nToken, int iStart, int iEnd) {
|
||||
const unsigned char *z = (const unsigned char*)pToken;
|
||||
int i = 0;
|
||||
int rc = SQLITE_OK;
|
||||
|
||||
/* Fast path: no CJK anywhere → pass through untouched. */
|
||||
int hasCjk = 0;
|
||||
while (i < nToken) {
|
||||
unsigned int cp;
|
||||
i += cjk_utf8_decode(z + i, nToken - i, &cp);
|
||||
if (cjk_is_cjk(cp)) { hasCjk = 1; break; }
|
||||
}
|
||||
if (!hasCjk) {
|
||||
return p->xOuterToken(p->pOuterCtx, tflags, pToken, nToken, iStart, iEnd);
|
||||
}
|
||||
|
||||
#define CJK_CLAMP_END(v) ((iStart + (v)) > iEnd ? iEnd : (iStart + (v)))
|
||||
i = 0;
|
||||
while (i < nToken && rc == SQLITE_OK) {
|
||||
unsigned int cp;
|
||||
int segStart = i;
|
||||
int len = cjk_utf8_decode(z + i, nToken - i, &cp);
|
||||
if (!cjk_is_cjk(cp)) {
|
||||
/* non-CJK segment: extend to the next CJK char (or end) */
|
||||
i += len;
|
||||
while (i < nToken) {
|
||||
int l2 = cjk_utf8_decode(z + i, nToken - i, &cp);
|
||||
if (cjk_is_cjk(cp)) break;
|
||||
i += l2;
|
||||
}
|
||||
rc = p->xOuterToken(p->pOuterCtx, tflags,
|
||||
pToken + segStart, i - segStart,
|
||||
CJK_CLAMP_END(segStart), CJK_CLAMP_END(i));
|
||||
} else {
|
||||
/* CJK run: collect char byte-boundaries, emit bigrams. */
|
||||
int bounds[3]; /* rolling window: start, mid, end */
|
||||
bounds[0] = segStart;
|
||||
bounds[1] = segStart + len;
|
||||
i += len;
|
||||
int nChars = 1;
|
||||
while (i < nToken) {
|
||||
int l2 = cjk_utf8_decode(z + i, nToken - i, &cp);
|
||||
if (!cjk_is_cjk(cp)) break;
|
||||
i += l2;
|
||||
nChars++;
|
||||
if (nChars >= 2) {
|
||||
bounds[2] = i;
|
||||
if (nChars == 2) {
|
||||
/* first bigram spans bounds[0]..bounds[2] */
|
||||
}
|
||||
rc = p->xOuterToken(p->pOuterCtx, tflags,
|
||||
pToken + bounds[0], bounds[2] - bounds[0],
|
||||
CJK_CLAMP_END(bounds[0]), CJK_CLAMP_END(bounds[2]));
|
||||
if (rc != SQLITE_OK) break;
|
||||
bounds[0] = bounds[1];
|
||||
bounds[1] = bounds[2];
|
||||
}
|
||||
}
|
||||
if (rc == SQLITE_OK && nChars == 1) {
|
||||
/* lone CJK char: emit as unigram */
|
||||
rc = p->xOuterToken(p->pOuterCtx, tflags,
|
||||
pToken + segStart, bounds[1] - segStart,
|
||||
CJK_CLAMP_END(segStart), CJK_CLAMP_END(bounds[1]));
|
||||
}
|
||||
}
|
||||
}
|
||||
#undef CJK_CLAMP_END
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int cjkInnerCallback(void *pCtx, int tflags,
|
||||
const char *pToken, int nToken,
|
||||
int iStart, int iEnd) {
|
||||
return cjk_emit((CjkCallbackCtx*)pCtx, tflags, pToken, nToken, iStart, iEnd);
|
||||
}
|
||||
|
||||
static int cjkCreate(void *pApiCtx, const char **azArg, int nArg,
|
||||
Fts5Tokenizer **ppOut) {
|
||||
fts5_api *pApi = (fts5_api*)pApiCtx;
|
||||
CjkTokenizer *p;
|
||||
void *pInnerCtx = 0;
|
||||
int rc;
|
||||
|
||||
p = (CjkTokenizer*)sqlite3_malloc(sizeof(CjkTokenizer));
|
||||
if (!p) return SQLITE_NOMEM;
|
||||
memset(p, 0, sizeof(*p));
|
||||
|
||||
rc = pApi->xFindTokenizer(pApi, "unicode61", &pInnerCtx, &p->inner);
|
||||
if (rc == SQLITE_OK) {
|
||||
rc = p->inner.xCreate(pInnerCtx, azArg, nArg, &p->pInner);
|
||||
}
|
||||
if (rc != SQLITE_OK) {
|
||||
sqlite3_free(p);
|
||||
return rc;
|
||||
}
|
||||
*ppOut = (Fts5Tokenizer*)p;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static void cjkDelete(Fts5Tokenizer *pTok) {
|
||||
CjkTokenizer *p = (CjkTokenizer*)pTok;
|
||||
if (p) {
|
||||
if (p->pInner) p->inner.xDelete(p->pInner);
|
||||
sqlite3_free(p);
|
||||
}
|
||||
}
|
||||
|
||||
static int cjkTokenize(Fts5Tokenizer *pTok, void *pCtx, int flags,
|
||||
const char *pText, int nText,
|
||||
int (*xToken)(void*, int, const char*, int, int, int)) {
|
||||
CjkTokenizer *p = (CjkTokenizer*)pTok;
|
||||
CjkCallbackCtx cb;
|
||||
cb.pOuterCtx = pCtx;
|
||||
cb.xOuterToken = xToken;
|
||||
return p->inner.xTokenize(p->pInner, &cb, flags, pText, nText,
|
||||
cjkInnerCallback);
|
||||
}
|
||||
|
||||
/* ── registration ────────────────────────────────────────────────────── */
|
||||
|
||||
static fts5_api *cjkFts5Api(sqlite3 *db) {
|
||||
fts5_api *pRet = 0;
|
||||
sqlite3_stmt *pStmt = 0;
|
||||
if (sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0) == SQLITE_OK) {
|
||||
sqlite3_bind_pointer(pStmt, 1, (void*)&pRet, "fts5_api_ptr", 0);
|
||||
sqlite3_step(pStmt);
|
||||
}
|
||||
sqlite3_finalize(pStmt);
|
||||
return pRet;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
__declspec(dllexport)
|
||||
#endif
|
||||
int sqlite3_ftscjk_init(sqlite3 *db, char **pzErrMsg,
|
||||
const sqlite3_api_routines *pApi) {
|
||||
fts5_api *pFts;
|
||||
static fts5_tokenizer tok = { cjkCreate, cjkDelete, cjkTokenize };
|
||||
SQLITE_EXTENSION_INIT2(pApi);
|
||||
(void)pzErrMsg;
|
||||
pFts = cjkFts5Api(db);
|
||||
if (!pFts) {
|
||||
if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("fts5_cjk: FTS5 unavailable");
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
return pFts->xCreateTokenizer(pFts, "cjk_unicode61", (void*)pFts, &tok, 0);
|
||||
}
|
||||
|
||||
/* Alias for callers that spell out the underscored basename. */
|
||||
#ifdef _WIN32
|
||||
__declspec(dllexport)
|
||||
#endif
|
||||
int sqlite3_fts5_cjk_init(sqlite3 *db, char **pzErrMsg,
|
||||
const sqlite3_api_routines *pApi) {
|
||||
return sqlite3_ftscjk_init(db, pzErrMsg, pApi);
|
||||
}
|
||||
Vendored
+13775
File diff suppressed because it is too large
Load Diff
Vendored
+723
@@ -0,0 +1,723 @@
|
||||
/*
|
||||
** 2006 June 7
|
||||
**
|
||||
** The author disclaims copyright to this source code. In place of
|
||||
** a legal notice, here is a blessing:
|
||||
**
|
||||
** May you do good and not evil.
|
||||
** May you find forgiveness for yourself and forgive others.
|
||||
** May you share freely, never taking more than you give.
|
||||
**
|
||||
*************************************************************************
|
||||
** This header file defines the SQLite interface for use by
|
||||
** shared libraries that want to be imported as extensions into
|
||||
** an SQLite instance. Shared libraries that intend to be loaded
|
||||
** as extensions by SQLite should #include this file instead of
|
||||
** sqlite3.h.
|
||||
*/
|
||||
#ifndef SQLITE3EXT_H
|
||||
#define SQLITE3EXT_H
|
||||
#include "sqlite3.h"
|
||||
|
||||
/*
|
||||
** The following structure holds pointers to all of the SQLite API
|
||||
** routines.
|
||||
**
|
||||
** WARNING: In order to maintain backwards compatibility, add new
|
||||
** interfaces to the end of this structure only. If you insert new
|
||||
** interfaces in the middle of this structure, then older different
|
||||
** versions of SQLite will not be able to load each other's shared
|
||||
** libraries!
|
||||
*/
|
||||
struct sqlite3_api_routines {
|
||||
void * (*aggregate_context)(sqlite3_context*,int nBytes);
|
||||
int (*aggregate_count)(sqlite3_context*);
|
||||
int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*));
|
||||
int (*bind_double)(sqlite3_stmt*,int,double);
|
||||
int (*bind_int)(sqlite3_stmt*,int,int);
|
||||
int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64);
|
||||
int (*bind_null)(sqlite3_stmt*,int);
|
||||
int (*bind_parameter_count)(sqlite3_stmt*);
|
||||
int (*bind_parameter_index)(sqlite3_stmt*,const char*zName);
|
||||
const char * (*bind_parameter_name)(sqlite3_stmt*,int);
|
||||
int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*));
|
||||
int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*));
|
||||
int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*);
|
||||
int (*busy_handler)(sqlite3*,int(*)(void*,int),void*);
|
||||
int (*busy_timeout)(sqlite3*,int ms);
|
||||
int (*changes)(sqlite3*);
|
||||
int (*close)(sqlite3*);
|
||||
int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,
|
||||
int eTextRep,const char*));
|
||||
int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,
|
||||
int eTextRep,const void*));
|
||||
const void * (*column_blob)(sqlite3_stmt*,int iCol);
|
||||
int (*column_bytes)(sqlite3_stmt*,int iCol);
|
||||
int (*column_bytes16)(sqlite3_stmt*,int iCol);
|
||||
int (*column_count)(sqlite3_stmt*pStmt);
|
||||
const char * (*column_database_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_database_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_decltype)(sqlite3_stmt*,int i);
|
||||
const void * (*column_decltype16)(sqlite3_stmt*,int);
|
||||
double (*column_double)(sqlite3_stmt*,int iCol);
|
||||
int (*column_int)(sqlite3_stmt*,int iCol);
|
||||
sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol);
|
||||
const char * (*column_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_origin_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_origin_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_table_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_table_name16)(sqlite3_stmt*,int);
|
||||
const unsigned char * (*column_text)(sqlite3_stmt*,int iCol);
|
||||
const void * (*column_text16)(sqlite3_stmt*,int iCol);
|
||||
int (*column_type)(sqlite3_stmt*,int iCol);
|
||||
sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol);
|
||||
void * (*commit_hook)(sqlite3*,int(*)(void*),void*);
|
||||
int (*complete)(const char*sql);
|
||||
int (*complete16)(const void*sql);
|
||||
int (*create_collation)(sqlite3*,const char*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*));
|
||||
int (*create_collation16)(sqlite3*,const void*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*));
|
||||
int (*create_function)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*));
|
||||
int (*create_function16)(sqlite3*,const void*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*));
|
||||
int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*);
|
||||
int (*data_count)(sqlite3_stmt*pStmt);
|
||||
sqlite3 * (*db_handle)(sqlite3_stmt*);
|
||||
int (*declare_vtab)(sqlite3*,const char*);
|
||||
int (*enable_shared_cache)(int);
|
||||
int (*errcode)(sqlite3*db);
|
||||
const char * (*errmsg)(sqlite3*);
|
||||
const void * (*errmsg16)(sqlite3*);
|
||||
int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**);
|
||||
int (*expired)(sqlite3_stmt*);
|
||||
int (*finalize)(sqlite3_stmt*pStmt);
|
||||
void (*free)(void*);
|
||||
void (*free_table)(char**result);
|
||||
int (*get_autocommit)(sqlite3*);
|
||||
void * (*get_auxdata)(sqlite3_context*,int);
|
||||
int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**);
|
||||
int (*global_recover)(void);
|
||||
void (*interruptx)(sqlite3*);
|
||||
sqlite_int64 (*last_insert_rowid)(sqlite3*);
|
||||
const char * (*libversion)(void);
|
||||
int (*libversion_number)(void);
|
||||
void *(*malloc)(int);
|
||||
char * (*mprintf)(const char*,...);
|
||||
int (*open)(const char*,sqlite3**);
|
||||
int (*open16)(const void*,sqlite3**);
|
||||
int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
|
||||
int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
|
||||
void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*);
|
||||
void (*progress_handler)(sqlite3*,int,int(*)(void*),void*);
|
||||
void *(*realloc)(void*,int);
|
||||
int (*reset)(sqlite3_stmt*pStmt);
|
||||
void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_double)(sqlite3_context*,double);
|
||||
void (*result_error)(sqlite3_context*,const char*,int);
|
||||
void (*result_error16)(sqlite3_context*,const void*,int);
|
||||
void (*result_int)(sqlite3_context*,int);
|
||||
void (*result_int64)(sqlite3_context*,sqlite_int64);
|
||||
void (*result_null)(sqlite3_context*);
|
||||
void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*));
|
||||
void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_value)(sqlite3_context*,sqlite3_value*);
|
||||
void * (*rollback_hook)(sqlite3*,void(*)(void*),void*);
|
||||
int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
|
||||
const char*,const char*),void*);
|
||||
void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
|
||||
char * (*xsnprintf)(int,char*,const char*,...);
|
||||
int (*step)(sqlite3_stmt*);
|
||||
int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
|
||||
char const**,char const**,int*,int*,int*);
|
||||
void (*thread_cleanup)(void);
|
||||
int (*total_changes)(sqlite3*);
|
||||
void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*);
|
||||
int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*);
|
||||
void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,
|
||||
sqlite_int64),void*);
|
||||
void * (*user_data)(sqlite3_context*);
|
||||
const void * (*value_blob)(sqlite3_value*);
|
||||
int (*value_bytes)(sqlite3_value*);
|
||||
int (*value_bytes16)(sqlite3_value*);
|
||||
double (*value_double)(sqlite3_value*);
|
||||
int (*value_int)(sqlite3_value*);
|
||||
sqlite_int64 (*value_int64)(sqlite3_value*);
|
||||
int (*value_numeric_type)(sqlite3_value*);
|
||||
const unsigned char * (*value_text)(sqlite3_value*);
|
||||
const void * (*value_text16)(sqlite3_value*);
|
||||
const void * (*value_text16be)(sqlite3_value*);
|
||||
const void * (*value_text16le)(sqlite3_value*);
|
||||
int (*value_type)(sqlite3_value*);
|
||||
char *(*vmprintf)(const char*,va_list);
|
||||
/* Added ??? */
|
||||
int (*overload_function)(sqlite3*, const char *zFuncName, int nArg);
|
||||
/* Added by 3.3.13 */
|
||||
int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
|
||||
int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
|
||||
int (*clear_bindings)(sqlite3_stmt*);
|
||||
/* Added by 3.4.1 */
|
||||
int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,
|
||||
void (*xDestroy)(void *));
|
||||
/* Added by 3.5.0 */
|
||||
int (*bind_zeroblob)(sqlite3_stmt*,int,int);
|
||||
int (*blob_bytes)(sqlite3_blob*);
|
||||
int (*blob_close)(sqlite3_blob*);
|
||||
int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,
|
||||
int,sqlite3_blob**);
|
||||
int (*blob_read)(sqlite3_blob*,void*,int,int);
|
||||
int (*blob_write)(sqlite3_blob*,const void*,int,int);
|
||||
int (*create_collation_v2)(sqlite3*,const char*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*),
|
||||
void(*)(void*));
|
||||
int (*file_control)(sqlite3*,const char*,int,void*);
|
||||
sqlite3_int64 (*memory_highwater)(int);
|
||||
sqlite3_int64 (*memory_used)(void);
|
||||
sqlite3_mutex *(*mutex_alloc)(int);
|
||||
void (*mutex_enter)(sqlite3_mutex*);
|
||||
void (*mutex_free)(sqlite3_mutex*);
|
||||
void (*mutex_leave)(sqlite3_mutex*);
|
||||
int (*mutex_try)(sqlite3_mutex*);
|
||||
int (*open_v2)(const char*,sqlite3**,int,const char*);
|
||||
int (*release_memory)(int);
|
||||
void (*result_error_nomem)(sqlite3_context*);
|
||||
void (*result_error_toobig)(sqlite3_context*);
|
||||
int (*sleep)(int);
|
||||
void (*soft_heap_limit)(int);
|
||||
sqlite3_vfs *(*vfs_find)(const char*);
|
||||
int (*vfs_register)(sqlite3_vfs*,int);
|
||||
int (*vfs_unregister)(sqlite3_vfs*);
|
||||
int (*xthreadsafe)(void);
|
||||
void (*result_zeroblob)(sqlite3_context*,int);
|
||||
void (*result_error_code)(sqlite3_context*,int);
|
||||
int (*test_control)(int, ...);
|
||||
void (*randomness)(int,void*);
|
||||
sqlite3 *(*context_db_handle)(sqlite3_context*);
|
||||
int (*extended_result_codes)(sqlite3*,int);
|
||||
int (*limit)(sqlite3*,int,int);
|
||||
sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*);
|
||||
const char *(*sql)(sqlite3_stmt*);
|
||||
int (*status)(int,int*,int*,int);
|
||||
int (*backup_finish)(sqlite3_backup*);
|
||||
sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*);
|
||||
int (*backup_pagecount)(sqlite3_backup*);
|
||||
int (*backup_remaining)(sqlite3_backup*);
|
||||
int (*backup_step)(sqlite3_backup*,int);
|
||||
const char *(*compileoption_get)(int);
|
||||
int (*compileoption_used)(const char*);
|
||||
int (*create_function_v2)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*),
|
||||
void(*xDestroy)(void*));
|
||||
int (*db_config)(sqlite3*,int,...);
|
||||
sqlite3_mutex *(*db_mutex)(sqlite3*);
|
||||
int (*db_status)(sqlite3*,int,int*,int*,int);
|
||||
int (*extended_errcode)(sqlite3*);
|
||||
void (*log)(int,const char*,...);
|
||||
sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64);
|
||||
const char *(*sourceid)(void);
|
||||
int (*stmt_status)(sqlite3_stmt*,int,int);
|
||||
int (*strnicmp)(const char*,const char*,int);
|
||||
int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
|
||||
int (*wal_autocheckpoint)(sqlite3*,int);
|
||||
int (*wal_checkpoint)(sqlite3*,const char*);
|
||||
void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
|
||||
int (*blob_reopen)(sqlite3_blob*,sqlite3_int64);
|
||||
int (*vtab_config)(sqlite3*,int op,...);
|
||||
int (*vtab_on_conflict)(sqlite3*);
|
||||
/* Version 3.7.16 and later */
|
||||
int (*close_v2)(sqlite3*);
|
||||
const char *(*db_filename)(sqlite3*,const char*);
|
||||
int (*db_readonly)(sqlite3*,const char*);
|
||||
int (*db_release_memory)(sqlite3*);
|
||||
const char *(*errstr)(int);
|
||||
int (*stmt_busy)(sqlite3_stmt*);
|
||||
int (*stmt_readonly)(sqlite3_stmt*);
|
||||
int (*stricmp)(const char*,const char*);
|
||||
int (*uri_boolean)(const char*,const char*,int);
|
||||
sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64);
|
||||
const char *(*uri_parameter)(const char*,const char*);
|
||||
char *(*xvsnprintf)(int,char*,const char*,va_list);
|
||||
int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*);
|
||||
/* Version 3.8.7 and later */
|
||||
int (*auto_extension)(void(*)(void));
|
||||
int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64,
|
||||
void(*)(void*));
|
||||
int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64,
|
||||
void(*)(void*),unsigned char);
|
||||
int (*cancel_auto_extension)(void(*)(void));
|
||||
int (*load_extension)(sqlite3*,const char*,const char*,char**);
|
||||
void *(*malloc64)(sqlite3_uint64);
|
||||
sqlite3_uint64 (*msize)(void*);
|
||||
void *(*realloc64)(void*,sqlite3_uint64);
|
||||
void (*reset_auto_extension)(void);
|
||||
void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64,
|
||||
void(*)(void*));
|
||||
void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64,
|
||||
void(*)(void*), unsigned char);
|
||||
int (*strglob)(const char*,const char*);
|
||||
/* Version 3.8.11 and later */
|
||||
sqlite3_value *(*value_dup)(const sqlite3_value*);
|
||||
void (*value_free)(sqlite3_value*);
|
||||
int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64);
|
||||
int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64);
|
||||
/* Version 3.9.0 and later */
|
||||
unsigned int (*value_subtype)(sqlite3_value*);
|
||||
void (*result_subtype)(sqlite3_context*,unsigned int);
|
||||
/* Version 3.10.0 and later */
|
||||
int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int);
|
||||
int (*strlike)(const char*,const char*,unsigned int);
|
||||
int (*db_cacheflush)(sqlite3*);
|
||||
/* Version 3.12.0 and later */
|
||||
int (*system_errno)(sqlite3*);
|
||||
/* Version 3.14.0 and later */
|
||||
int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*);
|
||||
char *(*expanded_sql)(sqlite3_stmt*);
|
||||
/* Version 3.18.0 and later */
|
||||
void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64);
|
||||
/* Version 3.20.0 and later */
|
||||
int (*prepare_v3)(sqlite3*,const char*,int,unsigned int,
|
||||
sqlite3_stmt**,const char**);
|
||||
int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int,
|
||||
sqlite3_stmt**,const void**);
|
||||
int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*));
|
||||
void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*));
|
||||
void *(*value_pointer)(sqlite3_value*,const char*);
|
||||
int (*vtab_nochange)(sqlite3_context*);
|
||||
int (*value_nochange)(sqlite3_value*);
|
||||
const char *(*vtab_collation)(sqlite3_index_info*,int);
|
||||
/* Version 3.24.0 and later */
|
||||
int (*keyword_count)(void);
|
||||
int (*keyword_name)(int,const char**,int*);
|
||||
int (*keyword_check)(const char*,int);
|
||||
sqlite3_str *(*str_new)(sqlite3*);
|
||||
char *(*str_finish)(sqlite3_str*);
|
||||
void (*str_appendf)(sqlite3_str*, const char *zFormat, ...);
|
||||
void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list);
|
||||
void (*str_append)(sqlite3_str*, const char *zIn, int N);
|
||||
void (*str_appendall)(sqlite3_str*, const char *zIn);
|
||||
void (*str_appendchar)(sqlite3_str*, int N, char C);
|
||||
void (*str_reset)(sqlite3_str*);
|
||||
int (*str_errcode)(sqlite3_str*);
|
||||
int (*str_length)(sqlite3_str*);
|
||||
char *(*str_value)(sqlite3_str*);
|
||||
/* Version 3.25.0 and later */
|
||||
int (*create_window_function)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*),
|
||||
void (*xValue)(sqlite3_context*),
|
||||
void (*xInv)(sqlite3_context*,int,sqlite3_value**),
|
||||
void(*xDestroy)(void*));
|
||||
/* Version 3.26.0 and later */
|
||||
const char *(*normalized_sql)(sqlite3_stmt*);
|
||||
/* Version 3.28.0 and later */
|
||||
int (*stmt_isexplain)(sqlite3_stmt*);
|
||||
int (*value_frombind)(sqlite3_value*);
|
||||
/* Version 3.30.0 and later */
|
||||
int (*drop_modules)(sqlite3*,const char**);
|
||||
/* Version 3.31.0 and later */
|
||||
sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64);
|
||||
const char *(*uri_key)(const char*,int);
|
||||
const char *(*filename_database)(const char*);
|
||||
const char *(*filename_journal)(const char*);
|
||||
const char *(*filename_wal)(const char*);
|
||||
/* Version 3.32.0 and later */
|
||||
const char *(*create_filename)(const char*,const char*,const char*,
|
||||
int,const char**);
|
||||
void (*free_filename)(const char*);
|
||||
sqlite3_file *(*database_file_object)(const char*);
|
||||
/* Version 3.34.0 and later */
|
||||
int (*txn_state)(sqlite3*,const char*);
|
||||
/* Version 3.36.1 and later */
|
||||
sqlite3_int64 (*changes64)(sqlite3*);
|
||||
sqlite3_int64 (*total_changes64)(sqlite3*);
|
||||
/* Version 3.37.0 and later */
|
||||
int (*autovacuum_pages)(sqlite3*,
|
||||
unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int),
|
||||
void*, void(*)(void*));
|
||||
/* Version 3.38.0 and later */
|
||||
int (*error_offset)(sqlite3*);
|
||||
int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**);
|
||||
int (*vtab_distinct)(sqlite3_index_info*);
|
||||
int (*vtab_in)(sqlite3_index_info*,int,int);
|
||||
int (*vtab_in_first)(sqlite3_value*,sqlite3_value**);
|
||||
int (*vtab_in_next)(sqlite3_value*,sqlite3_value**);
|
||||
/* Version 3.39.0 and later */
|
||||
int (*deserialize)(sqlite3*,const char*,unsigned char*,
|
||||
sqlite3_int64,sqlite3_int64,unsigned);
|
||||
unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*,
|
||||
unsigned int);
|
||||
const char *(*db_name)(sqlite3*,int);
|
||||
/* Version 3.40.0 and later */
|
||||
int (*value_encoding)(sqlite3_value*);
|
||||
/* Version 3.41.0 and later */
|
||||
int (*is_interrupted)(sqlite3*);
|
||||
/* Version 3.43.0 and later */
|
||||
int (*stmt_explain)(sqlite3_stmt*,int);
|
||||
/* Version 3.44.0 and later */
|
||||
void *(*get_clientdata)(sqlite3*,const char*);
|
||||
int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*));
|
||||
/* Version 3.50.0 and later */
|
||||
int (*setlk_timeout)(sqlite3*,int,int);
|
||||
};
|
||||
|
||||
/*
|
||||
** This is the function signature used for all extension entry points. It
|
||||
** is also defined in the file "loadext.c".
|
||||
*/
|
||||
typedef int (*sqlite3_loadext_entry)(
|
||||
sqlite3 *db, /* Handle to the database. */
|
||||
char **pzErrMsg, /* Used to set error string on failure. */
|
||||
const sqlite3_api_routines *pThunk /* Extension API function pointers. */
|
||||
);
|
||||
|
||||
/*
|
||||
** The following macros redefine the API routines so that they are
|
||||
** redirected through the global sqlite3_api structure.
|
||||
**
|
||||
** This header file is also used by the loadext.c source file
|
||||
** (part of the main SQLite library - not an extension) so that
|
||||
** it can get access to the sqlite3_api_routines structure
|
||||
** definition. But the main library does not want to redefine
|
||||
** the API. So the redefinition macros are only valid if the
|
||||
** SQLITE_CORE macros is undefined.
|
||||
*/
|
||||
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
|
||||
#define sqlite3_aggregate_context sqlite3_api->aggregate_context
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_aggregate_count sqlite3_api->aggregate_count
|
||||
#endif
|
||||
#define sqlite3_bind_blob sqlite3_api->bind_blob
|
||||
#define sqlite3_bind_double sqlite3_api->bind_double
|
||||
#define sqlite3_bind_int sqlite3_api->bind_int
|
||||
#define sqlite3_bind_int64 sqlite3_api->bind_int64
|
||||
#define sqlite3_bind_null sqlite3_api->bind_null
|
||||
#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count
|
||||
#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index
|
||||
#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name
|
||||
#define sqlite3_bind_text sqlite3_api->bind_text
|
||||
#define sqlite3_bind_text16 sqlite3_api->bind_text16
|
||||
#define sqlite3_bind_value sqlite3_api->bind_value
|
||||
#define sqlite3_busy_handler sqlite3_api->busy_handler
|
||||
#define sqlite3_busy_timeout sqlite3_api->busy_timeout
|
||||
#define sqlite3_changes sqlite3_api->changes
|
||||
#define sqlite3_close sqlite3_api->close
|
||||
#define sqlite3_collation_needed sqlite3_api->collation_needed
|
||||
#define sqlite3_collation_needed16 sqlite3_api->collation_needed16
|
||||
#define sqlite3_column_blob sqlite3_api->column_blob
|
||||
#define sqlite3_column_bytes sqlite3_api->column_bytes
|
||||
#define sqlite3_column_bytes16 sqlite3_api->column_bytes16
|
||||
#define sqlite3_column_count sqlite3_api->column_count
|
||||
#define sqlite3_column_database_name sqlite3_api->column_database_name
|
||||
#define sqlite3_column_database_name16 sqlite3_api->column_database_name16
|
||||
#define sqlite3_column_decltype sqlite3_api->column_decltype
|
||||
#define sqlite3_column_decltype16 sqlite3_api->column_decltype16
|
||||
#define sqlite3_column_double sqlite3_api->column_double
|
||||
#define sqlite3_column_int sqlite3_api->column_int
|
||||
#define sqlite3_column_int64 sqlite3_api->column_int64
|
||||
#define sqlite3_column_name sqlite3_api->column_name
|
||||
#define sqlite3_column_name16 sqlite3_api->column_name16
|
||||
#define sqlite3_column_origin_name sqlite3_api->column_origin_name
|
||||
#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16
|
||||
#define sqlite3_column_table_name sqlite3_api->column_table_name
|
||||
#define sqlite3_column_table_name16 sqlite3_api->column_table_name16
|
||||
#define sqlite3_column_text sqlite3_api->column_text
|
||||
#define sqlite3_column_text16 sqlite3_api->column_text16
|
||||
#define sqlite3_column_type sqlite3_api->column_type
|
||||
#define sqlite3_column_value sqlite3_api->column_value
|
||||
#define sqlite3_commit_hook sqlite3_api->commit_hook
|
||||
#define sqlite3_complete sqlite3_api->complete
|
||||
#define sqlite3_complete16 sqlite3_api->complete16
|
||||
#define sqlite3_create_collation sqlite3_api->create_collation
|
||||
#define sqlite3_create_collation16 sqlite3_api->create_collation16
|
||||
#define sqlite3_create_function sqlite3_api->create_function
|
||||
#define sqlite3_create_function16 sqlite3_api->create_function16
|
||||
#define sqlite3_create_module sqlite3_api->create_module
|
||||
#define sqlite3_create_module_v2 sqlite3_api->create_module_v2
|
||||
#define sqlite3_data_count sqlite3_api->data_count
|
||||
#define sqlite3_db_handle sqlite3_api->db_handle
|
||||
#define sqlite3_declare_vtab sqlite3_api->declare_vtab
|
||||
#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache
|
||||
#define sqlite3_errcode sqlite3_api->errcode
|
||||
#define sqlite3_errmsg sqlite3_api->errmsg
|
||||
#define sqlite3_errmsg16 sqlite3_api->errmsg16
|
||||
#define sqlite3_exec sqlite3_api->exec
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_expired sqlite3_api->expired
|
||||
#endif
|
||||
#define sqlite3_finalize sqlite3_api->finalize
|
||||
#define sqlite3_free sqlite3_api->free
|
||||
#define sqlite3_free_table sqlite3_api->free_table
|
||||
#define sqlite3_get_autocommit sqlite3_api->get_autocommit
|
||||
#define sqlite3_get_auxdata sqlite3_api->get_auxdata
|
||||
#define sqlite3_get_table sqlite3_api->get_table
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_global_recover sqlite3_api->global_recover
|
||||
#endif
|
||||
#define sqlite3_interrupt sqlite3_api->interruptx
|
||||
#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid
|
||||
#define sqlite3_libversion sqlite3_api->libversion
|
||||
#define sqlite3_libversion_number sqlite3_api->libversion_number
|
||||
#define sqlite3_malloc sqlite3_api->malloc
|
||||
#define sqlite3_mprintf sqlite3_api->mprintf
|
||||
#define sqlite3_open sqlite3_api->open
|
||||
#define sqlite3_open16 sqlite3_api->open16
|
||||
#define sqlite3_prepare sqlite3_api->prepare
|
||||
#define sqlite3_prepare16 sqlite3_api->prepare16
|
||||
#define sqlite3_prepare_v2 sqlite3_api->prepare_v2
|
||||
#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2
|
||||
#define sqlite3_profile sqlite3_api->profile
|
||||
#define sqlite3_progress_handler sqlite3_api->progress_handler
|
||||
#define sqlite3_realloc sqlite3_api->realloc
|
||||
#define sqlite3_reset sqlite3_api->reset
|
||||
#define sqlite3_result_blob sqlite3_api->result_blob
|
||||
#define sqlite3_result_double sqlite3_api->result_double
|
||||
#define sqlite3_result_error sqlite3_api->result_error
|
||||
#define sqlite3_result_error16 sqlite3_api->result_error16
|
||||
#define sqlite3_result_int sqlite3_api->result_int
|
||||
#define sqlite3_result_int64 sqlite3_api->result_int64
|
||||
#define sqlite3_result_null sqlite3_api->result_null
|
||||
#define sqlite3_result_text sqlite3_api->result_text
|
||||
#define sqlite3_result_text16 sqlite3_api->result_text16
|
||||
#define sqlite3_result_text16be sqlite3_api->result_text16be
|
||||
#define sqlite3_result_text16le sqlite3_api->result_text16le
|
||||
#define sqlite3_result_value sqlite3_api->result_value
|
||||
#define sqlite3_rollback_hook sqlite3_api->rollback_hook
|
||||
#define sqlite3_set_authorizer sqlite3_api->set_authorizer
|
||||
#define sqlite3_set_auxdata sqlite3_api->set_auxdata
|
||||
#define sqlite3_snprintf sqlite3_api->xsnprintf
|
||||
#define sqlite3_step sqlite3_api->step
|
||||
#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata
|
||||
#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup
|
||||
#define sqlite3_total_changes sqlite3_api->total_changes
|
||||
#define sqlite3_trace sqlite3_api->trace
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings
|
||||
#endif
|
||||
#define sqlite3_update_hook sqlite3_api->update_hook
|
||||
#define sqlite3_user_data sqlite3_api->user_data
|
||||
#define sqlite3_value_blob sqlite3_api->value_blob
|
||||
#define sqlite3_value_bytes sqlite3_api->value_bytes
|
||||
#define sqlite3_value_bytes16 sqlite3_api->value_bytes16
|
||||
#define sqlite3_value_double sqlite3_api->value_double
|
||||
#define sqlite3_value_int sqlite3_api->value_int
|
||||
#define sqlite3_value_int64 sqlite3_api->value_int64
|
||||
#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type
|
||||
#define sqlite3_value_text sqlite3_api->value_text
|
||||
#define sqlite3_value_text16 sqlite3_api->value_text16
|
||||
#define sqlite3_value_text16be sqlite3_api->value_text16be
|
||||
#define sqlite3_value_text16le sqlite3_api->value_text16le
|
||||
#define sqlite3_value_type sqlite3_api->value_type
|
||||
#define sqlite3_vmprintf sqlite3_api->vmprintf
|
||||
#define sqlite3_vsnprintf sqlite3_api->xvsnprintf
|
||||
#define sqlite3_overload_function sqlite3_api->overload_function
|
||||
#define sqlite3_prepare_v2 sqlite3_api->prepare_v2
|
||||
#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2
|
||||
#define sqlite3_clear_bindings sqlite3_api->clear_bindings
|
||||
#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob
|
||||
#define sqlite3_blob_bytes sqlite3_api->blob_bytes
|
||||
#define sqlite3_blob_close sqlite3_api->blob_close
|
||||
#define sqlite3_blob_open sqlite3_api->blob_open
|
||||
#define sqlite3_blob_read sqlite3_api->blob_read
|
||||
#define sqlite3_blob_write sqlite3_api->blob_write
|
||||
#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2
|
||||
#define sqlite3_file_control sqlite3_api->file_control
|
||||
#define sqlite3_memory_highwater sqlite3_api->memory_highwater
|
||||
#define sqlite3_memory_used sqlite3_api->memory_used
|
||||
#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc
|
||||
#define sqlite3_mutex_enter sqlite3_api->mutex_enter
|
||||
#define sqlite3_mutex_free sqlite3_api->mutex_free
|
||||
#define sqlite3_mutex_leave sqlite3_api->mutex_leave
|
||||
#define sqlite3_mutex_try sqlite3_api->mutex_try
|
||||
#define sqlite3_open_v2 sqlite3_api->open_v2
|
||||
#define sqlite3_release_memory sqlite3_api->release_memory
|
||||
#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem
|
||||
#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig
|
||||
#define sqlite3_sleep sqlite3_api->sleep
|
||||
#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit
|
||||
#define sqlite3_vfs_find sqlite3_api->vfs_find
|
||||
#define sqlite3_vfs_register sqlite3_api->vfs_register
|
||||
#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister
|
||||
#define sqlite3_threadsafe sqlite3_api->xthreadsafe
|
||||
#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob
|
||||
#define sqlite3_result_error_code sqlite3_api->result_error_code
|
||||
#define sqlite3_test_control sqlite3_api->test_control
|
||||
#define sqlite3_randomness sqlite3_api->randomness
|
||||
#define sqlite3_context_db_handle sqlite3_api->context_db_handle
|
||||
#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes
|
||||
#define sqlite3_limit sqlite3_api->limit
|
||||
#define sqlite3_next_stmt sqlite3_api->next_stmt
|
||||
#define sqlite3_sql sqlite3_api->sql
|
||||
#define sqlite3_status sqlite3_api->status
|
||||
#define sqlite3_backup_finish sqlite3_api->backup_finish
|
||||
#define sqlite3_backup_init sqlite3_api->backup_init
|
||||
#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount
|
||||
#define sqlite3_backup_remaining sqlite3_api->backup_remaining
|
||||
#define sqlite3_backup_step sqlite3_api->backup_step
|
||||
#define sqlite3_compileoption_get sqlite3_api->compileoption_get
|
||||
#define sqlite3_compileoption_used sqlite3_api->compileoption_used
|
||||
#define sqlite3_create_function_v2 sqlite3_api->create_function_v2
|
||||
#define sqlite3_db_config sqlite3_api->db_config
|
||||
#define sqlite3_db_mutex sqlite3_api->db_mutex
|
||||
#define sqlite3_db_status sqlite3_api->db_status
|
||||
#define sqlite3_extended_errcode sqlite3_api->extended_errcode
|
||||
#define sqlite3_log sqlite3_api->log
|
||||
#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64
|
||||
#define sqlite3_sourceid sqlite3_api->sourceid
|
||||
#define sqlite3_stmt_status sqlite3_api->stmt_status
|
||||
#define sqlite3_strnicmp sqlite3_api->strnicmp
|
||||
#define sqlite3_unlock_notify sqlite3_api->unlock_notify
|
||||
#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
|
||||
#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
|
||||
#define sqlite3_wal_hook sqlite3_api->wal_hook
|
||||
#define sqlite3_blob_reopen sqlite3_api->blob_reopen
|
||||
#define sqlite3_vtab_config sqlite3_api->vtab_config
|
||||
#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict
|
||||
/* Version 3.7.16 and later */
|
||||
#define sqlite3_close_v2 sqlite3_api->close_v2
|
||||
#define sqlite3_db_filename sqlite3_api->db_filename
|
||||
#define sqlite3_db_readonly sqlite3_api->db_readonly
|
||||
#define sqlite3_db_release_memory sqlite3_api->db_release_memory
|
||||
#define sqlite3_errstr sqlite3_api->errstr
|
||||
#define sqlite3_stmt_busy sqlite3_api->stmt_busy
|
||||
#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly
|
||||
#define sqlite3_stricmp sqlite3_api->stricmp
|
||||
#define sqlite3_uri_boolean sqlite3_api->uri_boolean
|
||||
#define sqlite3_uri_int64 sqlite3_api->uri_int64
|
||||
#define sqlite3_uri_parameter sqlite3_api->uri_parameter
|
||||
#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf
|
||||
#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2
|
||||
/* Version 3.8.7 and later */
|
||||
#define sqlite3_auto_extension sqlite3_api->auto_extension
|
||||
#define sqlite3_bind_blob64 sqlite3_api->bind_blob64
|
||||
#define sqlite3_bind_text64 sqlite3_api->bind_text64
|
||||
#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension
|
||||
#define sqlite3_load_extension sqlite3_api->load_extension
|
||||
#define sqlite3_malloc64 sqlite3_api->malloc64
|
||||
#define sqlite3_msize sqlite3_api->msize
|
||||
#define sqlite3_realloc64 sqlite3_api->realloc64
|
||||
#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension
|
||||
#define sqlite3_result_blob64 sqlite3_api->result_blob64
|
||||
#define sqlite3_result_text64 sqlite3_api->result_text64
|
||||
#define sqlite3_strglob sqlite3_api->strglob
|
||||
/* Version 3.8.11 and later */
|
||||
#define sqlite3_value_dup sqlite3_api->value_dup
|
||||
#define sqlite3_value_free sqlite3_api->value_free
|
||||
#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64
|
||||
#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64
|
||||
/* Version 3.9.0 and later */
|
||||
#define sqlite3_value_subtype sqlite3_api->value_subtype
|
||||
#define sqlite3_result_subtype sqlite3_api->result_subtype
|
||||
/* Version 3.10.0 and later */
|
||||
#define sqlite3_status64 sqlite3_api->status64
|
||||
#define sqlite3_strlike sqlite3_api->strlike
|
||||
#define sqlite3_db_cacheflush sqlite3_api->db_cacheflush
|
||||
/* Version 3.12.0 and later */
|
||||
#define sqlite3_system_errno sqlite3_api->system_errno
|
||||
/* Version 3.14.0 and later */
|
||||
#define sqlite3_trace_v2 sqlite3_api->trace_v2
|
||||
#define sqlite3_expanded_sql sqlite3_api->expanded_sql
|
||||
/* Version 3.18.0 and later */
|
||||
#define sqlite3_set_last_insert_rowid sqlite3_api->set_last_insert_rowid
|
||||
/* Version 3.20.0 and later */
|
||||
#define sqlite3_prepare_v3 sqlite3_api->prepare_v3
|
||||
#define sqlite3_prepare16_v3 sqlite3_api->prepare16_v3
|
||||
#define sqlite3_bind_pointer sqlite3_api->bind_pointer
|
||||
#define sqlite3_result_pointer sqlite3_api->result_pointer
|
||||
#define sqlite3_value_pointer sqlite3_api->value_pointer
|
||||
/* Version 3.22.0 and later */
|
||||
#define sqlite3_vtab_nochange sqlite3_api->vtab_nochange
|
||||
#define sqlite3_value_nochange sqlite3_api->value_nochange
|
||||
#define sqlite3_vtab_collation sqlite3_api->vtab_collation
|
||||
/* Version 3.24.0 and later */
|
||||
#define sqlite3_keyword_count sqlite3_api->keyword_count
|
||||
#define sqlite3_keyword_name sqlite3_api->keyword_name
|
||||
#define sqlite3_keyword_check sqlite3_api->keyword_check
|
||||
#define sqlite3_str_new sqlite3_api->str_new
|
||||
#define sqlite3_str_finish sqlite3_api->str_finish
|
||||
#define sqlite3_str_appendf sqlite3_api->str_appendf
|
||||
#define sqlite3_str_vappendf sqlite3_api->str_vappendf
|
||||
#define sqlite3_str_append sqlite3_api->str_append
|
||||
#define sqlite3_str_appendall sqlite3_api->str_appendall
|
||||
#define sqlite3_str_appendchar sqlite3_api->str_appendchar
|
||||
#define sqlite3_str_reset sqlite3_api->str_reset
|
||||
#define sqlite3_str_errcode sqlite3_api->str_errcode
|
||||
#define sqlite3_str_length sqlite3_api->str_length
|
||||
#define sqlite3_str_value sqlite3_api->str_value
|
||||
/* Version 3.25.0 and later */
|
||||
#define sqlite3_create_window_function sqlite3_api->create_window_function
|
||||
/* Version 3.26.0 and later */
|
||||
#define sqlite3_normalized_sql sqlite3_api->normalized_sql
|
||||
/* Version 3.28.0 and later */
|
||||
#define sqlite3_stmt_isexplain sqlite3_api->stmt_isexplain
|
||||
#define sqlite3_value_frombind sqlite3_api->value_frombind
|
||||
/* Version 3.30.0 and later */
|
||||
#define sqlite3_drop_modules sqlite3_api->drop_modules
|
||||
/* Version 3.31.0 and later */
|
||||
#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64
|
||||
#define sqlite3_uri_key sqlite3_api->uri_key
|
||||
#define sqlite3_filename_database sqlite3_api->filename_database
|
||||
#define sqlite3_filename_journal sqlite3_api->filename_journal
|
||||
#define sqlite3_filename_wal sqlite3_api->filename_wal
|
||||
/* Version 3.32.0 and later */
|
||||
#define sqlite3_create_filename sqlite3_api->create_filename
|
||||
#define sqlite3_free_filename sqlite3_api->free_filename
|
||||
#define sqlite3_database_file_object sqlite3_api->database_file_object
|
||||
/* Version 3.34.0 and later */
|
||||
#define sqlite3_txn_state sqlite3_api->txn_state
|
||||
/* Version 3.36.1 and later */
|
||||
#define sqlite3_changes64 sqlite3_api->changes64
|
||||
#define sqlite3_total_changes64 sqlite3_api->total_changes64
|
||||
/* Version 3.37.0 and later */
|
||||
#define sqlite3_autovacuum_pages sqlite3_api->autovacuum_pages
|
||||
/* Version 3.38.0 and later */
|
||||
#define sqlite3_error_offset sqlite3_api->error_offset
|
||||
#define sqlite3_vtab_rhs_value sqlite3_api->vtab_rhs_value
|
||||
#define sqlite3_vtab_distinct sqlite3_api->vtab_distinct
|
||||
#define sqlite3_vtab_in sqlite3_api->vtab_in
|
||||
#define sqlite3_vtab_in_first sqlite3_api->vtab_in_first
|
||||
#define sqlite3_vtab_in_next sqlite3_api->vtab_in_next
|
||||
/* Version 3.39.0 and later */
|
||||
#ifndef SQLITE_OMIT_DESERIALIZE
|
||||
#define sqlite3_deserialize sqlite3_api->deserialize
|
||||
#define sqlite3_serialize sqlite3_api->serialize
|
||||
#endif
|
||||
#define sqlite3_db_name sqlite3_api->db_name
|
||||
/* Version 3.40.0 and later */
|
||||
#define sqlite3_value_encoding sqlite3_api->value_encoding
|
||||
/* Version 3.41.0 and later */
|
||||
#define sqlite3_is_interrupted sqlite3_api->is_interrupted
|
||||
/* Version 3.43.0 and later */
|
||||
#define sqlite3_stmt_explain sqlite3_api->stmt_explain
|
||||
/* Version 3.44.0 and later */
|
||||
#define sqlite3_get_clientdata sqlite3_api->get_clientdata
|
||||
#define sqlite3_set_clientdata sqlite3_api->set_clientdata
|
||||
/* Version 3.50.0 and later */
|
||||
#define sqlite3_setlk_timeout sqlite3_api->setlk_timeout
|
||||
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
|
||||
|
||||
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
|
||||
/* This case when the file really is being compiled as a loadable
|
||||
** extension */
|
||||
# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0;
|
||||
# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v;
|
||||
# define SQLITE_EXTENSION_INIT3 \
|
||||
extern const sqlite3_api_routines *sqlite3_api;
|
||||
#else
|
||||
/* This case when the file is being statically linked into the
|
||||
** application */
|
||||
# define SQLITE_EXTENSION_INIT1 /*no-op*/
|
||||
# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */
|
||||
# define SQLITE_EXTENSION_INIT3 /*no-op*/
|
||||
#endif
|
||||
|
||||
#endif /* SQLITE3EXT_H */
|
||||
@@ -0,0 +1,60 @@
|
||||
# Hermes Plugin Catalog
|
||||
|
||||
Curated, Nous-approved Hermes plugins. Each YAML file in this directory
|
||||
(except `removed.yaml`) is one catalog entry, discoverable via
|
||||
`hermes plugins catalog` / `hermes plugins search` and installable with
|
||||
`hermes plugins install <name>`.
|
||||
|
||||
## Admission policy
|
||||
|
||||
Presence in this directory **is** the trust signal. The rules that keep it
|
||||
meaningful:
|
||||
|
||||
1. **Human-merged gate.** Entries are added *only* via a PR to the
|
||||
`hermes-agent` repository, reviewed and merged by a maintainer. There is
|
||||
no self-serve registry, no automated ingestion.
|
||||
2. **Exact SHA pins are mandatory.** Every entry pins a full 40-character
|
||||
commit SHA. Branches, tags, and short SHAs are rejected by the loader.
|
||||
Installs clone the repository and check out exactly that commit.
|
||||
3. **Pin maturity.** The pinned release should be **at least 2 weeks old**
|
||||
at pin time, mirroring the supply-chain policy used for `optional-mcps/`
|
||||
and pyproject dependencies. This gives the community time to notice a
|
||||
compromised release before Hermes ships a pointer to it.
|
||||
4. **SHA bumps are new PRs.** Updating an entry's pin is a new PR whose diff
|
||||
(old SHA → new SHA) is re-reviewed like any other change — reviewers are
|
||||
expected to look at the upstream commit range being adopted.
|
||||
5. **Owner-or-major-contributor submissions only.** An entry may only be
|
||||
submitted by the plugin repository's owner or a major contributor to it.
|
||||
Drive-by submissions of third-party repos are declined.
|
||||
6. **Declared capabilities must match reality.** The `capabilities:` block
|
||||
(tools, hooks, middleware, env vars) must match what the plugin actually
|
||||
registers at the pinned commit. Validation fails the entry otherwise —
|
||||
undeclared capability creep is treated as a security issue.
|
||||
|
||||
## Entry schema
|
||||
|
||||
```yaml
|
||||
name: example-plugin # [a-z0-9_-]{1,64}, the catalog key
|
||||
repo: https://github.com/owner/repo # https:// only
|
||||
sha: <40-hex commit sha> # mandatory exact pin
|
||||
subdir: "" # optional path within the repo
|
||||
description: One-line description.
|
||||
maintainer: OwnerName
|
||||
tier: official # official | community (default community)
|
||||
requires_hermes: ">=0.19" # optional
|
||||
docs_url: "" # optional
|
||||
platforms: [] # optional, e.g. [linux, macos]; empty = all
|
||||
capabilities:
|
||||
provides_tools: []
|
||||
provides_hooks: []
|
||||
provides_middleware: []
|
||||
requires_env: []
|
||||
```
|
||||
|
||||
## removed.yaml — the blocklist
|
||||
|
||||
When an entry is pulled from the catalog for security or policy reasons, it
|
||||
is recorded in `removed.yaml` with a reason and date. The installer refuses
|
||||
to install anything matching a removed entry's name or repo URL, so a
|
||||
malicious plugin cannot be re-installed from a stale identifier after
|
||||
removal. Removals, like additions, land via reviewed PRs.
|
||||
@@ -0,0 +1,14 @@
|
||||
name: plugin-llm-example
|
||||
repo: https://github.com/NousResearch/hermes-example-plugins
|
||||
sha: 38fe0fb53eff98d477f807432e965429e665ca33
|
||||
subdir: "plugin-llm-example"
|
||||
description: Reference plugin showing host-owned structured LLM access via ctx.llm.
|
||||
maintainer: NousResearch
|
||||
tier: official
|
||||
docs_url: ""
|
||||
platforms: []
|
||||
capabilities:
|
||||
provides_tools: []
|
||||
provides_hooks: []
|
||||
provides_middleware: []
|
||||
requires_env: []
|
||||
@@ -0,0 +1,6 @@
|
||||
# Blocklist for plugins pulled from the catalog for security or policy
|
||||
# reasons. The installer refuses to install anything whose name or repo URL
|
||||
# matches an entry here (unless the caller explicitly bypasses the check).
|
||||
# Each entry: {name, repo, reason, date}. Removals land via reviewed PRs,
|
||||
# same as additions.
|
||||
removed: []
|
||||
@@ -1037,6 +1037,7 @@ class DiscordAdapter(BasePlatformAdapter):
|
||||
"""Connect to Discord and start receiving events."""
|
||||
if not DISCORD_AVAILABLE:
|
||||
logger.error("[%s] discord.py not installed. Run: pip install discord.py", self.name)
|
||||
self._set_fatal_error("missing_dependency", "discord.py not installed", retryable=False)
|
||||
return False
|
||||
|
||||
# Load opus codec for voice channel support
|
||||
@@ -1073,6 +1074,7 @@ class DiscordAdapter(BasePlatformAdapter):
|
||||
|
||||
if not self.config.token:
|
||||
logger.error("[%s] No bot token configured", self.name)
|
||||
self._set_fatal_error("missing_credentials", "No bot token configured", retryable=False)
|
||||
return False
|
||||
|
||||
try:
|
||||
|
||||
+952
-149
File diff suppressed because it is too large
Load Diff
@@ -3434,10 +3434,12 @@ class TelegramAdapter(BasePlatformAdapter):
|
||||
"[%s] python-telegram-bot not installed. Run: pip install python-telegram-bot",
|
||||
self.name,
|
||||
)
|
||||
self._set_fatal_error("missing_dependency", "python-telegram-bot not installed", retryable=False)
|
||||
return False
|
||||
|
||||
if not self.config.token:
|
||||
logger.error("[%s] No bot token configured", self.name)
|
||||
self._set_fatal_error("missing_credentials", "No bot token configured", retryable=False)
|
||||
return False
|
||||
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Standalone structural validator for plugin-catalog entry files.
|
||||
|
||||
Validates ``plugin-catalog/*.yaml`` catalog entries and
|
||||
``plugin-catalog/removed.yaml`` against the catalog contract schema, using
|
||||
only stdlib + PyYAML so the admission CI (and third-party repos) can run it
|
||||
WITHOUT installing hermes-agent.
|
||||
|
||||
NOTE: this script intentionally duplicates the schema rules instead of
|
||||
importing ``hermes_cli`` — the whole point is the no-install requirement for
|
||||
cheap cross-repo CI use. The runtime twin of this schema lives in
|
||||
``hermes_cli/plugin_catalog.py``; if the contract changes there, update the
|
||||
rules here in lockstep.
|
||||
|
||||
Usage:
|
||||
python3 scripts/validate_plugin_catalog.py plugin-catalog/
|
||||
python3 scripts/validate_plugin_catalog.py entry.yaml removed.yaml
|
||||
python3 scripts/validate_plugin_catalog.py --json plugin-catalog/
|
||||
|
||||
Exit codes: 0 = all files valid (warnings allowed), 1 = at least one error.
|
||||
Human output is one ``<file>: ERROR: ...`` / ``<file>: warning: ...`` line
|
||||
per finding; ``--json`` emits a machine-readable report on stdout instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover - dependency guidance only
|
||||
print(
|
||||
"ERROR: PyYAML is required (pip install pyyaml)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
NAME_RE = re.compile(r"^[a-z0-9_-]{1,64}$")
|
||||
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
TIERS = ("official", "community")
|
||||
PLATFORMS = ("linux", "macos", "windows")
|
||||
CAPABILITY_KEYS = (
|
||||
"provides_tools",
|
||||
"provides_hooks",
|
||||
"provides_middleware",
|
||||
"requires_env",
|
||||
)
|
||||
# Top-level keys the contract knows about. Unknown keys WARN (forward
|
||||
# compatibility: newer catalogs must stay valid under older validators).
|
||||
KNOWN_KEYS = {
|
||||
"name",
|
||||
"repo",
|
||||
"sha",
|
||||
"subdir",
|
||||
"description",
|
||||
"maintainer",
|
||||
"tier",
|
||||
"requires_hermes",
|
||||
"docs_url",
|
||||
"platforms",
|
||||
"capabilities",
|
||||
}
|
||||
REQUIRED_KEYS = ("name", "repo", "sha", "description", "maintainer")
|
||||
|
||||
# One comparator clause of a requires_hermes spec, e.g. ">=0.19" or "!=1.2.3".
|
||||
_COMPARATOR_RE = re.compile(r"^(>=|<=|==|!=|>|<)\s*\d+(\.\d+)*$")
|
||||
|
||||
|
||||
def _is_nonempty_str(value: object) -> bool:
|
||||
return isinstance(value, str) and value.strip() != ""
|
||||
|
||||
|
||||
def _check_requires_hermes(spec: object, errors: list[str]) -> None:
|
||||
if not isinstance(spec, str):
|
||||
errors.append(f"requires_hermes must be a string, got {type(spec).__name__}")
|
||||
return
|
||||
if spec.strip() == "":
|
||||
return # empty = no constraint
|
||||
for clause in spec.split(","):
|
||||
if not _COMPARATOR_RE.match(clause.strip()):
|
||||
errors.append(
|
||||
f"requires_hermes clause {clause.strip()!r} is not a valid "
|
||||
"comparator spec (expected e.g. '>=0.19')"
|
||||
)
|
||||
|
||||
|
||||
def validate_entry(data: object) -> tuple[list[str], list[str]]:
|
||||
"""Validate one catalog entry document. Returns (errors, warnings)."""
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return ["top-level document must be a YAML mapping"], warnings
|
||||
|
||||
for key in REQUIRED_KEYS:
|
||||
if key not in data:
|
||||
errors.append(f"missing required key: {key}")
|
||||
|
||||
for key in sorted(set(data) - KNOWN_KEYS):
|
||||
warnings.append(f"unknown top-level key {key!r} (ignored by this validator)")
|
||||
|
||||
name = data.get("name")
|
||||
if "name" in data and (not isinstance(name, str) or not NAME_RE.match(name)):
|
||||
errors.append(f"name {name!r} must match [a-z0-9_-]{{1,64}}")
|
||||
|
||||
repo = data.get("repo")
|
||||
if "repo" in data and (
|
||||
not isinstance(repo, str) or not repo.startswith("https://")
|
||||
):
|
||||
errors.append(f"repo {repo!r} must be an https:// URL")
|
||||
|
||||
sha = data.get("sha")
|
||||
if "sha" in data and (not isinstance(sha, str) or not SHA_RE.match(sha)):
|
||||
errors.append(f"sha {sha!r} must be exactly 40 lowercase hex characters")
|
||||
|
||||
for key in ("description", "maintainer"):
|
||||
if key in data and not _is_nonempty_str(data[key]):
|
||||
errors.append(f"{key} must be a non-empty string")
|
||||
|
||||
tier = data.get("tier", "community")
|
||||
if tier not in TIERS:
|
||||
errors.append(f"tier {tier!r} must be one of {list(TIERS)}")
|
||||
|
||||
if "requires_hermes" in data:
|
||||
_check_requires_hermes(data["requires_hermes"], errors)
|
||||
|
||||
platforms = data.get("platforms", [])
|
||||
if platforms is None:
|
||||
platforms = []
|
||||
if not isinstance(platforms, list):
|
||||
errors.append("platforms must be a list")
|
||||
else:
|
||||
bad = [p for p in platforms if p not in PLATFORMS]
|
||||
if bad:
|
||||
errors.append(f"platforms {bad!r} not in allowed set {list(PLATFORMS)}")
|
||||
|
||||
caps = data.get("capabilities", {})
|
||||
if caps is None:
|
||||
caps = {}
|
||||
if not isinstance(caps, dict):
|
||||
errors.append("capabilities must be a mapping")
|
||||
else:
|
||||
for key in sorted(set(caps) - set(CAPABILITY_KEYS)):
|
||||
warnings.append(f"unknown capabilities key {key!r}")
|
||||
for key in CAPABILITY_KEYS:
|
||||
if key not in caps:
|
||||
continue
|
||||
value = caps[key]
|
||||
if not isinstance(value, list) or not all(
|
||||
isinstance(item, str) for item in value
|
||||
):
|
||||
errors.append(f"capabilities.{key} must be a list of strings")
|
||||
|
||||
return errors, warnings
|
||||
|
||||
|
||||
def validate_removed(data: object) -> tuple[list[str], list[str]]:
|
||||
"""Validate the removed.yaml document. Returns (errors, warnings)."""
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return ["top-level document must be a YAML mapping"], warnings
|
||||
|
||||
removed = data.get("removed")
|
||||
if removed is None:
|
||||
errors.append("missing required key: removed")
|
||||
return errors, warnings
|
||||
if not isinstance(removed, list):
|
||||
errors.append("removed must be a list")
|
||||
return errors, warnings
|
||||
|
||||
for i, item in enumerate(removed):
|
||||
if not isinstance(item, dict):
|
||||
errors.append(f"removed[{i}] must be a mapping")
|
||||
continue
|
||||
if not _is_nonempty_str(item.get("name")):
|
||||
errors.append(f"removed[{i}] missing non-empty 'name'")
|
||||
for key in ("repo", "reason", "date"):
|
||||
if key in item and not isinstance(item[key], str):
|
||||
errors.append(f"removed[{i}].{key} must be a string")
|
||||
|
||||
return errors, warnings
|
||||
|
||||
|
||||
def validate_file(path: Path) -> tuple[list[str], list[str]]:
|
||||
"""Validate one YAML file (dispatching on filename). Returns (errors, warnings)."""
|
||||
try:
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
except OSError as exc:
|
||||
return [f"cannot read file: {exc}"], []
|
||||
except yaml.YAMLError as exc:
|
||||
return [f"invalid YAML: {exc}"], []
|
||||
|
||||
if path.name == "removed.yaml":
|
||||
return validate_removed(data)
|
||||
return validate_entry(data)
|
||||
|
||||
|
||||
def collect_paths(args: list[str]) -> list[Path]:
|
||||
paths: list[Path] = []
|
||||
for arg in args:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
paths.extend(sorted(p.glob("*.yaml")))
|
||||
paths.extend(sorted(p.glob("*.yml")))
|
||||
else:
|
||||
paths.append(p)
|
||||
return paths
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Standalone structural validator for plugin-catalog entry files."
|
||||
)
|
||||
parser.add_argument(
|
||||
"paths",
|
||||
nargs="+",
|
||||
help="catalog entry files, removed.yaml, or a directory of them",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="emit a machine-readable JSON report on stdout",
|
||||
)
|
||||
opts = parser.parse_args(argv)
|
||||
|
||||
files = collect_paths(opts.paths)
|
||||
if not files:
|
||||
print("ERROR: no YAML files found", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
report = []
|
||||
any_errors = False
|
||||
for path in files:
|
||||
errors, warnings = validate_file(path)
|
||||
any_errors = any_errors or bool(errors)
|
||||
report.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"ok": not errors,
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
}
|
||||
)
|
||||
|
||||
if opts.json:
|
||||
print(json.dumps({"ok": not any_errors, "files": report}, indent=2))
|
||||
else:
|
||||
for entry in report:
|
||||
for err in entry["errors"]:
|
||||
print(f"{entry['path']}: ERROR: {err}")
|
||||
for warn in entry["warnings"]:
|
||||
print(f"{entry['path']}: warning: {warn}")
|
||||
checked = len(report)
|
||||
bad = sum(1 for e in report if not e["ok"])
|
||||
if any_errors:
|
||||
print(f"FAIL: {bad}/{checked} file(s) invalid")
|
||||
else:
|
||||
print(f"OK: {checked} file(s) valid")
|
||||
|
||||
return 1 if any_errors else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Strict redaction at every compaction text boundary (issue #43666 item 2).
|
||||
|
||||
Compaction summaries persist across sessions and re-enter every subsequent
|
||||
summarizer prompt, so ``_redact_compaction_text()`` applies strict mode
|
||||
(``force=True, redact_url_credentials=True``) at each boundary:
|
||||
|
||||
- serializer input (``_serialize_for_summary``: message content + tool args)
|
||||
- deterministic fallback summary (``_build_static_fallback_summary``)
|
||||
- summarizer LLM output (``_generate_summary`` return / ``_previous_summary``)
|
||||
- focus text (manual ``/compress <focus>`` and ``_derive_auto_focus_topic``)
|
||||
- previous-summary re-entry into the iterative-update prompt
|
||||
|
||||
Every test disables the global redaction flag (simulating
|
||||
``security.redact_secrets: false``) to prove ``force=True`` still redacts at
|
||||
the persistence boundary, and uses an OAuth-callback-style URL to prove
|
||||
``redact_url_credentials=True`` strips opaque URL tokens that default-mode
|
||||
redaction deliberately passes through.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.context_compressor import (
|
||||
ContextCompressor,
|
||||
SUMMARY_PREFIX,
|
||||
_redact_compaction_text,
|
||||
)
|
||||
|
||||
SECRET = "sk-proj-" + ("a" * 40)
|
||||
OAUTH_URL = (
|
||||
"https://localhost/callback?code=opaque-code-123"
|
||||
"&access_token=opaque-token-456&state=keep"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _redaction_globally_disabled(monkeypatch):
|
||||
"""Simulate security.redact_secrets: false — force=True must still win."""
|
||||
monkeypatch.setattr("agent.redact._REDACT_ENABLED", False)
|
||||
|
||||
|
||||
def _compressor() -> ContextCompressor:
|
||||
with patch(
|
||||
"agent.context_compressor.get_model_context_length",
|
||||
return_value=100000,
|
||||
):
|
||||
return ContextCompressor(model="test/model", quiet_mode=True)
|
||||
|
||||
|
||||
def _response(content: str):
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = content
|
||||
return mock_response
|
||||
|
||||
|
||||
def _assert_clean(text: str):
|
||||
assert SECRET not in text
|
||||
assert "sk-proj-" not in text
|
||||
assert "code=opaque-code-123" not in text
|
||||
assert "access_token=opaque-token-456" not in text
|
||||
assert "code=***" in text
|
||||
assert "access_token=***" in text
|
||||
assert "state=keep" in text
|
||||
|
||||
|
||||
def test_helper_is_strict_even_when_redaction_disabled():
|
||||
result = _redact_compaction_text(f"key {SECRET} url {OAUTH_URL}")
|
||||
_assert_clean(result)
|
||||
# None-safety: helper is used on optional fields.
|
||||
assert _redact_compaction_text(None) == ""
|
||||
|
||||
|
||||
def test_serializer_input_redacts_content_and_tool_args():
|
||||
c = _compressor()
|
||||
messages = [
|
||||
{"role": "user", "content": f"token {SECRET} url {OAUTH_URL}"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"function": {
|
||||
"name": "terminal",
|
||||
"arguments": (
|
||||
f'{{"command": "curl {OAUTH_URL}",'
|
||||
f' "note": "{SECRET}"}}'
|
||||
),
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call-1", "content": f"got {SECRET}"},
|
||||
]
|
||||
|
||||
serialized = c._serialize_for_summary(messages)
|
||||
|
||||
_assert_clean(serialized)
|
||||
|
||||
|
||||
def test_fallback_summary_redacts_secrets():
|
||||
c = _compressor()
|
||||
turns = [
|
||||
{"role": "user", "content": f"deploy with {SECRET} via {OAUTH_URL}"},
|
||||
{"role": "assistant", "content": f"ran curl {OAUTH_URL}"},
|
||||
]
|
||||
|
||||
summary = c._build_static_fallback_summary(turns, reason="test outage")
|
||||
|
||||
_assert_clean(summary)
|
||||
|
||||
|
||||
def test_summary_output_redacts_llm_echoed_secrets():
|
||||
c = _compressor()
|
||||
leaked = f"Summary leaked OPENAI_API_KEY {SECRET} and {OAUTH_URL}"
|
||||
|
||||
with patch(
|
||||
"agent.context_compressor.call_llm", return_value=_response(leaked)
|
||||
):
|
||||
summary = c._generate_summary([{"role": "user", "content": "hi"}])
|
||||
|
||||
assert summary is not None
|
||||
_assert_clean(summary)
|
||||
# The stored iterative-update seed must be clean too.
|
||||
_assert_clean(c._previous_summary)
|
||||
|
||||
|
||||
def test_manual_focus_topic_redacted_before_summary_prompt():
|
||||
c = _compressor()
|
||||
turns = [
|
||||
{"role": "user", "content": "Summarize safely"},
|
||||
{"role": "assistant", "content": "OK"},
|
||||
]
|
||||
|
||||
with patch(
|
||||
"agent.context_compressor.call_llm",
|
||||
return_value=_response("## Goal\nSafe summary."),
|
||||
) as mock_call:
|
||||
result = c._generate_summary(
|
||||
turns, focus_topic=f"manual focus {SECRET} {OAUTH_URL}"
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
|
||||
_assert_clean(prompt)
|
||||
|
||||
|
||||
def test_auto_focus_topic_redacted():
|
||||
c = _compressor()
|
||||
|
||||
focus = c._derive_auto_focus_topic(
|
||||
[
|
||||
{"role": "assistant", "content": "older assistant turn"},
|
||||
{"role": "user", "content": f"focus has {SECRET} and {OAUTH_URL}"},
|
||||
]
|
||||
)
|
||||
|
||||
assert focus is not None
|
||||
_assert_clean(focus)
|
||||
|
||||
|
||||
def test_previous_summary_redacted_before_iterative_prompt_reentry():
|
||||
"""Legacy persisted summaries may predate compaction redaction."""
|
||||
c = _compressor()
|
||||
c._previous_summary = f"Old summary leaked {SECRET} and {OAUTH_URL}"
|
||||
|
||||
with patch(
|
||||
"agent.context_compressor.call_llm",
|
||||
return_value=_response("updated summary"),
|
||||
) as mock_call:
|
||||
result = c._generate_summary(
|
||||
[
|
||||
{"role": "user", "content": "new turn"},
|
||||
{"role": "assistant", "content": "new work"},
|
||||
]
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
|
||||
assert "PREVIOUS SUMMARY:" in prompt
|
||||
_assert_clean(prompt)
|
||||
# After generation, _previous_summary holds the new (clean) LLM output —
|
||||
# the leaked secret must not have survived anywhere in it.
|
||||
assert SECRET not in c._previous_summary
|
||||
assert "access_token=opaque-token-456" not in c._previous_summary
|
||||
|
||||
|
||||
def test_resumed_handoff_summary_redacted_before_iterative_prompt():
|
||||
"""Persisted handoff messages may contain pre-fix secrets after resume."""
|
||||
with patch(
|
||||
"agent.context_compressor.get_model_context_length",
|
||||
return_value=100000,
|
||||
):
|
||||
c = ContextCompressor(
|
||||
model="test/model",
|
||||
threshold_percent=0.85,
|
||||
protect_first_n=1,
|
||||
protect_last_n=1,
|
||||
quiet_mode=True,
|
||||
)
|
||||
old_summary = f"RESUMED-SUMMARY leaked {SECRET} and {OAUTH_URL}"
|
||||
messages = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": f"{SUMMARY_PREFIX}\n{old_summary}"},
|
||||
{"role": "assistant", "content": "handoff acknowledged after resume"},
|
||||
{"role": "user", "content": "new user turn after resume"},
|
||||
{"role": "assistant", "content": "new assistant work after resume"},
|
||||
{"role": "user", "content": "more new work after resume"},
|
||||
{"role": "assistant", "content": "latest tail response"},
|
||||
{"role": "user", "content": "final active request stays in tail"},
|
||||
]
|
||||
|
||||
with patch(
|
||||
"agent.context_compressor.call_llm",
|
||||
return_value=_response("updated summary"),
|
||||
) as mock_call:
|
||||
c.compress(messages)
|
||||
|
||||
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
|
||||
assert "PREVIOUS SUMMARY:" in prompt
|
||||
_assert_clean(prompt)
|
||||
@@ -0,0 +1,151 @@
|
||||
import json
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from agent.conversation_compression import compress_context
|
||||
from agent.context_compressor import ContextCompressor
|
||||
|
||||
|
||||
class _TodoStore:
|
||||
def format_for_injection(self):
|
||||
return ""
|
||||
|
||||
|
||||
class _Agent:
|
||||
def __init__(self, compressor):
|
||||
self.context_compressor = compressor
|
||||
self.session_id = "session-telemetry-test"
|
||||
self.platform = "cli"
|
||||
self.model = "test/main-model"
|
||||
self.provider = "test-provider"
|
||||
self.tools = []
|
||||
self._compression_feasibility_checked = True
|
||||
self.compression_in_place = False
|
||||
self._memory_manager = None
|
||||
self._session_db = None
|
||||
self._todo_store = _TodoStore()
|
||||
self._cached_system_prompt = None
|
||||
|
||||
def _emit_status(self, _message):
|
||||
pass
|
||||
|
||||
def _emit_warning(self, _message):
|
||||
pass
|
||||
|
||||
def _invalidate_system_prompt(self):
|
||||
self._cached_system_prompt = None
|
||||
|
||||
def _build_system_prompt(self, system_message):
|
||||
return system_message
|
||||
|
||||
def commit_memory_session(self, _messages):
|
||||
pass
|
||||
|
||||
|
||||
def _messages(secret_text="TOPSECRET_TRANSCRIPT_TEXT"):
|
||||
msgs = [{"role": "system", "content": "system prompt"}]
|
||||
for idx in range(10):
|
||||
msgs.append({"role": "user", "content": f"user message {idx} {secret_text}"})
|
||||
msgs.append({"role": "assistant", "content": f"assistant reply {idx} {secret_text}"})
|
||||
return msgs
|
||||
|
||||
|
||||
def _extract_telemetry(caplog):
|
||||
records = [
|
||||
record.getMessage()
|
||||
for record in caplog.records
|
||||
if "context compression attempt telemetry:" in record.getMessage()
|
||||
]
|
||||
assert len(records) == 1
|
||||
return json.loads(records[0].split("context compression attempt telemetry: ", 1)[1])
|
||||
|
||||
|
||||
def test_compression_attempt_telemetry_is_metadata_only(caplog):
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
compressor = ContextCompressor(
|
||||
model="test/main-model",
|
||||
provider="test-provider",
|
||||
threshold_percent=0.50,
|
||||
quiet_mode=True,
|
||||
config_context_length=100_000,
|
||||
)
|
||||
compressor.tail_token_budget = 10
|
||||
agent = _Agent(compressor)
|
||||
|
||||
with patch.object(compressor, "_generate_summary", return_value="SANITIZED SUMMARY"):
|
||||
with caplog.at_level(logging.INFO, logger="agent.conversation_compression"):
|
||||
compressed, system_prompt = compress_context(
|
||||
agent,
|
||||
_messages(),
|
||||
"system prompt",
|
||||
approx_tokens=75_000,
|
||||
force=True,
|
||||
)
|
||||
|
||||
assert system_prompt == "system prompt"
|
||||
assert compressed is not None
|
||||
payload = _extract_telemetry(caplog)
|
||||
|
||||
assert payload["event"] == "compression_attempt"
|
||||
assert payload["attempt_id"]
|
||||
assert payload["session_id"] == "session-telemetry-test"
|
||||
assert payload["trigger_source"] == "manual"
|
||||
assert payload["main_model"] == "test/main-model"
|
||||
assert payload["main_context_limit"] == 100_000
|
||||
assert payload["current_estimated_tokens"] == 75_000
|
||||
assert payload["effective_threshold"] == compressor.threshold_tokens
|
||||
assert payload["protected_head_tokens"] is not None
|
||||
assert payload["protected_tail_tokens"] is not None
|
||||
assert payload["middle_window_tokens"] is not None
|
||||
assert payload["chunking"] is False
|
||||
assert payload["chunk_count"] in {0, 1}
|
||||
assert payload["commit_status"] == "committed"
|
||||
assert payload["split_status"] == "not_applicable"
|
||||
assert payload["fallback_used"] is False
|
||||
assert isinstance(payload["total_duration_ms"], int)
|
||||
|
||||
raw_log = json.dumps(payload)
|
||||
assert "TOPSECRET_TRANSCRIPT_TEXT" not in raw_log
|
||||
assert "SANITIZED SUMMARY" not in raw_log
|
||||
assert "user message" not in raw_log
|
||||
assert "assistant reply" not in raw_log
|
||||
|
||||
|
||||
def test_aux_call_telemetry_records_durations_without_content(caplog):
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
compressor = ContextCompressor(
|
||||
model="test/main-model",
|
||||
provider="test-provider",
|
||||
threshold_percent=0.50,
|
||||
quiet_mode=True,
|
||||
config_context_length=100_000,
|
||||
)
|
||||
compressor.tail_token_budget = 10
|
||||
agent = _Agent(compressor)
|
||||
response = SimpleNamespace(
|
||||
choices=[SimpleNamespace(message=SimpleNamespace(content="SANITIZED SUMMARY"))]
|
||||
)
|
||||
|
||||
with patch("agent.context_compressor.call_llm", return_value=response):
|
||||
with caplog.at_level(logging.INFO, logger="agent.conversation_compression"):
|
||||
compress_context(
|
||||
agent,
|
||||
_messages(),
|
||||
"system prompt",
|
||||
approx_tokens=75_000,
|
||||
)
|
||||
|
||||
payload = _extract_telemetry(caplog)
|
||||
assert payload["aux_prompt_tokens"] is not None
|
||||
# Current main intentionally omits max_tokens from the aux summary call
|
||||
# (the summary budget is prompt-level guidance only), so no output
|
||||
# reservation is recorded.
|
||||
assert payload["aux_output_reservation"] is None
|
||||
assert isinstance(payload["aux_call_duration_ms"], int)
|
||||
assert payload["aux_provider"]
|
||||
assert payload["aux_model"]
|
||||
|
||||
raw_log = json.dumps(payload)
|
||||
assert "TOPSECRET_TRANSCRIPT_TEXT" not in raw_log
|
||||
assert "SANITIZED SUMMARY" not in raw_log
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Regression tests for blank user echoes displacing actionable compaction state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.context_compressor import (
|
||||
COMPRESSED_SUMMARY_METADATA_KEY,
|
||||
SUMMARY_PREFIX,
|
||||
ContextCompressor,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def compressor() -> ContextCompressor:
|
||||
with patch(
|
||||
"agent.context_compressor.get_model_context_length",
|
||||
return_value=100_000,
|
||||
):
|
||||
instance = ContextCompressor(
|
||||
model="test/model",
|
||||
threshold_percent=0.85,
|
||||
protect_first_n=2,
|
||||
protect_last_n=2,
|
||||
quiet_mode=True,
|
||||
)
|
||||
instance.tail_token_budget = 10
|
||||
return instance
|
||||
|
||||
|
||||
def _append_tool_run(messages: list[dict], prefix: str, count: int = 6) -> None:
|
||||
for index in range(count):
|
||||
call_id = f"{prefix}-{index}"
|
||||
messages.extend(
|
||||
[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": call_id,
|
||||
"function": {"name": "read_file", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": call_id,
|
||||
"content": "x" * 400,
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _compress(compressor: ContextCompressor, messages: list[dict]) -> list[dict]:
|
||||
with patch.object(
|
||||
compressor,
|
||||
"_generate_summary",
|
||||
return_value=f"{SUMMARY_PREFIX}\nsummary of older work",
|
||||
):
|
||||
return compressor.compress(messages, current_tokens=90_000)
|
||||
|
||||
|
||||
def _assert_no_adjacent_user_roles(messages: list[dict]) -> None:
|
||||
for previous, current in zip(messages, messages[1:]):
|
||||
assert (previous.get("role"), current.get("role")) != ("user", "user")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"blank",
|
||||
[
|
||||
"",
|
||||
" \n\t",
|
||||
None,
|
||||
[],
|
||||
[{"type": "text", "text": " "}],
|
||||
[{"type": "input_text", "text": " "}],
|
||||
],
|
||||
)
|
||||
def test_blank_echo_does_not_displace_async_completion(compressor, blank):
|
||||
completion = "[ASYNC DELEGATION BATCH COMPLETE — deleg_current]\nnew result"
|
||||
messages = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "old request"},
|
||||
{"role": "assistant", "content": "old reply"},
|
||||
{"role": "user", "content": completion},
|
||||
{"role": "user", "content": blank},
|
||||
{"role": "assistant", "content": "working from the completion"},
|
||||
]
|
||||
|
||||
assert compressor._find_last_user_message_idx(messages, head_end=1) == 3
|
||||
|
||||
|
||||
def test_leading_blank_without_actionable_user_is_not_removed(compressor):
|
||||
messages = [
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "assistant", "content": "visible reply"},
|
||||
]
|
||||
|
||||
assert compressor._find_last_user_message_idx(messages, 0) == -1
|
||||
assert compressor._blank_echo_indices_after(messages, -1) == set()
|
||||
|
||||
|
||||
def test_image_only_user_turn_survives_compaction(compressor):
|
||||
image_content = [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,AA=="},
|
||||
}
|
||||
]
|
||||
messages: list[dict] = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "old request"},
|
||||
{"role": "assistant", "content": "old reply"},
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": f"older question {index}"}
|
||||
if index % 2 == 0
|
||||
else {"role": "assistant", "content": f"older reply {index}"}
|
||||
for index in range(6)
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": image_content},
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "assistant", "content": "analyzing the image"},
|
||||
]
|
||||
_append_tool_run(messages, "image")
|
||||
|
||||
result = _compress(compressor, messages)
|
||||
|
||||
assert any(message.get("content") == image_content for message in result)
|
||||
assert all(not compressor._is_blank_user_turn(message) for message in result)
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
[{"type": "audio", "source": {"data": "AA=="}}],
|
||||
[{"type": "input_audio", "input_audio": {"data": "AA=="}}],
|
||||
[{"type": "future_input", "payload": {"value": 7}}],
|
||||
],
|
||||
ids=["audio", "input-audio", "unknown-structured"],
|
||||
)
|
||||
def test_structured_non_text_user_turn_survives_compaction(compressor, payload):
|
||||
messages: list[dict] = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "old request"},
|
||||
{"role": "assistant", "content": "old reply"},
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": f"older question {index}"}
|
||||
if index % 2 == 0
|
||||
else {"role": "assistant", "content": f"older reply {index}"}
|
||||
for index in range(6)
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": payload},
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "assistant", "content": "processing structured input"},
|
||||
]
|
||||
_append_tool_run(messages, "structured")
|
||||
|
||||
result = _compress(compressor, messages)
|
||||
|
||||
assert any(message.get("content") == payload for message in result)
|
||||
assert all(not compressor._is_blank_user_turn(message) for message in result)
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
|
||||
|
||||
def test_completion_survives_compaction_verbatim_after_blank_echo(compressor):
|
||||
completion = (
|
||||
"[ASYNC DELEGATION BATCH COMPLETE — deleg_current]\n"
|
||||
"The newest result that must remain actionable."
|
||||
)
|
||||
messages: list[dict] = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "initial request"},
|
||||
{"role": "assistant", "content": "initial reply"},
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": f"older question {index}"}
|
||||
if index % 2 == 0
|
||||
else {"role": "assistant", "content": f"older reply {index}"}
|
||||
for index in range(6)
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": completion},
|
||||
{"role": "user", "content": " \n"},
|
||||
{"role": "assistant", "content": "working from the completion"},
|
||||
]
|
||||
_append_tool_run(messages, "tail")
|
||||
|
||||
result = _compress(compressor, messages)
|
||||
|
||||
completion_rows = [message for message in result if message.get("content") == completion]
|
||||
assert len(completion_rows) == 1
|
||||
assert not completion_rows[0].get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
summary_rows = [
|
||||
message for message in result if message.get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
]
|
||||
assert len(summary_rows) == 1
|
||||
assert summary_rows[0].get("role") == "user"
|
||||
assert all(not compressor._is_blank_user_turn(message) for message in result)
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
|
||||
second_result = _compress(compressor, result)
|
||||
second_completion_rows = [
|
||||
message for message in second_result if message.get("content") == completion
|
||||
]
|
||||
assert len(second_completion_rows) == 1
|
||||
assert not second_completion_rows[0].get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
|
||||
|
||||
def test_completion_at_compress_start_survives_when_blank_echo_is_compress_end(
|
||||
compressor,
|
||||
):
|
||||
completion = "latest actionable completion at the compression boundary"
|
||||
messages = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "initial request"},
|
||||
{"role": "assistant", "content": "initial reply"},
|
||||
{"role": "user", "content": completion},
|
||||
{"role": "user", "content": ""},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "boundary-call",
|
||||
"function": {"name": "read_file", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "boundary-call", "content": "result"},
|
||||
{"role": "assistant", "content": "working from the completion"},
|
||||
]
|
||||
|
||||
with (
|
||||
patch.object(compressor, "_protect_head_size", return_value=3),
|
||||
patch.object(compressor, "_find_tail_cut_by_tokens", return_value=3),
|
||||
patch.object(compressor, "_generate_summary") as generate_summary,
|
||||
):
|
||||
result = compressor.compress(messages, current_tokens=90_000)
|
||||
|
||||
completion_rows = [message for message in result if message.get("content") == completion]
|
||||
assert len(completion_rows) == 1
|
||||
assert not completion_rows[0].get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
assert not any(
|
||||
message.get(COMPRESSED_SUMMARY_METADATA_KEY) for message in result
|
||||
)
|
||||
assert len(result) == len(messages) - 1
|
||||
assert compressor.compression_count == 0
|
||||
assert compressor._last_compression_savings_pct == 0.0
|
||||
generate_summary.assert_not_called()
|
||||
assert any(message.get("tool_call_id") == "boundary-call" for message in result)
|
||||
assert result[-1].get("content") == "working from the completion"
|
||||
assert [message.get("role") for message in result] == [
|
||||
"system",
|
||||
"user",
|
||||
"assistant",
|
||||
"user",
|
||||
"assistant",
|
||||
"tool",
|
||||
"assistant",
|
||||
]
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
|
||||
|
||||
def test_tool_call_head_compacts_without_rewriting_event(compressor):
|
||||
completion = "latest actionable completion"
|
||||
messages: list[dict] = [
|
||||
{"role": "user", "content": "initial request"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "head-call",
|
||||
"function": {"name": "read_file", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "head-call", "content": "head result"},
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": f"older question {index}"}
|
||||
if index % 2 == 0
|
||||
else {"role": "assistant", "content": f"older reply {index}"}
|
||||
for index in range(6)
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": completion},
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "assistant", "content": "working"},
|
||||
]
|
||||
_append_tool_run(messages, "tail")
|
||||
|
||||
result = _compress(compressor, messages)
|
||||
|
||||
assert compressor._last_compress_aborted is False
|
||||
assert any(message.get("content") == completion for message in result)
|
||||
head = next(
|
||||
message
|
||||
for message in result
|
||||
if any(call.get("id") == "head-call" for call in message.get("tool_calls", []))
|
||||
)
|
||||
assert not head.get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
assert any(message.get("tool_call_id") == "head-call" for message in result)
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
@@ -245,7 +245,9 @@ class TestCompress:
|
||||
assert "Summary generation was unavailable" in combined
|
||||
assert "removed to free context space but could not be summarized" not in combined
|
||||
assert c._last_summary_fallback_used is True
|
||||
assert c._last_summary_dropped_count == 3
|
||||
# The assistant immediately before the latest actionable user turn is
|
||||
# retained as a role bridge, so only the two genuinely older rows drop.
|
||||
assert c._last_summary_dropped_count == 2
|
||||
|
||||
def test_fallback_summary_does_not_triplicate_latest_user_ask(self):
|
||||
"""Regression for #49307: the deterministic fallback summary used to
|
||||
@@ -3165,6 +3167,207 @@ class TestUpdateModelResetsCalibration:
|
||||
assert comp._summary_failure_cooldown_until == cooldown_until
|
||||
|
||||
|
||||
class TestThresholdTokensCap:
|
||||
"""Tests for the absolute token cap (compression.threshold_tokens).
|
||||
|
||||
The cap takes the lower of the ratio-based threshold and the absolute
|
||||
count. It must survive model switches (update_model re-applies it)
|
||||
and be clamped to the model's context length.
|
||||
"""
|
||||
|
||||
def test_cap_lower_than_ratio_uses_cap(self):
|
||||
"""When the cap is lower than the ratio-based threshold, the cap wins."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=50_000,
|
||||
)
|
||||
# Ratio-based: 200000 * 0.50 = 100000. Cap: 50000. Effective: 50000.
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
def test_cap_higher_than_ratio_uses_ratio(self):
|
||||
"""When the cap is higher than the ratio-based threshold, the ratio wins."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=2_000_000,
|
||||
)
|
||||
# Ratio-based: 1000000 * 0.50 = 500000. Cap: 2000000, clamped to 1000000.
|
||||
# Effective: min(500000, 1000000) = 500000.
|
||||
assert comp.threshold_tokens == 500_000
|
||||
|
||||
def test_no_cap_uses_ratio_only(self):
|
||||
"""Without a cap, the ratio-based threshold is used."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
)
|
||||
assert comp.threshold_tokens == 500_000
|
||||
assert comp.threshold_tokens_cap is None
|
||||
|
||||
def test_cap_survives_model_switch(self):
|
||||
"""The cap must be re-applied after update_model() switches to a
|
||||
different context length. This is the core sweeper feedback: the
|
||||
old PR's post-construction patch was undone by update_model()
|
||||
restoring _configured_threshold_percent."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=40_000,
|
||||
)
|
||||
assert comp.threshold_tokens == 40_000 # cap wins on 200K model
|
||||
|
||||
# Switch to a 100K model — ratio-based would be 50000, but cap is 40000
|
||||
comp.update_model("model-b", context_length=100_000)
|
||||
assert comp.threshold_tokens == 40_000 # cap still wins
|
||||
|
||||
def test_cap_survives_model_switch_to_smaller_window(self):
|
||||
"""When switching to a model whose ratio-based threshold is below
|
||||
the cap, the ratio-based threshold wins (cap is a ceiling, not a floor)."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=50_000,
|
||||
)
|
||||
assert comp.threshold_tokens == 50_000 # cap wins on 200K (ratio=100K)
|
||||
|
||||
# Switch to a 64K model — ratio-based floor is 64000 (MINIMUM_CONTEXT_LENGTH)
|
||||
# which is > 50000 cap, so... actually 64000 > 50000 means cap still wins
|
||||
# Let's test with a 80K model: ratio=40000, cap=50000 → ratio wins
|
||||
comp.update_model("model-b", context_length=80_000)
|
||||
assert comp.threshold_tokens <= 50_000 # cap is a ceiling
|
||||
# 80000 * 0.50 = 40000, floored to 64000, cap 50000 → min(64000, 50000) = 50000
|
||||
# The floor raises it to 64000, then cap clamps to 50000
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
def test_cap_clamped_to_context_length(self):
|
||||
"""A cap larger than the context length is clamped, so the
|
||||
ratio-based threshold wins for small-context models."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=64_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=500_000,
|
||||
)
|
||||
# 64000 * 0.50 = 32000, floored to 64000 (MINIMUM_CONTEXT_LENGTH),
|
||||
# degenerate: floored >= window → 85% of 64000 = 54400.
|
||||
# Cap 500000 clamped to 64000. min(54400, 64000) = 54400.
|
||||
assert comp.threshold_tokens == 54400 # ratio-based wins
|
||||
|
||||
def test_cap_with_max_tokens_reservation(self):
|
||||
"""The cap applies after max_tokens reservation is factored in."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
max_tokens=32_768,
|
||||
threshold_tokens_cap=50_000,
|
||||
)
|
||||
# effective_window = 200000 - 32768 = 167232
|
||||
# ratio: 167232 * 0.50 = 83616, floored to max(83616, 64000) = 83616
|
||||
# cap: min(50000, 200000) = 50000. min(83616, 50000) = 50000.
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
def test_cap_survives_model_switch_with_max_tokens(self):
|
||||
"""The cap survives model switch even when max_tokens changes."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
max_tokens=32_768,
|
||||
threshold_tokens_cap=50_000,
|
||||
)
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
# Switch to a smaller model with different max_tokens
|
||||
comp.update_model("model-b", context_length=100_000, max_tokens=16_384)
|
||||
# effective_window = 100000 - 16384 = 83616
|
||||
# ratio: 83616 * 0.50 = 41808, floored to max(41808, 64000) = 64000
|
||||
# degenerate: floored (64000) >= effective_window (83616)? No, 64000 < 83616.
|
||||
# So threshold = 64000. cap: min(50000, 100000) = 50000. min(64000, 50000) = 50000.
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
def test_invalid_cap_treated_as_none(self):
|
||||
"""Non-numeric, zero, or negative cap values are treated as None."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
comp0 = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=0,
|
||||
)
|
||||
assert comp0.threshold_tokens_cap is None
|
||||
assert comp0.threshold_tokens == 500_000
|
||||
|
||||
comp_neg = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=-100,
|
||||
)
|
||||
assert comp_neg.threshold_tokens_cap is None
|
||||
|
||||
comp_str = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap="not-a-number",
|
||||
)
|
||||
assert comp_str.threshold_tokens_cap is None
|
||||
|
||||
def test_should_compress_fires_at_cap_below_ratio_threshold(self):
|
||||
"""Behavioral: with a cap below the ratio-based threshold,
|
||||
should_compress() fires once usage crosses the cap — even though
|
||||
the percentage threshold has not been reached (first-fires-wins)."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=200_000,
|
||||
)
|
||||
# Ratio-based would be 500K; cap pulls the trigger down to 200K.
|
||||
assert comp.should_compress(150_000) is False # below cap
|
||||
assert comp.should_compress(200_000) is True # at cap (below 500K pct)
|
||||
assert comp.should_compress(250_000) is True # above cap
|
||||
|
||||
def test_default_config_disabled_and_no_behavior_change(self):
|
||||
"""DEFAULT_CONFIG ships threshold_tokens=None (disabled) and both
|
||||
None and 0 leave the ratio-based trigger byte-identical."""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
assert DEFAULT_CONFIG["compression"]["threshold_tokens"] is None
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
baseline = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
)
|
||||
comp_none = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=None,
|
||||
)
|
||||
comp_zero = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=0,
|
||||
)
|
||||
assert comp_none.threshold_tokens == baseline.threshold_tokens
|
||||
assert comp_zero.threshold_tokens == baseline.threshold_tokens
|
||||
# And after a model switch, still identical to baseline.
|
||||
baseline.update_model("model-b", context_length=200_000)
|
||||
comp_none.update_model("model-b", context_length=200_000)
|
||||
comp_zero.update_model("model-b", context_length=200_000)
|
||||
assert comp_none.threshold_tokens == baseline.threshold_tokens
|
||||
assert comp_zero.threshold_tokens == baseline.threshold_tokens
|
||||
|
||||
def test_pct_floor_unaffected_by_cap(self):
|
||||
"""The small-context pct floor (raise-only to 0.75 under 512K) is
|
||||
computed independently of the cap: the cap clamps the resulting
|
||||
token threshold but never changes threshold_percent, and a
|
||||
cap-free small-context model keeps the floored pct."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=100_000,
|
||||
)
|
||||
# Floor raised pct to 0.75 (200K < 512K) regardless of the cap.
|
||||
assert comp.threshold_percent == 0.75
|
||||
# Cap clamps the token trigger below the floored pct value (150K).
|
||||
assert comp.threshold_tokens == 100_000
|
||||
# Switching to a large-context model drops the pct back to the
|
||||
# configured 0.50 — cap presence doesn't perturb the re-derivation.
|
||||
comp.update_model("model-b", context_length=1_000_000)
|
||||
assert comp.threshold_percent == 0.50
|
||||
assert comp.threshold_tokens == 100_000 # cap still wins over 500K
|
||||
|
||||
|
||||
class TestTruncateToolCallArgsJson:
|
||||
"""Regression tests for #11762.
|
||||
|
||||
|
||||
@@ -2,7 +2,14 @@
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from agent.context_compressor import ContextCompressor, SUMMARY_PREFIX
|
||||
from agent.context_compressor import (
|
||||
COMPRESSED_SUMMARY_METADATA_KEY,
|
||||
ContextCompressor,
|
||||
SUMMARY_PREFIX,
|
||||
_MERGED_PRIOR_CONTEXT_HEADER,
|
||||
_MERGED_SUMMARY_DELIMITER,
|
||||
_SUMMARY_END_MARKER,
|
||||
)
|
||||
|
||||
|
||||
def _compressor() -> ContextCompressor:
|
||||
@@ -36,6 +43,21 @@ def _messages_with_handoff(summary_body: str):
|
||||
]
|
||||
|
||||
|
||||
def _messages_with_merged_handoff(summary_body: str, prior_tail: str):
|
||||
merged = {
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"{_MERGED_PRIOR_CONTEXT_HEADER}\n{prior_tail}\n\n"
|
||||
f"{_MERGED_SUMMARY_DELIMITER}\n\n"
|
||||
f"{SUMMARY_PREFIX}\n{summary_body}\n\n{_SUMMARY_END_MARKER}"
|
||||
),
|
||||
COMPRESSED_SUMMARY_METADATA_KEY: True,
|
||||
}
|
||||
messages = _messages_with_handoff(summary_body)
|
||||
messages[1] = merged
|
||||
return messages
|
||||
|
||||
|
||||
def test_existing_previous_summary_is_not_serialized_again_as_new_turn():
|
||||
"""Same-process iterative compression should not feed the old handoff twice."""
|
||||
compressor = _compressor()
|
||||
@@ -112,3 +134,127 @@ def test_handoff_in_protected_head_is_replaced_not_duplicated():
|
||||
assert "UPDATED summary body" in str(summary_messages[0]["content"])
|
||||
assert old_summary not in str(summary_messages[0]["content"])
|
||||
assert old_summary not in "\n".join(str(msg.get("content") or "") for msg in compressed)
|
||||
|
||||
|
||||
def test_recompression_drops_prior_protected_handoff_from_output():
|
||||
"""Repeated compression must not preserve stale handoff bubbles forever."""
|
||||
compressor = _compressor()
|
||||
old_summary = "DUPLICATE-HANDOFF-BODY unique old facts"
|
||||
|
||||
with patch.object(
|
||||
compressor,
|
||||
"_generate_summary",
|
||||
return_value=ContextCompressor._with_summary_prefix(
|
||||
"updated summary with old facts folded in"
|
||||
),
|
||||
):
|
||||
result = compressor.compress(_messages_with_handoff(old_summary))
|
||||
|
||||
joined = "\n".join(str(message.get("content", "")) for message in result)
|
||||
assert old_summary not in joined
|
||||
assert joined.count(SUMMARY_PREFIX) == 1
|
||||
assert "updated summary with old facts folded in" in joined
|
||||
|
||||
|
||||
def test_legacy_string_merged_handoff_preserves_real_tail_text():
|
||||
"""Pre-delimiter string handoffs still unwrap content after the end marker."""
|
||||
message = {
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"{SUMMARY_PREFIX}\nold summary\n\n"
|
||||
f"{_SUMMARY_END_MARKER}\n\nreal tail message"
|
||||
),
|
||||
COMPRESSED_SUMMARY_METADATA_KEY: True,
|
||||
}
|
||||
|
||||
result = ContextCompressor._strip_context_summary_handoff_message(message)
|
||||
|
||||
assert result == {"role": "user", "content": "real tail message"}
|
||||
|
||||
|
||||
def test_recompression_of_current_merged_handoff_preserves_prior_tail_once():
|
||||
"""Current merged handoffs lose only stale summary data on recompression."""
|
||||
compressor = _compressor()
|
||||
old_summary = "CURRENT-MERGED-OLD-SUMMARY unique continuity facts"
|
||||
prior_tail = "PRESERVED-PRIOR-TAIL real user content"
|
||||
|
||||
with patch.object(
|
||||
compressor,
|
||||
"_generate_summary",
|
||||
return_value=ContextCompressor._with_summary_prefix(
|
||||
"fresh replacement summary"
|
||||
),
|
||||
):
|
||||
result = compressor.compress(
|
||||
_messages_with_merged_handoff(old_summary, prior_tail)
|
||||
)
|
||||
|
||||
joined = "\n".join(str(message.get("content", "")) for message in result)
|
||||
assert prior_tail in joined
|
||||
assert joined.count(prior_tail) == 1
|
||||
assert old_summary not in joined
|
||||
assert joined.count(SUMMARY_PREFIX) == 1
|
||||
assert "fresh replacement summary" in joined
|
||||
|
||||
|
||||
def test_current_multimodal_merged_handoff_preserves_original_blocks():
|
||||
"""Unwrapping current list content must retain text and image blocks."""
|
||||
prior_text = {"type": "text", "text": "real multimodal tail"}
|
||||
prior_image = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,AAAA"},
|
||||
}
|
||||
message = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": f"{_MERGED_PRIOR_CONTEXT_HEADER}\n"},
|
||||
prior_text,
|
||||
prior_image,
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
f"\n\n{_MERGED_SUMMARY_DELIMITER}\n\n"
|
||||
f"{SUMMARY_PREFIX}\nstale summary\n\n{_SUMMARY_END_MARKER}"
|
||||
),
|
||||
},
|
||||
],
|
||||
COMPRESSED_SUMMARY_METADATA_KEY: True,
|
||||
}
|
||||
|
||||
result = ContextCompressor._strip_context_summary_handoff_message(message)
|
||||
|
||||
assert result == {
|
||||
"role": "user",
|
||||
"content": [prior_text, prior_image],
|
||||
}
|
||||
|
||||
|
||||
def test_legacy_multimodal_merged_handoff_preserves_original_blocks():
|
||||
"""Persisted pre-delimiter list handoffs must not lose their real tail."""
|
||||
prior_text = {"type": "text", "text": "legacy real tail"}
|
||||
prior_image = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,BBBB"},
|
||||
}
|
||||
message = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
f"{SUMMARY_PREFIX}\nlegacy stale summary\n\n"
|
||||
f"{_SUMMARY_END_MARKER}\n\n"
|
||||
),
|
||||
},
|
||||
prior_text,
|
||||
prior_image,
|
||||
],
|
||||
COMPRESSED_SUMMARY_METADATA_KEY: True,
|
||||
}
|
||||
|
||||
result = ContextCompressor._strip_context_summary_handoff_message(message)
|
||||
|
||||
assert result == {
|
||||
"role": "user",
|
||||
"content": [prior_text, prior_image],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""config.yaml sessions.* bridges for the search-index knobs (config-authoritative).
|
||||
|
||||
Salvaged from PR #65544 (adapted: agent.fts_v2_read → sessions.cjk_fts).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
import gateway.run as gateway_run
|
||||
|
||||
|
||||
def _write_home(tmp_path: Path, sessions_cfg: dict, env_text: str = "") -> Path:
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
yaml.safe_dump({"sessions": sessions_cfg}), encoding="utf-8"
|
||||
)
|
||||
(hermes_home / ".env").write_text(env_text, encoding="utf-8")
|
||||
return hermes_home
|
||||
|
||||
|
||||
def test_cjk_fts_bridged_from_config(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"cjk_fts": False})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "1")
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_CJK_FTS"] == "False"
|
||||
|
||||
|
||||
def test_search_slow_ms_bridged_from_config(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"search_slow_ms": 250})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.delenv("HERMES_SEARCH_SLOW_MS", raising=False)
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_SEARCH_SLOW_MS"] == "250"
|
||||
|
||||
|
||||
def test_env_survives_when_config_omits_search_knobs(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"auto_prune": False})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "0")
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "700")
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_CJK_FTS"] == "0"
|
||||
assert os.environ["HERMES_SEARCH_SLOW_MS"] == "700"
|
||||
|
||||
|
||||
def test_search_knobs_have_documented_defaults():
|
||||
"""The advertised config surface must exist in DEFAULT_CONFIG (no
|
||||
user-facing env switch): cjk index default ON, slow-search log at 1s."""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
|
||||
assert DEFAULT_CONFIG["sessions"]["cjk_fts"] is True
|
||||
assert DEFAULT_CONFIG["sessions"]["search_slow_ms"] == 1000
|
||||
|
||||
|
||||
def test_config_false_disables_cjk_semantics(tmp_path, monkeypatch):
|
||||
"""The bridged 'False' string must parse as OFF in hermes_state."""
|
||||
from hermes_state import _cjk_fts_config_enabled
|
||||
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "False")
|
||||
assert not _cjk_fts_config_enabled()
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "True")
|
||||
assert _cjk_fts_config_enabled()
|
||||
monkeypatch.delenv("HERMES_CJK_FTS", raising=False)
|
||||
assert _cjk_fts_config_enabled() # default on
|
||||
@@ -1076,3 +1076,37 @@ async def test_safe_sync_detects_contexts_drift():
|
||||
fake_http.edit_global_command.assert_not_awaited()
|
||||
fake_http.delete_global_command.assert_awaited_once_with(999, 77)
|
||||
fake_http.upsert_global_command.assert_awaited_once_with(999, desired)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# #31049: unconfigured platform skips reconnection (non-retryable fatal error)
|
||||
# ============================================================================
|
||||
|
||||
class TestDiscordUnconfiguredNonRetryable:
|
||||
"""Verify that missing dependency/token sets a non-retryable fatal error
|
||||
so the gateway does not queue the platform for background reconnection."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_discord_lib_sets_non_retryable_fatal(self, monkeypatch):
|
||||
"""connect() with discord.py unavailable → non-retryable fatal error."""
|
||||
_ensure_discord_mock()
|
||||
adapter = DiscordAdapter(PlatformConfig(enabled=True, token="fake"))
|
||||
# Simulate discord.py not installed
|
||||
monkeypatch.setattr(discord_platform, "DISCORD_AVAILABLE", False)
|
||||
result = await adapter.connect()
|
||||
assert result is False
|
||||
assert adapter.has_fatal_error is True
|
||||
assert adapter.fatal_error_retryable is False
|
||||
assert adapter.fatal_error_code == "missing_dependency"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_bot_token_sets_non_retryable_fatal(self, monkeypatch):
|
||||
"""connect() with empty token → non-retryable fatal error."""
|
||||
_ensure_discord_mock()
|
||||
monkeypatch.setattr(discord_platform, "DISCORD_AVAILABLE", True)
|
||||
adapter = DiscordAdapter(PlatformConfig(enabled=True, token=""))
|
||||
result = await adapter.connect()
|
||||
assert result is False
|
||||
assert adapter.has_fatal_error is True
|
||||
assert adapter.fatal_error_retryable is False
|
||||
assert adapter.fatal_error_code == "missing_credentials"
|
||||
|
||||
@@ -299,6 +299,49 @@ class TestBuildSessionContextPrompt:
|
||||
assert "pin" in prompt.lower()
|
||||
assert "current message's slack block/attachment payload" in prompt.lower()
|
||||
|
||||
def test_shared_slack_prompt_warns_against_guessed_self_mentions(self):
|
||||
"""Shared Slack threads must instruct the agent to bind mention
|
||||
targets to the current turn's sender prefix (#17916)."""
|
||||
config = GatewayConfig(
|
||||
platforms={
|
||||
Platform.SLACK: PlatformConfig(enabled=True, token="fake"),
|
||||
},
|
||||
)
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="C123",
|
||||
chat_name="team-channel",
|
||||
chat_type="group",
|
||||
user_id="U123",
|
||||
user_name="Alice",
|
||||
thread_id="171.000",
|
||||
)
|
||||
ctx = build_session_context(source, config)
|
||||
prompt = build_session_context_prompt(ctx)
|
||||
|
||||
assert "current turn's sender prefix" in prompt
|
||||
assert "Do not guess or reuse `<@U...>` mentions" in prompt
|
||||
|
||||
def test_non_shared_slack_prompt_omits_self_mention_guidance(self):
|
||||
"""1:1 Slack DMs are single-user: the shared-thread mention guidance
|
||||
must not appear."""
|
||||
config = GatewayConfig(
|
||||
platforms={
|
||||
Platform.SLACK: PlatformConfig(enabled=True, token="fake"),
|
||||
},
|
||||
)
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="D123",
|
||||
chat_type="dm",
|
||||
user_id="U123",
|
||||
user_name="Alice",
|
||||
)
|
||||
ctx = build_session_context(source, config)
|
||||
prompt = build_session_context_prompt(ctx)
|
||||
|
||||
assert "current turn's sender prefix" not in prompt
|
||||
|
||||
def test_discord_prompt_with_channel_topic(self):
|
||||
"""Channel topic should appear in the session context prompt."""
|
||||
config = GatewayConfig(
|
||||
@@ -1571,6 +1614,91 @@ class TestLastPromptTokens:
|
||||
store.update_session("k1", last_prompt_tokens=0)
|
||||
assert entry.last_prompt_tokens == 0
|
||||
|
||||
|
||||
class TestSessionMetadata:
|
||||
"""SessionEntry metadata should persist arbitrary lightweight state."""
|
||||
|
||||
def test_session_entry_metadata_roundtrip(self):
|
||||
from gateway.session import SessionEntry
|
||||
from datetime import datetime
|
||||
|
||||
entry = SessionEntry(
|
||||
session_key="test",
|
||||
session_id="s1",
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
metadata={"slack_thread_watermark:C123:123.000": "123.456"},
|
||||
)
|
||||
|
||||
restored = SessionEntry.from_dict(entry.to_dict())
|
||||
assert restored.metadata == {"slack_thread_watermark:C123:123.000": "123.456"}
|
||||
|
||||
def test_store_session_metadata_get_set(self, tmp_path):
|
||||
"""set/get_session_metadata round-trips through the store and
|
||||
persists via _save (restart survival is provided by the routing
|
||||
index — state.db gateway_routing + sessions.json mirror)."""
|
||||
config = GatewayConfig()
|
||||
with patch("gateway.session.SessionStore._ensure_loaded"):
|
||||
store = SessionStore(sessions_dir=tmp_path, config=config)
|
||||
store._loaded = True
|
||||
store._db = None
|
||||
store._save = MagicMock()
|
||||
|
||||
from gateway.session import SessionEntry
|
||||
from datetime import datetime
|
||||
entry = SessionEntry(
|
||||
session_key="k1",
|
||||
session_id="s1",
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
store._entries = {"k1": entry}
|
||||
|
||||
assert store.set_session_metadata(
|
||||
"k1", "slack_thread_watermark:C123:123.000", "123.456"
|
||||
)
|
||||
store._save.assert_called_once()
|
||||
assert (
|
||||
store.get_session_metadata("k1", "slack_thread_watermark:C123:123.000")
|
||||
== "123.456"
|
||||
)
|
||||
# Missing entry / missing key fall back safely.
|
||||
assert store.set_session_metadata("missing", "k", "v") is False
|
||||
assert store.get_session_metadata("missing", "k", "dflt") == "dflt"
|
||||
assert store.get_session_metadata("k1", "other", "dflt") == "dflt"
|
||||
|
||||
def test_session_metadata_survives_reload(self, tmp_path):
|
||||
"""Metadata written through the store must survive a full reload
|
||||
from disk (simulated gateway restart)."""
|
||||
config = GatewayConfig()
|
||||
store = SessionStore(sessions_dir=tmp_path, config=config)
|
||||
store._db = None # force sessions.json path
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="C123",
|
||||
chat_type="group",
|
||||
user_id="U123",
|
||||
thread_id="123.000",
|
||||
)
|
||||
|
||||
entry = store.get_or_create_session(source)
|
||||
assert store.set_session_metadata(
|
||||
entry.session_key,
|
||||
"slack_thread_watermark:C123:123.000",
|
||||
"123.456",
|
||||
)
|
||||
|
||||
reloaded = SessionStore(sessions_dir=tmp_path, config=config)
|
||||
reloaded._db = None
|
||||
assert (
|
||||
reloaded.get_session_metadata(
|
||||
entry.session_key,
|
||||
"slack_thread_watermark:C123:123.000",
|
||||
)
|
||||
== "123.456"
|
||||
)
|
||||
|
||||
|
||||
class TestRewriteTranscriptPreservesReasoning:
|
||||
"""rewrite_transcript must not drop reasoning fields from SQLite."""
|
||||
|
||||
|
||||
@@ -68,3 +68,64 @@ async def test_preprocess_keeps_plain_text_for_default_group_sessions():
|
||||
)
|
||||
|
||||
assert result == "hello"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preprocess_includes_slack_author_mention_for_shared_thread():
|
||||
"""Shared Slack threads expose the current author's verifiable user ID
|
||||
next to the display name so 'mention me again' requests can bind the
|
||||
mention to the CURRENT speaker (#17916)."""
|
||||
runner = _make_runner(
|
||||
GatewayConfig(
|
||||
platforms={
|
||||
Platform.SLACK: PlatformConfig(enabled=True, token="fake"),
|
||||
},
|
||||
)
|
||||
)
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="C123",
|
||||
chat_name="team-channel",
|
||||
chat_type="group",
|
||||
user_id="U123",
|
||||
user_name="Alice",
|
||||
thread_id="171.000",
|
||||
)
|
||||
event = MessageEvent(text="mention me again", source=source)
|
||||
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event,
|
||||
source=source,
|
||||
history=[],
|
||||
)
|
||||
|
||||
assert result == "[Alice | Slack user <@U123>] mention me again"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preprocess_slack_shared_thread_without_user_id_keeps_name_only():
|
||||
"""No user_id on the source → fall back to the plain name prefix."""
|
||||
runner = _make_runner(
|
||||
GatewayConfig(
|
||||
platforms={
|
||||
Platform.SLACK: PlatformConfig(enabled=True, token="fake"),
|
||||
},
|
||||
)
|
||||
)
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="C123",
|
||||
chat_name="team-channel",
|
||||
chat_type="group",
|
||||
user_name="Alice",
|
||||
thread_id="171.000",
|
||||
)
|
||||
event = MessageEvent(text="hello", source=source)
|
||||
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event,
|
||||
source=source,
|
||||
history=[],
|
||||
)
|
||||
|
||||
assert result == "[Alice] hello"
|
||||
|
||||
@@ -12,6 +12,7 @@ import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch, call
|
||||
|
||||
import pytest
|
||||
@@ -467,6 +468,7 @@ class TestSlackConnectCleanup:
|
||||
)
|
||||
|
||||
second_handler = MagicMock()
|
||||
second_handler.close_async = AsyncMock(return_value=None)
|
||||
# _start_socket_mode_handler awaits the result of start_async via
|
||||
# asyncio.create_task — so the stub must return a real coroutine, not a
|
||||
# bare MagicMock.
|
||||
@@ -489,6 +491,62 @@ class TestSlackConnectCleanup:
|
||||
first_handler.close_async.assert_awaited_once_with()
|
||||
assert adapter._handler is second_handler
|
||||
|
||||
with patch("gateway.status.release_scoped_lock"):
|
||||
await adapter.disconnect()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_closes_workspace_clients_and_clears_runtime_state(self):
|
||||
"""Regression for #51465: shutdown must close Slack WebClients.
|
||||
|
||||
``hermes gateway run --replace`` takes the old process through the
|
||||
normal adapter.disconnect() path. If Slack leaves AsyncWebClient
|
||||
instances open there, aiohttp logs ``Unclosed client session`` while
|
||||
the old gateway exits after SIGTERM.
|
||||
"""
|
||||
config = PlatformConfig(enabled=True, token="xoxb-fake")
|
||||
adapter = SlackAdapter(config)
|
||||
|
||||
socket_task = asyncio.create_task(_pending_for_fake_task())
|
||||
handler = MagicMock()
|
||||
handler.close_async = AsyncMock(return_value=None)
|
||||
|
||||
primary_client = MagicMock()
|
||||
primary_client.close = AsyncMock(return_value=None)
|
||||
team_client = MagicMock()
|
||||
team_client.close = AsyncMock(return_value=None)
|
||||
|
||||
adapter._running = True
|
||||
adapter._handler = handler
|
||||
adapter._socket_mode_task = socket_task
|
||||
adapter._app = MagicMock()
|
||||
adapter._app.client = primary_client
|
||||
adapter._team_clients = {"T_FAKE": team_client}
|
||||
adapter._team_bot_user_ids = {"T_FAKE": "U_BOT"}
|
||||
adapter._channel_team = {"C_FAKE": "T_FAKE"}
|
||||
adapter._platform_lock_scope = "slack-app-token"
|
||||
adapter._platform_lock_identity = "xapp-fake"
|
||||
adapter._app_token = "xapp-fake"
|
||||
adapter._proxy_url = "http://proxy.example.com:3128"
|
||||
adapter._bot_user_id = "U_BOT"
|
||||
|
||||
with patch("gateway.status.release_scoped_lock") as mock_release:
|
||||
await adapter.disconnect()
|
||||
|
||||
handler.close_async.assert_awaited_once_with()
|
||||
primary_client.close.assert_awaited_once_with()
|
||||
team_client.close.assert_awaited_once_with()
|
||||
assert socket_task.cancelled()
|
||||
assert adapter._app is None
|
||||
assert adapter._handler is None
|
||||
assert adapter._socket_mode_task is None
|
||||
assert adapter._team_clients == {}
|
||||
assert adapter._team_bot_user_ids == {}
|
||||
assert adapter._channel_team == {}
|
||||
assert adapter._bot_user_id is None
|
||||
assert adapter._app_token is None
|
||||
assert adapter._proxy_url is None
|
||||
mock_release.assert_called_once_with("slack-app-token", "xapp-fake")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestSlackSocketWatchdog
|
||||
@@ -802,6 +860,85 @@ class TestSlackSocketWatchdog:
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
# -- ping/pong staleness: heals the wedged transport that is_connected() misses --
|
||||
|
||||
def _adapter_with_fake_client(self, **client_attrs):
|
||||
adapter = SlackAdapter(PlatformConfig(enabled=True, token="xoxb-fake"))
|
||||
client = MagicMock()
|
||||
for key, value in client_attrs.items():
|
||||
setattr(client, key, value)
|
||||
adapter._handler = MagicMock(client=client)
|
||||
return adapter
|
||||
|
||||
def test_ping_pong_stale_when_last_ping_old(self):
|
||||
adapter = self._adapter_with_fake_client(
|
||||
ping_interval=30, last_ping_pong_time=time.time() - 1000
|
||||
)
|
||||
assert adapter._socket_ping_pong_stale() is True
|
||||
|
||||
def test_ping_pong_fresh_when_last_ping_recent(self):
|
||||
adapter = self._adapter_with_fake_client(
|
||||
ping_interval=30, last_ping_pong_time=time.time() - 5
|
||||
)
|
||||
assert adapter._socket_ping_pong_stale() is False
|
||||
|
||||
def test_ping_pong_none_within_grace_not_stale(self):
|
||||
adapter = self._adapter_with_fake_client(
|
||||
ping_interval=30, last_ping_pong_time=None
|
||||
)
|
||||
adapter._socket_handler_started_monotonic = time.monotonic()
|
||||
assert adapter._socket_ping_pong_stale() is False
|
||||
|
||||
def test_ping_pong_none_beyond_grace_is_stale(self):
|
||||
adapter = self._adapter_with_fake_client(
|
||||
ping_interval=30, last_ping_pong_time=None
|
||||
)
|
||||
adapter._socket_first_ping_grace_s = 0.0
|
||||
adapter._socket_handler_started_monotonic = time.monotonic() - 200
|
||||
assert adapter._socket_ping_pong_stale() is True
|
||||
|
||||
def test_ping_pong_no_handler_not_stale(self):
|
||||
adapter = SlackAdapter(PlatformConfig(enabled=True, token="xoxb-fake"))
|
||||
adapter._handler = None
|
||||
assert adapter._socket_ping_pong_stale() is False
|
||||
|
||||
def test_ping_pong_nonnumeric_attrs_not_stale(self):
|
||||
# A mocked/partial client (MagicMock attrs) must never trigger reconnect.
|
||||
adapter = SlackAdapter(PlatformConfig(enabled=True, token="xoxb-fake"))
|
||||
adapter._handler = MagicMock()
|
||||
assert adapter._socket_ping_pong_stale() is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_watchdog_reconnects_when_ping_pong_stale_despite_is_connected_true(self):
|
||||
adapter = SlackAdapter(PlatformConfig(enabled=True, token="xoxb-fake"))
|
||||
adapter._socket_watchdog_interval_s = 0.01
|
||||
factory, instances = self._make_fake_handler_factory()
|
||||
|
||||
with contextlib.ExitStack() as stack:
|
||||
for p in self._patch_stack(factory):
|
||||
stack.enter_context(p)
|
||||
|
||||
try:
|
||||
assert await adapter.connect() is True
|
||||
assert len(instances) == 1
|
||||
|
||||
# Transport lies: is_connected() stays True while ping/pong has
|
||||
# gone stale (the wedged "Session is closed" zombie).
|
||||
instances[0].client.is_connected = lambda: True
|
||||
instances[0].client.ping_interval = 30
|
||||
instances[0].client.last_ping_pong_time = time.time() - 1000
|
||||
|
||||
for _ in range(40):
|
||||
if len(instances) >= 2:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
assert len(instances) >= 2, "watchdog did not heal wedged (lying) transport"
|
||||
assert instances[0].closed is True
|
||||
assert adapter._handler is instances[-1]
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestSlackProxyBehavior
|
||||
@@ -1346,6 +1483,55 @@ class TestBangPrefixCommands:
|
||||
# same thread.
|
||||
assert msg_event.source.thread_id == "1111111111.000001"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_queue_survives_first_thread_context_backfill(self, adapter):
|
||||
"""Backfill stays out of command text while remaining available."""
|
||||
adapter._has_active_session_for_thread = MagicMock(return_value=False)
|
||||
adapter._fetch_thread_context = AsyncMock(
|
||||
return_value=(
|
||||
"[Slack thread context — earlier messages]\n"
|
||||
"Alice: prior request\n"
|
||||
"[End of thread context]\n\n"
|
||||
)
|
||||
)
|
||||
adapter._fetch_thread_parent_text = AsyncMock(return_value="prior request")
|
||||
|
||||
evt = self._make_event(
|
||||
"!queue follow up after the current task",
|
||||
thread_ts="1111111111.000001",
|
||||
)
|
||||
await adapter._handle_slack_message(evt)
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "/queue follow up after the current task"
|
||||
assert msg_event.message_type == MessageType.COMMAND
|
||||
assert msg_event.get_command() == "queue"
|
||||
assert msg_event.get_command_args() == "follow up after the current task"
|
||||
assert msg_event.channel_context.startswith("[Slack thread context")
|
||||
assert "prior request" in msg_event.channel_context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_command_thread_backfill_uses_channel_context(self, adapter):
|
||||
"""Normal thread text remains separate without losing its backfill."""
|
||||
adapter._has_active_session_for_thread = MagicMock(return_value=False)
|
||||
adapter._fetch_thread_context = AsyncMock(
|
||||
return_value="[Slack thread context]\nAlice: earlier note\n"
|
||||
)
|
||||
adapter._fetch_thread_parent_text = AsyncMock(return_value="earlier note")
|
||||
|
||||
evt = self._make_event(
|
||||
"follow up",
|
||||
thread_ts="1111111111.000001",
|
||||
)
|
||||
await adapter._handle_slack_message(evt)
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "follow up"
|
||||
assert msg_event.message_type == MessageType.TEXT
|
||||
assert msg_event.channel_context == (
|
||||
"[Slack thread context]\nAlice: earlier note\n"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_unknown_token_passes_through_unchanged(self, adapter):
|
||||
"""``!nice work`` is just a casual message — must NOT be rewritten."""
|
||||
@@ -3334,6 +3520,90 @@ class TestThreadReplyHandling:
|
||||
msg_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "Follow-up question"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_reply_routes_when_parent_mentioned_bot(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""A plain thread reply should route when the thread parent mentioned
|
||||
the bot (#24848) — e.g. parent says '<@bot> check this and ask me
|
||||
before running', a later bare 'run' reply must wake the bot even
|
||||
with no session and no in-memory mention tracking (restart-safe)."""
|
||||
mock_session_store._entries = {}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=False
|
||||
)
|
||||
mock_session_store.get_session_metadata = MagicMock(return_value="")
|
||||
adapter_with_session_store._app.client.conversations_replies = AsyncMock(
|
||||
side_effect=[
|
||||
# _bot_authored_thread_root miss path → full context fetch
|
||||
# (parent is human-authored, so check 4 fails).
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"ts": "123.000",
|
||||
"user": "U_USER",
|
||||
"text": "<@U_BOT> check this and ask me for run",
|
||||
},
|
||||
],
|
||||
},
|
||||
# Any later fetch (cold-start context) reuses cache or refetches.
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"ts": "123.000",
|
||||
"user": "U_USER",
|
||||
"text": "<@U_BOT> check this and ask me for run",
|
||||
},
|
||||
{"ts": "123.456", "user": "U_USER", "text": "run"},
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
adapter_with_session_store._user_name_cache = {("T_TEAM", "U_USER"): "Kai Yi"}
|
||||
|
||||
event = {
|
||||
"text": "run",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.456",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
}
|
||||
await adapter_with_session_store._handle_slack_message(event)
|
||||
|
||||
adapter_with_session_store.handle_message.assert_called_once()
|
||||
msg_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "run"
|
||||
# Cold-start context carries the parent so the agent sees the ask.
|
||||
assert "check this and ask me for run" in msg_event.channel_context
|
||||
# Thread remembered so later replies skip the parent fetch.
|
||||
assert "123.000" in adapter_with_session_store._mentioned_threads
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_top_level_mention_registers_thread_for_replies(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""A TOP-LEVEL @mention starts a thread (session-scoped thread_ts
|
||||
falls back to the message ts); replies to it must auto-trigger, so
|
||||
the synthetic root is registered in _mentioned_threads (#24848)."""
|
||||
mock_session_store._entries = {}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=False
|
||||
)
|
||||
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "<@U_BOT> kick off the deploy checklist",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "555.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
|
||||
adapter_with_session_store.handle_message.assert_called_once()
|
||||
assert "555.000" in adapter_with_session_store._mentioned_threads
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_reply_with_mention_strips_bot_id(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
@@ -3359,6 +3629,168 @@ class TestThreadReplyHandling:
|
||||
assert "<@U_BOT>" not in msg_event.text
|
||||
assert msg_event.text == "thanks for the help"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_active_thread_explicit_mention_refreshes_context_delta(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""Explicit @mention on an active thread must re-fetch the thread and
|
||||
inject only the delta past the stored watermark, as part of the NEW
|
||||
turn (channel_context) — never rewriting prior history (#23918)."""
|
||||
mock_session_store._entries = {"any": MagicMock()}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=True
|
||||
)
|
||||
# Persisted watermark: session has consumed up to 123.100.
|
||||
metadata = {"slack_thread_watermark:C123:123.000": "123.100"}
|
||||
mock_session_store.get_session_metadata = MagicMock(
|
||||
side_effect=lambda sk, k, d=None: metadata.get(k, d)
|
||||
)
|
||||
mock_session_store.set_session_metadata = MagicMock(
|
||||
side_effect=lambda sk, k, v: metadata.__setitem__(k, v) or True
|
||||
)
|
||||
adapter_with_session_store._app.client.conversations_replies = AsyncMock(
|
||||
return_value={
|
||||
"messages": [
|
||||
{"ts": "123.000", "user": "U_PARENT", "text": "Original question"},
|
||||
{"ts": "123.100", "user": "U_USER", "text": "Old context"},
|
||||
{"ts": "123.200", "user": "U_OTHER", "text": "Fresh update"},
|
||||
{"ts": "123.456", "user": "U_USER", "text": "<@U_BOT> what changed?"},
|
||||
]
|
||||
}
|
||||
)
|
||||
adapter_with_session_store._user_name_cache = {
|
||||
("T_TEAM", "U_PARENT"): "Parent",
|
||||
("T_TEAM", "U_USER"): "User",
|
||||
("T_TEAM", "U_OTHER"): "Other",
|
||||
}
|
||||
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "<@U_BOT> what changed?",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.456",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
|
||||
adapter_with_session_store._app.client.conversations_replies.assert_awaited_once()
|
||||
msg_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
# Delta arrives as new-turn channel_context, not baked into text.
|
||||
assert msg_event.text == "what changed?"
|
||||
assert "Fresh update" in msg_event.channel_context
|
||||
# Already-consumed messages must NOT be re-injected.
|
||||
assert "Old context" not in msg_event.channel_context
|
||||
# Watermark advanced to the trigger ts.
|
||||
assert metadata["slack_thread_watermark:C123:123.000"] == "123.456"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_active_thread_unmentioned_reply_does_not_refetch(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""Unmentioned replies in active threads keep the existing behavior:
|
||||
no thread re-fetch, no context injection (once the one-shot restart
|
||||
rehydration check has found no watermark)."""
|
||||
mock_session_store._entries = {"any": MagicMock()}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=True
|
||||
)
|
||||
# No persisted watermark → rehydration check is a no-op.
|
||||
mock_session_store.get_session_metadata = MagicMock(return_value="")
|
||||
adapter_with_session_store._app.client.conversations_replies = AsyncMock()
|
||||
adapter_with_session_store._fetch_thread_parent_text = AsyncMock(
|
||||
return_value=""
|
||||
)
|
||||
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "Follow-up without mention",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.456",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
|
||||
adapter_with_session_store.handle_message.assert_called_once()
|
||||
adapter_with_session_store._app.client.conversations_replies.assert_not_called()
|
||||
msg_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert msg_event.channel_context is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restart_rehydrates_thread_delta_once(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""After a gateway restart (fresh adapter instance, persisted session
|
||||
+ watermark), the FIRST ordinary thread reply injects messages the
|
||||
session missed while the gateway was down — exactly once. Subsequent
|
||||
replies do not re-fetch."""
|
||||
mock_session_store._entries = {"any": MagicMock()}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=True
|
||||
)
|
||||
# Persisted watermark survives the restart via the session store.
|
||||
metadata = {"slack_thread_watermark:C123:123.000": "123.100"}
|
||||
mock_session_store.get_session_metadata = MagicMock(
|
||||
side_effect=lambda sk, k, d=None: metadata.get(k, d)
|
||||
)
|
||||
mock_session_store.set_session_metadata = MagicMock(
|
||||
side_effect=lambda sk, k, v: metadata.__setitem__(k, v) or True
|
||||
)
|
||||
adapter_with_session_store._app.client.conversations_replies = AsyncMock(
|
||||
return_value={
|
||||
"messages": [
|
||||
{"ts": "123.000", "user": "U_PARENT", "text": "Original question"},
|
||||
{"ts": "123.100", "user": "U_USER", "text": "Old context"},
|
||||
{"ts": "123.200", "user": "U_OTHER", "text": "Missed while down"},
|
||||
{"ts": "123.456", "user": "U_USER", "text": "please continue"},
|
||||
]
|
||||
}
|
||||
)
|
||||
adapter_with_session_store._user_name_cache = {
|
||||
("T_TEAM", "U_PARENT"): "Parent",
|
||||
("T_TEAM", "U_USER"): "User",
|
||||
("T_TEAM", "U_OTHER"): "Other",
|
||||
}
|
||||
|
||||
# Fresh adapter instance == empty _thread_rehydration_checked, which
|
||||
# is exactly the post-restart state.
|
||||
assert adapter_with_session_store._thread_rehydration_checked == set()
|
||||
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "please continue",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.456",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
|
||||
first_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert first_event.text == "please continue"
|
||||
assert "Missed while down" in first_event.channel_context
|
||||
assert "Old context" not in first_event.channel_context
|
||||
assert metadata["slack_thread_watermark:C123:123.000"] == "123.456"
|
||||
|
||||
# Second ordinary reply: no re-fetch, no injection.
|
||||
adapter_with_session_store.handle_message.reset_mock()
|
||||
adapter_with_session_store._app.client.conversations_replies.reset_mock()
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "and another thing",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.500",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
adapter_with_session_store._app.client.conversations_replies.assert_not_called()
|
||||
second_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert second_event.channel_context is None
|
||||
# Watermark keeps advancing in steady state.
|
||||
assert metadata["slack_thread_watermark:C123:123.000"] == "123.500"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_top_level_message_requires_mention_even_with_session(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
@@ -5192,3 +5624,64 @@ class TestThreadContextAppMessages:
|
||||
)
|
||||
|
||||
assert "hello" in content # the real message survives; empty bot msg dropped
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Missing-credential handling — fatal-error contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMissingCredentials:
|
||||
"""Missing SLACK_BOT_TOKEN or SLACK_APP_TOKEN must set a non-retryable fatal error."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_bot_token_sets_fatal_error(self):
|
||||
"""When SLACK_BOT_TOKEN is absent from both config and env, connect()
|
||||
must set fatal_error with code 'missing_slack_bot_token' and retryable=False."""
|
||||
config = PlatformConfig(enabled=True, token=None) # no bot token
|
||||
adapter = SlackAdapter(config)
|
||||
|
||||
fatal_errors = []
|
||||
|
||||
def capture_fatal(code, message, *, retryable):
|
||||
fatal_errors.append({"code": code, "message": message, "retryable": retryable})
|
||||
|
||||
with (
|
||||
patch.object(adapter, "_set_fatal_error", side_effect=capture_fatal),
|
||||
patch.dict(os.environ, {}, clear=True),
|
||||
):
|
||||
result = await adapter.connect()
|
||||
|
||||
assert result is False
|
||||
assert len(fatal_errors) == 1
|
||||
assert fatal_errors[0]["code"] == "missing_slack_bot_token"
|
||||
assert fatal_errors[0]["retryable"] is False
|
||||
assert "SLACK_BOT_TOKEN" in fatal_errors[0]["message"]
|
||||
assert "hermes gateway setup" in fatal_errors[0]["message"].lower() or ".env" in fatal_errors[0]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_app_token_sets_fatal_error(self):
|
||||
"""When SLACK_APP_TOKEN is absent but SLACK_BOT_TOKEN is present,
|
||||
connect() must set fatal_error with code 'missing_slack_app_token'
|
||||
and retryable=False."""
|
||||
config = PlatformConfig(enabled=True, token="xoxb-fake")
|
||||
adapter = SlackAdapter(config)
|
||||
|
||||
fatal_errors = []
|
||||
|
||||
def capture_fatal(code, message, *, retryable):
|
||||
fatal_errors.append({"code": code, "message": message, "retryable": retryable})
|
||||
|
||||
with (
|
||||
patch.object(adapter, "_set_fatal_error", side_effect=capture_fatal),
|
||||
patch.dict(os.environ, {"SLACK_BOT_TOKEN": "xoxb-fake"}, clear=True),
|
||||
):
|
||||
result = await adapter.connect()
|
||||
|
||||
assert result is False
|
||||
assert len(fatal_errors) == 1
|
||||
assert fatal_errors[0]["code"] == "missing_slack_app_token"
|
||||
assert fatal_errors[0]["retryable"] is False
|
||||
assert "SLACK_APP_TOKEN" in fatal_errors[0]["message"]
|
||||
assert "hermes gateway setup" in fatal_errors[0]["message"].lower() or ".env" in fatal_errors[0]["message"]
|
||||
|
||||
|
||||
@@ -514,24 +514,28 @@ class TestSlackThreadContext:
|
||||
assert "<@U_BOT>" not in context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_bot_messages(self):
|
||||
"""Self-bot child replies are skipped to avoid circular context,
|
||||
but non-self bots (e.g. cron posts, third-party integrations) are kept.
|
||||
async def test_includes_self_bot_replies_as_assistant_on_cold_start(self):
|
||||
"""Cold-start contract (issue #38861): self-bot replies in the thread
|
||||
must be included in the context, labelled with an ``[assistant]``
|
||||
prefix so the agent can reconstruct its own prior turns. This method
|
||||
only runs on the cold-start path (guarded at the call site by
|
||||
``_has_active_session_for_thread``) — when an active session exists,
|
||||
the session history already carries those replies, so there is no
|
||||
risk of circular duplication.
|
||||
|
||||
Regression guard for the fix in _fetch_thread_context: previously ALL
|
||||
bot messages were dropped, which lost context when the bot was replying
|
||||
to a cron-posted thread parent."""
|
||||
Third-party bots (e.g. deploy notifications) must still be kept,
|
||||
attributed to their display name."""
|
||||
adapter = _make_adapter()
|
||||
mock_client = adapter._team_clients["T1"]
|
||||
mock_client.conversations_replies = AsyncMock(return_value={
|
||||
"messages": [
|
||||
{"ts": "1000.0", "user": "U1", "text": "Parent"},
|
||||
# Self-bot reply -> must be skipped (circular)
|
||||
# Self-bot reply -> kept on cold-start, prefixed [assistant]
|
||||
{
|
||||
"ts": "1000.1",
|
||||
"bot_id": "B_SELF",
|
||||
"user": "U_BOT",
|
||||
"text": "Previous bot self-reply (should be skipped)",
|
||||
"text": "Previous bot self-reply",
|
||||
},
|
||||
# Third-party bot child -> kept (useful context)
|
||||
{
|
||||
@@ -552,10 +556,13 @@ class TestSlackThreadContext:
|
||||
channel_id="C1", thread_ts="1000.0", current_ts="1000.2", team_id="T1"
|
||||
)
|
||||
|
||||
assert "Previous bot self-reply" not in context
|
||||
assert "Alice: Parent" in context
|
||||
# Third-party bot message must now be included
|
||||
# Self-bot reply must now be included with [assistant] label
|
||||
assert "[assistant] Previous bot self-reply" in context
|
||||
# Third-party bot message must still be included
|
||||
assert "Deploy succeeded" in context
|
||||
# The [assistant] label must NOT leak to user messages
|
||||
assert "[assistant] Alice" not in context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_thread(self):
|
||||
@@ -751,14 +758,19 @@ class TestSlackThreadContext:
|
||||
|
||||
assert "Incident triggered" in text
|
||||
assert "https://example.example/incident/42" in text
|
||||
"""Parent (non-self bot) is kept, self-bot child replies are dropped,
|
||||
user replies are kept."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_thread_context_includes_self_bot_replies_with_assistant_label(self):
|
||||
"""Cold-start: parent (non-self bot) kept with [thread parent],
|
||||
self-bot child replies kept with [assistant] label, user replies
|
||||
kept unchanged. The cold-start path is the ONLY caller of this
|
||||
method; circular-context risk does not apply here (see #38861)."""
|
||||
adapter = _make_adapter()
|
||||
mock_client = adapter._team_clients["T1"]
|
||||
mock_client.conversations_replies = AsyncMock(return_value={
|
||||
"messages": [
|
||||
{"ts": "1000.0", "bot_id": "B_CRON", "text": "Cron summary"},
|
||||
# Self-bot child reply -> excluded
|
||||
# Self-bot child reply -> kept with [assistant] label
|
||||
{
|
||||
"ts": "1000.1",
|
||||
"bot_id": "B_SELF",
|
||||
@@ -779,7 +791,8 @@ class TestSlackThreadContext:
|
||||
|
||||
assert "Cron summary" in context
|
||||
assert "[thread parent]" in context
|
||||
assert "Previous self reply" not in context
|
||||
# Self-bot child reply is now kept with the [assistant] label
|
||||
assert "[assistant] Previous self reply" in context
|
||||
assert "Follow-up question" in context
|
||||
assert "Current" not in context
|
||||
|
||||
@@ -808,7 +821,7 @@ class TestSlackThreadContext:
|
||||
"team": "T2",
|
||||
"text": "Cross-workspace bot reply",
|
||||
},
|
||||
# Self-bot for T2 — must be skipped
|
||||
# Self-bot for T2 — kept with the [assistant] label
|
||||
{
|
||||
"ts": "2000.2",
|
||||
"bot_id": "B_SELF_T2",
|
||||
@@ -827,7 +840,12 @@ class TestSlackThreadContext:
|
||||
|
||||
assert "Parent T2" in context
|
||||
assert "Cross-workspace bot reply" in context
|
||||
assert "Own T2 bot reply" not in context
|
||||
# T2's own self-bot reply is kept with the [assistant] label
|
||||
# (cold-start path includes self-replies; see #38861). The
|
||||
# per-workspace filter still applies: this assertion confirms
|
||||
# we use T2's bot id, not T1's, when deciding what counts as
|
||||
# self-bot.
|
||||
assert "[assistant] Own T2 bot reply" in context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_thread_context_current_ts_excluded(self):
|
||||
@@ -933,6 +951,34 @@ class TestSessionKeyFix:
|
||||
)
|
||||
assert result is False
|
||||
|
||||
def test_stale_session_returns_false(self):
|
||||
"""A session key that exists but would be rolled by the reset policy
|
||||
must NOT count as active — otherwise the reset-time first turn skips
|
||||
the thread-history reseed (#55239)."""
|
||||
adapter = _make_adapter()
|
||||
|
||||
class Store:
|
||||
config = MagicMock()
|
||||
config.group_sessions_per_user = False
|
||||
config.thread_sessions_per_user = False
|
||||
_entries = {
|
||||
"agent:main:slack:group:C1:1000.0": MagicMock()
|
||||
}
|
||||
|
||||
def _ensure_loaded(self):
|
||||
return None
|
||||
|
||||
def _should_reset(self, entry, source):
|
||||
return "idle"
|
||||
|
||||
adapter._session_store = Store()
|
||||
|
||||
result = adapter._has_active_session_for_thread(
|
||||
channel_id="C1", thread_ts="1000.0", user_id="U123"
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Thread engagement — bot-started threads & mentioned threads
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Tests for Slack Socket Mode dedup TTL (#4777).
|
||||
|
||||
Slack replays un-acked Socket Mode events when the websocket reconnects.
|
||||
The replay can land several minutes after the original; the dedup window
|
||||
must outlast that gap so the redelivered event is suppressed instead of
|
||||
producing a second bot reply. Regression for the 300s-default bug where
|
||||
replays >5 min later slipped through.
|
||||
|
||||
Follows the slack-bolt mocking pattern from test_slack_mention.py.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
def _ensure_slack_mock():
|
||||
if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"):
|
||||
return
|
||||
|
||||
slack_bolt = MagicMock()
|
||||
slack_bolt.async_app.AsyncApp = MagicMock
|
||||
slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock
|
||||
|
||||
slack_sdk = MagicMock()
|
||||
slack_sdk.web.async_client.AsyncWebClient = MagicMock
|
||||
|
||||
for name, mod in [
|
||||
("slack_bolt", slack_bolt),
|
||||
("slack_bolt.async_app", slack_bolt.async_app),
|
||||
("slack_bolt.adapter", slack_bolt.adapter),
|
||||
("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode),
|
||||
("slack_bolt.adapter.socket_mode.async_handler", slack_bolt.adapter.socket_mode.async_handler),
|
||||
("slack_sdk", slack_sdk),
|
||||
("slack_sdk.web", slack_sdk.web),
|
||||
("slack_sdk.web.async_client", slack_sdk.web.async_client),
|
||||
]:
|
||||
sys.modules.setdefault(name, mod)
|
||||
|
||||
|
||||
_ensure_slack_mock()
|
||||
|
||||
import plugins.platforms.slack.adapter as _slack_mod # noqa: E402
|
||||
|
||||
_slack_mod.SLACK_AVAILABLE = True
|
||||
|
||||
from gateway.platforms.helpers import MessageDeduplicator # noqa: E402
|
||||
from plugins.platforms.slack.adapter import _slack_dedup_ttl_seconds # noqa: E402
|
||||
|
||||
|
||||
def test_default_ttl_outlasts_slack_reconnect_redelivery_window():
|
||||
# The whole point of the fix: the window must be much longer than the
|
||||
# ~6 min reconnect-redelivery gap that caused the duplicate reply.
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
assert _slack_dedup_ttl_seconds() >= 1800.0
|
||||
|
||||
|
||||
def test_env_override_is_respected():
|
||||
with patch.dict(os.environ, {"SLACK_DEDUP_TTL_SECONDS": "120"}, clear=True):
|
||||
assert _slack_dedup_ttl_seconds() == 120.0
|
||||
|
||||
|
||||
def test_invalid_env_falls_back_to_default():
|
||||
with patch.dict(os.environ, {"SLACK_DEDUP_TTL_SECONDS": "not-a-number"}, clear=True):
|
||||
assert _slack_dedup_ttl_seconds() >= 1800.0
|
||||
with patch.dict(os.environ, {"SLACK_DEDUP_TTL_SECONDS": "0"}, clear=True):
|
||||
assert _slack_dedup_ttl_seconds() >= 1800.0
|
||||
|
||||
|
||||
def test_redelivery_six_minutes_later_is_suppressed():
|
||||
"""A replay 6 min after first processing must be treated as duplicate."""
|
||||
dedup = MessageDeduplicator(ttl_seconds=_slack_dedup_ttl_seconds())
|
||||
event_ts = "1733382960.001500"
|
||||
|
||||
# First delivery — recorded now.
|
||||
assert dedup.is_duplicate(event_ts) is False
|
||||
# Simulate the entry being stamped 6 minutes ago (reconnect redelivery gap).
|
||||
dedup._seen[event_ts] = time.time() - 360
|
||||
# Redelivery of the SAME event must still be caught.
|
||||
assert dedup.is_duplicate(event_ts) is True
|
||||
|
||||
|
||||
def test_old_default_300s_would_have_missed_it():
|
||||
"""Pins the regression: the prior 300s window let the replay through."""
|
||||
dedup = MessageDeduplicator(ttl_seconds=300)
|
||||
event_ts = "1733382960.001500"
|
||||
assert dedup.is_duplicate(event_ts) is False
|
||||
dedup._seen[event_ts] = time.time() - 360 # 6 min ago, past 300s TTL
|
||||
# Demonstrates the bug: replay treated as new → second reply.
|
||||
assert dedup.is_duplicate(event_ts) is False
|
||||
@@ -0,0 +1,158 @@
|
||||
"""
|
||||
Tests for Slack inbound mention humanization + bot identity grounding.
|
||||
|
||||
Slack delivers user mentions as opaque IDs (``<@U123>``). Passing those to the
|
||||
agent raw leaves it unable to tell one participant from another — or from
|
||||
itself — so it can misread a mention of a human as a self-mention and reply to
|
||||
messages addressed to that person (the reported "bot thinks it's @someone-else"
|
||||
bug). Two cooperating fixes:
|
||||
|
||||
* ``_humanize_user_mentions`` rewrites ``<@UID>`` → ``@DisplayName`` (the
|
||||
Slack equivalent of Discord's ``clean_content``).
|
||||
* ``_build_identity_prompt`` returns an ephemeral system-prompt line naming
|
||||
the bot's own Slack handle so the agent has a positive "that's me" anchor.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock slack-bolt if not installed (same pattern as test_slack_mention.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _ensure_slack_mock():
|
||||
if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"):
|
||||
return
|
||||
|
||||
slack_bolt = MagicMock()
|
||||
slack_bolt.async_app.AsyncApp = MagicMock
|
||||
slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock
|
||||
|
||||
slack_sdk = MagicMock()
|
||||
slack_sdk.web.async_client.AsyncWebClient = MagicMock
|
||||
|
||||
for name, mod in [
|
||||
("slack_bolt", slack_bolt),
|
||||
("slack_bolt.async_app", slack_bolt.async_app),
|
||||
("slack_bolt.adapter", slack_bolt.adapter),
|
||||
("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode),
|
||||
("slack_bolt.adapter.socket_mode.async_handler",
|
||||
slack_bolt.adapter.socket_mode.async_handler),
|
||||
("slack_sdk", slack_sdk),
|
||||
("slack_sdk.web", slack_sdk.web),
|
||||
("slack_sdk.web.async_client", slack_sdk.web.async_client),
|
||||
]:
|
||||
sys.modules.setdefault(name, mod)
|
||||
sys.modules.setdefault("aiohttp", MagicMock())
|
||||
|
||||
|
||||
_ensure_slack_mock()
|
||||
|
||||
import plugins.platforms.slack.adapter as _slack_mod # noqa: E402
|
||||
_slack_mod.SLACK_AVAILABLE = True
|
||||
|
||||
from plugins.platforms.slack.adapter import SlackAdapter # noqa: E402
|
||||
|
||||
|
||||
def _make_adapter():
|
||||
# object.__new__ skips __init__ (heavy setup) — established slack-test pattern.
|
||||
return object.__new__(SlackAdapter)
|
||||
|
||||
|
||||
def _adapter_with_names(names):
|
||||
"""Adapter whose _resolve_user_name returns from a fixed UID→name map."""
|
||||
adapter = _make_adapter()
|
||||
|
||||
async def _resolve(user_id, chat_id="", team_id=""):
|
||||
return names.get(user_id, user_id)
|
||||
|
||||
adapter._resolve_user_name = _resolve # type: ignore[assignment]
|
||||
return adapter
|
||||
|
||||
|
||||
# ----- _humanize_user_mentions -------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_humanizes_single_mention():
|
||||
adapter = _adapter_with_names({"U07ALICE": "Alice Example"})
|
||||
out = await adapter._humanize_user_mentions(
|
||||
"<@U07ALICE> I think thread is prob the right default", chat_id="C1"
|
||||
)
|
||||
assert out == "@Alice Example I think thread is prob the right default"
|
||||
assert "<@" not in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_humanizes_multiple_distinct_mentions():
|
||||
adapter = _adapter_with_names(
|
||||
{"U07ALICE": "Alice Example", "U07BOB": "Bob Example"}
|
||||
)
|
||||
out = await adapter._humanize_user_mentions(
|
||||
"hey <@U07ALICE> and <@U07BOB>", chat_id="C1"
|
||||
)
|
||||
assert out == "hey @Alice Example and @Bob Example"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_labelled_mention_form():
|
||||
# Slack sometimes sends <@UID|handle>; only the ID drives resolution.
|
||||
adapter = _adapter_with_names({"U07ALICE": "Alice Example"})
|
||||
out = await adapter._humanize_user_mentions("<@U07ALICE|alice> hi", chat_id="C1")
|
||||
assert out == "@Alice Example hi"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_mention_all_replaced():
|
||||
adapter = _adapter_with_names({"U07ALICE": "Alice Example"})
|
||||
out = await adapter._humanize_user_mentions(
|
||||
"<@U07ALICE> ping <@U07ALICE>", chat_id="C1"
|
||||
)
|
||||
assert out == "@Alice Example ping @Alice Example"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unresolvable_mention_falls_back_to_id():
|
||||
# Resolution returns the bare ID; keep the message intact, don't empty it.
|
||||
adapter = _adapter_with_names({})
|
||||
out = await adapter._humanize_user_mentions("<@U07GHOST> hi", chat_id="C1")
|
||||
assert out == "@U07GHOST hi"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_mentions_returns_unchanged():
|
||||
adapter = _adapter_with_names({"U07ALICE": "Alice Example"})
|
||||
out = await adapter._humanize_user_mentions("plain text, no pings", chat_id="C1")
|
||||
assert out == "plain text, no pings"
|
||||
|
||||
|
||||
# ----- _build_identity_prompt --------------------------------------------------
|
||||
|
||||
def test_identity_prompt_names_the_bot():
|
||||
adapter = _make_adapter()
|
||||
adapter._bot_display_name = "TestBot"
|
||||
adapter._team_bot_names = {}
|
||||
prompt = adapter._build_identity_prompt(team_id="T1")
|
||||
assert "@TestBot" in prompt
|
||||
# Must instruct that another participant's mention is not a self-mention.
|
||||
assert "not a mention of you" in prompt
|
||||
|
||||
|
||||
def test_identity_prompt_prefers_per_team_name():
|
||||
adapter = _make_adapter()
|
||||
adapter._bot_display_name = "PrimaryBot"
|
||||
adapter._team_bot_names = {"T2": "WorkspaceTwoBot"}
|
||||
prompt = adapter._build_identity_prompt(team_id="T2")
|
||||
assert "@WorkspaceTwoBot" in prompt
|
||||
assert "PrimaryBot" not in prompt
|
||||
|
||||
|
||||
def test_identity_prompt_empty_when_name_unknown():
|
||||
# Before connect (no name resolved) the prompt must be empty, not a
|
||||
# half-formed line — callers skip injecting an empty string.
|
||||
adapter = _make_adapter()
|
||||
adapter._bot_display_name = None
|
||||
adapter._team_bot_names = {}
|
||||
assert adapter._build_identity_prompt(team_id="T1") == ""
|
||||
@@ -0,0 +1,381 @@
|
||||
"""
|
||||
Tests for Slack Socket Mode teardown (issue #46990).
|
||||
|
||||
slack_sdk's SocketModeClient.connect() is an unconditional retry loop that
|
||||
swallows connection errors and never checks the client's ``closed`` flag. If a
|
||||
task is still inside that loop when the client's shared aiohttp session is
|
||||
closed, it keeps retrying forever and logs
|
||||
``Failed to connect (error: Session is closed); Retrying...`` against a session
|
||||
that can never work again.
|
||||
|
||||
These tests pin the ordering and cleanup that keep old-client background work
|
||||
from outliving a teardown.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock the slack-bolt package if it's not installed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ensure_slack_mock():
|
||||
"""Install mock slack modules so SlackAdapter can be imported."""
|
||||
if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"):
|
||||
return # Real library installed
|
||||
|
||||
slack_bolt = MagicMock()
|
||||
slack_bolt.async_app.AsyncApp = MagicMock
|
||||
slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock
|
||||
|
||||
slack_sdk = MagicMock()
|
||||
slack_sdk.web.async_client.AsyncWebClient = MagicMock
|
||||
|
||||
for name, mod in [
|
||||
("slack_bolt", slack_bolt),
|
||||
("slack_bolt.async_app", slack_bolt.async_app),
|
||||
("slack_bolt.adapter", slack_bolt.adapter),
|
||||
("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode),
|
||||
(
|
||||
"slack_bolt.adapter.socket_mode.async_handler",
|
||||
slack_bolt.adapter.socket_mode.async_handler,
|
||||
),
|
||||
("slack_sdk", slack_sdk),
|
||||
("slack_sdk.web", slack_sdk.web),
|
||||
("slack_sdk.web.async_client", slack_sdk.web.async_client),
|
||||
]:
|
||||
sys.modules.setdefault(name, mod)
|
||||
|
||||
sys.modules.setdefault("aiohttp", MagicMock())
|
||||
|
||||
|
||||
_ensure_slack_mock()
|
||||
|
||||
import plugins.platforms.slack.adapter as _slack_mod # noqa: E402
|
||||
|
||||
_slack_mod.SLACK_AVAILABLE = True
|
||||
|
||||
from plugins.platforms.slack.adapter import SlackAdapter # noqa: E402
|
||||
from gateway.config import PlatformConfig # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal stand-ins for the slack_sdk objects involved in teardown
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Stands in for the ``aiohttp.ClientSession`` SocketModeClient holds."""
|
||||
|
||||
def __init__(self, client=None) -> None:
|
||||
self.closed = False
|
||||
self.reachable = False
|
||||
self.ws_connect_after_close = 0
|
||||
self._client = client
|
||||
self.live_tasks_at_close: list = []
|
||||
|
||||
async def ws_connect(self):
|
||||
if self.closed:
|
||||
# This is the exact failure recorded in #46990.
|
||||
self.ws_connect_after_close += 1
|
||||
raise RuntimeError("Session is closed")
|
||||
if not self.reachable:
|
||||
raise ConnectionError("connection refused")
|
||||
return object()
|
||||
|
||||
async def close(self) -> None:
|
||||
# Record which client tasks were still alive at the instant the shared
|
||||
# session went away. Anything listed here could be inside connect().
|
||||
if self._client is not None:
|
||||
self.live_tasks_at_close = self._client.live_task_names()
|
||||
self.closed = True
|
||||
# Closing a real session performs I/O and yields control back to the
|
||||
# loop, which is what gives a surviving retry task a chance to run.
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
class _FakeSocketModeClient:
|
||||
"""Mirrors the parts of SocketModeClient that matter during teardown."""
|
||||
|
||||
_TASK_ATTRS = ("message_processor", "current_session_monitor", "message_receiver")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.aiohttp_client_session = _FakeSession(self)
|
||||
self.closed = False
|
||||
self.close_should_raise = False
|
||||
self.message_processor = None
|
||||
self.current_session_monitor = None
|
||||
self.message_receiver = None
|
||||
|
||||
def live_task_names(self) -> list:
|
||||
return [
|
||||
attr
|
||||
for attr in self._TASK_ATTRS
|
||||
if getattr(self, attr) is not None and not getattr(self, attr).done()
|
||||
]
|
||||
|
||||
async def connect_to_new_endpoint(self) -> None:
|
||||
# monitor_current_session() (on staleness) and receive_messages() (on a
|
||||
# CLOSE frame) both reach connect() through here, independently.
|
||||
await self.connect()
|
||||
|
||||
async def monitor_current_session(self) -> None:
|
||||
while not self.closed:
|
||||
await asyncio.sleep(0.001)
|
||||
await self.connect_to_new_endpoint()
|
||||
|
||||
async def connect(self) -> None:
|
||||
# Mirrors SocketModeClient.connect(): ``while True`` with a broad
|
||||
# ``except Exception``, so neither the closed flag nor a closed session
|
||||
# ends the loop.
|
||||
while True:
|
||||
try:
|
||||
await self.aiohttp_client_session.ws_connect()
|
||||
return
|
||||
except Exception:
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
async def close(self) -> None:
|
||||
self.closed = True
|
||||
if self.close_should_raise:
|
||||
# SocketModeClient.close() calls disconnect() before it cancels its
|
||||
# background tasks. A broken session makes disconnect() raise, so
|
||||
# the SDK never reaches those cancel() calls at all.
|
||||
raise RuntimeError("Session is closed")
|
||||
for task in (
|
||||
self.message_processor,
|
||||
self.current_session_monitor,
|
||||
self.message_receiver,
|
||||
):
|
||||
if task is not None:
|
||||
# The SDK requests cancellation but never awaits it.
|
||||
task.cancel()
|
||||
await self.aiohttp_client_session.close()
|
||||
|
||||
|
||||
class _FakeHandler:
|
||||
"""Stands in for AsyncSocketModeHandler."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.client = _FakeSocketModeClient()
|
||||
|
||||
async def start_async(self) -> None:
|
||||
await self.client.connect()
|
||||
await asyncio.sleep(float("inf"))
|
||||
|
||||
async def close_async(self) -> None:
|
||||
await self.client.close()
|
||||
|
||||
|
||||
async def _spin() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def adapter():
|
||||
config = PlatformConfig(enabled=True, token="xoxb-fake-token")
|
||||
a = SlackAdapter(config)
|
||||
a._app = MagicMock()
|
||||
a._app_token = "xapp-fake"
|
||||
a._proxy_url = None
|
||||
a._running = True
|
||||
a.handle_message = AsyncMock()
|
||||
return a
|
||||
|
||||
|
||||
def _attach(adapter, handler):
|
||||
"""Wire a handler into the adapter the way _start_socket_mode_handler does."""
|
||||
adapter._handler = handler
|
||||
task = asyncio.create_task(handler.start_async())
|
||||
adapter._socket_mode_task = task
|
||||
return task
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSocketModeTeardown:
|
||||
@pytest.mark.asyncio
|
||||
async def test_socket_task_stops_before_session_is_closed(self, adapter):
|
||||
"""The socket task must be stopped before close_async() kills the session.
|
||||
|
||||
The task is parked in the SDK's connect() retry loop, which is the state
|
||||
#46990 describes. If teardown closes the shared session first, that loop
|
||||
wakes up and retries against a session that is already gone.
|
||||
"""
|
||||
handler = _FakeHandler()
|
||||
task = _attach(adapter, handler)
|
||||
# Let the task settle into the retry loop.
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await adapter._stop_socket_mode_handler()
|
||||
# Give anything that survived a chance to make itself known.
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
session = handler.client.aiohttp_client_session
|
||||
assert session.ws_connect_after_close == 0, (
|
||||
"the old socket task retried against a closed session "
|
||||
f"{session.ws_connect_after_close} time(s) after close_async()"
|
||||
)
|
||||
assert task.done(), "the old socket task outlived teardown"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_background_tasks_do_not_outlive_teardown(self, adapter):
|
||||
"""Old-client background tasks must be cancelled even if close_async() fails.
|
||||
|
||||
SocketModeClient.close() cancels message_processor,
|
||||
current_session_monitor and message_receiver only after disconnect()
|
||||
returns, so a raising disconnect() leaves all three running. The adapter
|
||||
logs and moves on, so it has to clean them up itself.
|
||||
"""
|
||||
handler = _FakeHandler()
|
||||
client = handler.client
|
||||
client.close_should_raise = True
|
||||
client.message_processor = asyncio.create_task(_spin())
|
||||
client.current_session_monitor = asyncio.create_task(_spin())
|
||||
client.message_receiver = asyncio.create_task(_spin())
|
||||
|
||||
_attach(adapter, handler)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await adapter._stop_socket_mode_handler()
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
for name in (
|
||||
"message_processor",
|
||||
"current_session_monitor",
|
||||
"message_receiver",
|
||||
):
|
||||
assert getattr(client, name).done(), f"{name} outlived teardown"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_tasks_are_dead_before_the_session_closes(self, adapter):
|
||||
"""Nothing may still be inside connect() when the shared session closes.
|
||||
|
||||
monitor_current_session() and receive_messages() each reach
|
||||
connect_to_new_endpoint() on their own, and connect() rebinds
|
||||
current_session_monitor and message_receiver to fresh tasks on success.
|
||||
The live task set therefore changes across the awaits inside
|
||||
SocketModeClient.close(), so cancelling from a snapshot taken partway
|
||||
through races a moving target. Everything has to be stopped before the
|
||||
session is closed. See slackapi/python-slack-sdk#1913.
|
||||
"""
|
||||
handler = _FakeHandler()
|
||||
client = handler.client
|
||||
client.message_processor = asyncio.create_task(_spin())
|
||||
client.current_session_monitor = asyncio.create_task(
|
||||
client.monitor_current_session()
|
||||
)
|
||||
client.message_receiver = asyncio.create_task(client.monitor_current_session())
|
||||
|
||||
_attach(adapter, handler)
|
||||
# Let both reconnect loops settle inside connect().
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await adapter._stop_socket_mode_handler()
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
session = client.aiohttp_client_session
|
||||
assert session.live_tasks_at_close == [], (
|
||||
"client tasks were still running when the shared session was closed: "
|
||||
f"{session.live_tasks_at_close}"
|
||||
)
|
||||
assert session.ws_connect_after_close == 0, (
|
||||
"a client task retried against a closed session "
|
||||
f"{session.ws_connect_after_close} time(s)"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_clears_adapter_state(self, adapter):
|
||||
"""Teardown always drops its references, even when close_async() raises."""
|
||||
handler = _FakeHandler()
|
||||
handler.client.close_should_raise = True
|
||||
_attach(adapter, handler)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await adapter._stop_socket_mode_handler()
|
||||
|
||||
assert adapter._handler is None
|
||||
assert adapter._socket_mode_task is None
|
||||
|
||||
|
||||
class TestSocketModeRestart:
|
||||
@pytest.mark.asyncio
|
||||
async def test_restart_stops_old_handler_before_starting_new_one(self, adapter):
|
||||
"""A reconnect must fully retire the old handler before replacing it."""
|
||||
old = _FakeHandler()
|
||||
old_task = _attach(adapter, old)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
started: list[str] = []
|
||||
|
||||
def _fake_start() -> None:
|
||||
started.append("started")
|
||||
assert old_task.done(), (
|
||||
"the replacement handler was created while the old socket task "
|
||||
"was still running"
|
||||
)
|
||||
|
||||
with patch.object(adapter, "_start_socket_mode_handler", _fake_start):
|
||||
await adapter._restart_socket_mode("transport disconnected")
|
||||
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
assert started == ["started"]
|
||||
assert old.client.aiohttp_client_session.ws_connect_after_close == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_watchdog_restarts_when_socket_task_stops(self, adapter):
|
||||
"""The existing watchdog triggers still fire after the teardown change."""
|
||||
done_task = MagicMock()
|
||||
done_task.done.return_value = True
|
||||
adapter._socket_mode_task = done_task
|
||||
|
||||
reasons: list[str] = []
|
||||
|
||||
async def _fake_restart(reason: str) -> None:
|
||||
reasons.append(reason)
|
||||
adapter._running = False
|
||||
|
||||
adapter._restart_socket_mode = _fake_restart
|
||||
adapter._socket_transport_connected = AsyncMock(return_value=None)
|
||||
adapter._socket_watchdog_interval_s = 0.01
|
||||
|
||||
await adapter._socket_watchdog_loop()
|
||||
|
||||
assert reasons == ["socket task stopped"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_watchdog_restarts_when_transport_disconnected(self, adapter):
|
||||
"""A transport that reports itself down still triggers a reconnect."""
|
||||
live_task = MagicMock()
|
||||
live_task.done.return_value = False
|
||||
adapter._socket_mode_task = live_task
|
||||
adapter._handler = MagicMock()
|
||||
|
||||
reasons: list[str] = []
|
||||
|
||||
async def _fake_restart(reason: str) -> None:
|
||||
reasons.append(reason)
|
||||
adapter._running = False
|
||||
|
||||
adapter._restart_socket_mode = _fake_restart
|
||||
adapter._socket_transport_connected = AsyncMock(return_value=False)
|
||||
adapter._socket_watchdog_interval_s = 0.01
|
||||
|
||||
await adapter._socket_watchdog_loop()
|
||||
|
||||
assert reasons == ["transport disconnected"]
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Regression tests for #63530 — Slack adapter drops human replies in
|
||||
threads whose root was posted by the bot via direct chat.postMessage
|
||||
(outside the gateway's send() path).
|
||||
|
||||
Background: the adapter's wake-decision at the un-mentioned branch in
|
||||
_handle_slack_message uses three checks:
|
||||
|
||||
1. thread_ts ∈ _bot_message_ts (only populated by send() / files_upload_v2)
|
||||
2. thread_ts ∈ _mentioned_threads (only populated on @mention)
|
||||
3. _has_active_session_for_thread(...) (survives restarts)
|
||||
|
||||
When a skill posts a triage message into a Slack thread via the Web API
|
||||
directly (chat.postMessage, no gateway run), the bot's own ts is NOT
|
||||
recorded in _bot_message_ts. A human reply in that thread, without an
|
||||
@-mention and without an existing session, falls through all three
|
||||
checks and is silently dropped. The same gap opens after a gateway
|
||||
restart: _bot_message_ts is process memory, so threads the bot started
|
||||
before the restart no longer wake it.
|
||||
|
||||
Fix: a 4th check — was the thread root authored by the bot? Root
|
||||
authorship is derived from the Slack API (conversations.replies), so it
|
||||
survives restarts, unlike the in-memory ts set. The wake decision is
|
||||
extracted into _should_wake_on_unmentioned_message so it's directly
|
||||
testable without spinning up Slack.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# Mock slack-bolt / slack-sdk the same way test_slack_mention.py does.
|
||||
def _ensure_slack_mock():
|
||||
if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"):
|
||||
return
|
||||
slack_bolt = MagicMock()
|
||||
slack_bolt.async_app.AsyncApp = MagicMock
|
||||
slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock
|
||||
slack_sdk = MagicMock()
|
||||
slack_sdk.web.async_client.AsyncWebClient = MagicMock
|
||||
for name, mod in [
|
||||
("slack_bolt", slack_bolt),
|
||||
("slack_bolt.async_app", slack_bolt.async_app),
|
||||
("slack_bolt.adapter", slack_bolt.adapter),
|
||||
("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode),
|
||||
(
|
||||
"slack_bolt.adapter.socket_mode.async_handler",
|
||||
slack_bolt.adapter.socket_mode.async_handler,
|
||||
),
|
||||
("slack_sdk", slack_sdk),
|
||||
("slack_sdk.web", slack_sdk.web),
|
||||
("slack_sdk.web.async_client", slack_sdk.web.async_client),
|
||||
]:
|
||||
sys.modules.setdefault(name, mod)
|
||||
sys.modules.setdefault("aiohttp", MagicMock())
|
||||
|
||||
|
||||
_ensure_slack_mock()
|
||||
|
||||
import plugins.platforms.slack.adapter as _slack_mod # noqa: E402
|
||||
|
||||
_slack_mod.SLACK_AVAILABLE = True
|
||||
|
||||
from plugins.platforms.slack.adapter import ( # noqa: E402
|
||||
SlackAdapter,
|
||||
_ThreadContextCache,
|
||||
)
|
||||
|
||||
from gateway.config import Platform, PlatformConfig # noqa: E402
|
||||
|
||||
|
||||
BOT_USER_ID = "U_BOT_OWN"
|
||||
CHANNEL_ID = "C_incident"
|
||||
USER_ID = "U_engineer"
|
||||
THREAD_TS = "1700000000.000100"
|
||||
|
||||
|
||||
def _make_adapter(bot_authored_root: bool = False):
|
||||
"""Build a bare SlackAdapter with the wake-decision state controlled.
|
||||
|
||||
None of the 3 legacy in-memory checks pass by default: the bot didn't
|
||||
send via gateway, the thread wasn't @-mentioned, and there is no active
|
||||
session — exactly the post-restart / outside-send state.
|
||||
"""
|
||||
adapter = object.__new__(SlackAdapter)
|
||||
adapter.platform = Platform.SLACK
|
||||
adapter.config = PlatformConfig(
|
||||
enabled=True,
|
||||
extra={"require_mention": True, "strict_mention": False},
|
||||
)
|
||||
adapter._bot_user_id = BOT_USER_ID
|
||||
adapter._team_bot_user_ids = {}
|
||||
adapter._bot_message_ts = set()
|
||||
adapter._mentioned_threads = set()
|
||||
adapter._thread_context_cache = {}
|
||||
|
||||
adapter._has_active_session_for_thread = lambda **kw: False
|
||||
# Mock _fetch_thread_context so the miss-path doesn't make a real
|
||||
# Slack API call. Tests that need a populated cache pre-populate
|
||||
# _thread_context_cache directly.
|
||||
adapter._fetch_thread_context = AsyncMock(return_value="")
|
||||
# The 4th-check helper is mocked so wake-decision tests can control
|
||||
# its result without setting up the full cache path. Helper-specific
|
||||
# tests call the real method via the class instead.
|
||||
adapter._bot_authored_thread_root = AsyncMock(return_value=bot_authored_root)
|
||||
|
||||
return adapter
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _should_wake_on_unmentioned_message — composes all 4 checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_false_when_not_thread_reply():
|
||||
"""A top-level channel message (no thread_ts) should never wake the bot
|
||||
when require_mention is true — unchanged by this fix."""
|
||||
adapter = _make_adapter(bot_authored_root=True)
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=None,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=False,
|
||||
)
|
||||
assert wake is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_false_when_all_four_checks_miss():
|
||||
"""All four checks miss (no bot-message, no mention, no session, no
|
||||
bot-authored root) → wake decision is False."""
|
||||
adapter = _make_adapter(bot_authored_root=False)
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_true_when_bot_authored_thread_root():
|
||||
"""The new behavior (#63530): a human reply in a thread whose root was
|
||||
authored by the bot via direct chat.postMessage (outside gateway send)
|
||||
wakes the bot even though none of the legacy 3 checks pass — including
|
||||
after a restart, when _bot_message_ts is empty."""
|
||||
adapter = _make_adapter(bot_authored_root=True)
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is True, (
|
||||
"human reply in a thread whose root was bot-posted (not via gateway "
|
||||
"send) should wake the bot — #63530"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_true_when_legacy_check_1_hits():
|
||||
"""Regression guard: _bot_message_ts hit still wakes (additive check)."""
|
||||
adapter = _make_adapter(bot_authored_root=False)
|
||||
adapter._bot_message_ts = {THREAD_TS}
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_true_when_legacy_check_2_hits():
|
||||
"""Regression guard: _mentioned_threads hit still wakes (additive)."""
|
||||
adapter = _make_adapter(bot_authored_root=False)
|
||||
adapter._mentioned_threads = {THREAD_TS}
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_true_when_legacy_check_3_hits():
|
||||
"""Regression guard: an active session still wakes (additive)."""
|
||||
adapter = _make_adapter(bot_authored_root=False)
|
||||
adapter._has_active_session_for_thread = lambda **kw: True
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _bot_authored_thread_root — the API-derived, restart-surviving check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_true_from_cache():
|
||||
"""Cache hit whose parent_user_id matches the bot's user_id → True."""
|
||||
adapter = _make_adapter()
|
||||
adapter._thread_context_cache = {
|
||||
f"{CHANNEL_ID}:{THREAD_TS}:": _ThreadContextCache(
|
||||
content="[Thread context — prior messages...]",
|
||||
fetched_at=0,
|
||||
message_count=1,
|
||||
parent_text="triage analysis",
|
||||
parent_user_id=BOT_USER_ID,
|
||||
),
|
||||
}
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS
|
||||
)
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_false_for_human_authored_root():
|
||||
"""A human-authored root must return False even on a cache hit — guards
|
||||
against waking on any thread reply just because the cache is warm."""
|
||||
adapter = _make_adapter()
|
||||
adapter._thread_context_cache = {
|
||||
f"{CHANNEL_ID}:{THREAD_TS}:": _ThreadContextCache(
|
||||
content="[Thread context — prior messages...]",
|
||||
fetched_at=0,
|
||||
message_count=1,
|
||||
parent_text="someone else's message",
|
||||
parent_user_id="U_other_user",
|
||||
),
|
||||
}
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS
|
||||
)
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_false_on_empty_thread_ts():
|
||||
"""Defensive: empty thread_ts short-circuits to False without any
|
||||
cache lookup or network call."""
|
||||
adapter = _make_adapter()
|
||||
result = await SlackAdapter._bot_authored_thread_root(adapter, CHANNEL_ID, "")
|
||||
assert result is False
|
||||
adapter._fetch_thread_context.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_fetches_on_cache_miss():
|
||||
"""Cache miss → _fetch_thread_context runs; a successful fetch that
|
||||
populates parent_user_id with the bot's id yields True. This is the
|
||||
restart path: fresh process, empty caches, root authorship recovered
|
||||
from the Slack API."""
|
||||
adapter = _make_adapter()
|
||||
|
||||
async def _fake_fetch(channel_id, thread_ts, current_ts, team_id=""):
|
||||
adapter._thread_context_cache[f"{channel_id}:{thread_ts}:{team_id}"] = (
|
||||
_ThreadContextCache(
|
||||
content="ctx",
|
||||
fetched_at=0,
|
||||
message_count=1,
|
||||
parent_text="bot-posted root",
|
||||
parent_user_id=BOT_USER_ID,
|
||||
)
|
||||
)
|
||||
return "ctx"
|
||||
|
||||
adapter._fetch_thread_context = AsyncMock(side_effect=_fake_fetch)
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS
|
||||
)
|
||||
assert result is True
|
||||
adapter._fetch_thread_context.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_false_when_fetch_fails():
|
||||
"""Fetch failure (empty result, nothing cached) → False, no wake."""
|
||||
adapter = _make_adapter()
|
||||
adapter._fetch_thread_context = AsyncMock(return_value="")
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS
|
||||
)
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_uses_per_team_bot_id():
|
||||
"""Multi-workspace: the comparison must use the team's bot user id,
|
||||
not the primary workspace's."""
|
||||
adapter = _make_adapter()
|
||||
adapter._team_bot_user_ids = {"T2": "U_BOT_T2"}
|
||||
adapter._thread_context_cache = {
|
||||
f"{CHANNEL_ID}:{THREAD_TS}:T2": _ThreadContextCache(
|
||||
content="ctx",
|
||||
fetched_at=0,
|
||||
message_count=1,
|
||||
parent_text="root",
|
||||
parent_user_id="U_BOT_T2",
|
||||
),
|
||||
}
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS, team_id="T2"
|
||||
)
|
||||
assert result is True
|
||||
# And the primary bot id must NOT match in that workspace.
|
||||
adapter._thread_context_cache[f"{CHANNEL_ID}:{THREAD_TS}:T2"].parent_user_id = (
|
||||
BOT_USER_ID
|
||||
)
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS, team_id="T2"
|
||||
)
|
||||
assert result is False
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Tests for Telegram connect() non-retryable fatal error on missing credentials.
|
||||
|
||||
When Telegram has no bot token or no python-telegram-bot installed, connect()
|
||||
must set a non-retryable fatal error so the gateway does not queue it for
|
||||
background reconnection (#31049).
|
||||
"""
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from gateway.config import PlatformConfig
|
||||
|
||||
|
||||
def _ensure_telegram_mock():
|
||||
if "telegram" in sys.modules and hasattr(sys.modules["telegram"], "__file__"):
|
||||
return
|
||||
|
||||
telegram_mod = MagicMock()
|
||||
telegram_mod.ext.ContextTypes.DEFAULT_TYPE = type(None)
|
||||
telegram_mod.constants.ParseMode.MARKDOWN_V2 = "MarkdownV2"
|
||||
telegram_mod.constants.ChatType.GROUP = "group"
|
||||
telegram_mod.constants.ChatType.SUPERGROUP = "supergroup"
|
||||
telegram_mod.constants.ChatType.CHANNEL = "channel"
|
||||
telegram_mod.constants.ChatType.PRIVATE = "private"
|
||||
|
||||
telegram_mod.error.NetworkError = type("NetworkError", (OSError,), {})
|
||||
telegram_mod.error.TimedOut = type("TimedOut", (OSError,), {})
|
||||
telegram_mod.error.BadRequest = type("BadRequest", (Exception,), {})
|
||||
|
||||
for name in ("telegram", "telegram.ext", "telegram.constants", "telegram.request"):
|
||||
sys.modules.setdefault(name, telegram_mod)
|
||||
sys.modules.setdefault("telegram.error", telegram_mod.error)
|
||||
|
||||
|
||||
_ensure_telegram_mock()
|
||||
|
||||
import plugins.platforms.telegram.adapter as telegram_mod # noqa: E402
|
||||
from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: E402
|
||||
|
||||
|
||||
class TestTelegramUnconfiguredNonRetryable:
|
||||
"""Verify that missing dependency/token sets a non-retryable fatal error."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_telegram_lib_sets_non_retryable_fatal(self, monkeypatch):
|
||||
"""connect() with python-telegram-bot unavailable → non-retryable fatal error."""
|
||||
adapter = TelegramAdapter(PlatformConfig(enabled=True, token="fake"))
|
||||
monkeypatch.setattr(telegram_mod, "TELEGRAM_AVAILABLE", False)
|
||||
result = await adapter.connect()
|
||||
assert result is False
|
||||
assert adapter.has_fatal_error is True
|
||||
assert adapter.fatal_error_retryable is False
|
||||
assert adapter.fatal_error_code == "missing_dependency"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_bot_token_sets_non_retryable_fatal(self, monkeypatch):
|
||||
"""connect() with empty token → non-retryable fatal error."""
|
||||
monkeypatch.setattr(telegram_mod, "TELEGRAM_AVAILABLE", True)
|
||||
adapter = TelegramAdapter(PlatformConfig(enabled=True, token=""))
|
||||
result = await adapter.connect()
|
||||
assert result is False
|
||||
assert adapter.has_fatal_error is True
|
||||
assert adapter.fatal_error_retryable is False
|
||||
assert adapter.fatal_error_code == "missing_credentials"
|
||||
@@ -0,0 +1,594 @@
|
||||
"""Tests for the Hermes plugin catalog (hermes_cli.plugin_catalog) and the
|
||||
catalog-driven install/manifest extensions in plugins_cmd.py / plugins.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from hermes_cli.plugin_catalog import (
|
||||
CATALOG_TIERS,
|
||||
PluginCatalogEntry,
|
||||
RemovedEntry,
|
||||
entry_capability_summary,
|
||||
find_removed,
|
||||
get_catalog_dir,
|
||||
get_catalog_entry,
|
||||
load_catalog,
|
||||
load_removed_list,
|
||||
search_catalog,
|
||||
)
|
||||
|
||||
|
||||
VALID_SHA = "38fe0fb53eff98d477f807432e965429e665ca33"
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _write_entry(catalog_dir: Path, name: str, **overrides) -> Path:
|
||||
"""Write a minimal valid catalog entry yaml, applying overrides."""
|
||||
data = {
|
||||
"name": name,
|
||||
"repo": f"https://github.com/example/{name}",
|
||||
"sha": VALID_SHA,
|
||||
"description": f"Test entry {name}.",
|
||||
"maintainer": "Example",
|
||||
}
|
||||
data.update(overrides)
|
||||
catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = catalog_dir / f"{name}.yaml"
|
||||
path.write_text(yaml.safe_dump(data), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _write_removed(catalog_dir: Path, removed: list) -> Path:
|
||||
catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = catalog_dir / "removed.yaml"
|
||||
path.write_text(yaml.safe_dump({"removed": removed}), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def catalog_dir(tmp_path, monkeypatch):
|
||||
d = tmp_path / "catalog"
|
||||
d.mkdir()
|
||||
monkeypatch.setenv("HERMES_PLUGIN_CATALOG_DIR", str(d))
|
||||
return d
|
||||
|
||||
|
||||
# ── get_catalog_dir ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGetCatalogDir:
|
||||
def test_env_override_wins(self, catalog_dir):
|
||||
assert get_catalog_dir() == catalog_dir
|
||||
|
||||
def test_default_is_repo_plugin_catalog(self, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_PLUGIN_CATALOG_DIR", raising=False)
|
||||
d = get_catalog_dir()
|
||||
assert d.name == "plugin-catalog"
|
||||
|
||||
|
||||
# ── load_catalog ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestLoadCatalog:
|
||||
def test_valid_entry_parses(self, catalog_dir):
|
||||
_write_entry(
|
||||
catalog_dir,
|
||||
"my-plugin",
|
||||
tier="official",
|
||||
requires_hermes=">=0.19",
|
||||
subdir="plugins/my-plugin",
|
||||
docs_url="https://example.com/docs",
|
||||
platforms=["linux"],
|
||||
capabilities={
|
||||
"provides_tools": ["my_tool"],
|
||||
"provides_hooks": ["on_start"],
|
||||
"provides_middleware": ["llm_request"],
|
||||
"requires_env": ["MY_API_KEY"],
|
||||
},
|
||||
)
|
||||
entries = load_catalog()
|
||||
assert len(entries) == 1
|
||||
e = entries[0]
|
||||
assert isinstance(e, PluginCatalogEntry)
|
||||
assert e.name == "my-plugin"
|
||||
assert e.repo == "https://github.com/example/my-plugin"
|
||||
assert e.sha == VALID_SHA
|
||||
assert e.tier == "official"
|
||||
assert e.requires_hermes == ">=0.19"
|
||||
assert e.subdir == "plugins/my-plugin"
|
||||
assert e.docs_url == "https://example.com/docs"
|
||||
assert e.platforms == ["linux"]
|
||||
assert e.capabilities.provides_tools == ["my_tool"]
|
||||
assert e.capabilities.provides_hooks == ["on_start"]
|
||||
assert e.capabilities.provides_middleware == ["llm_request"]
|
||||
assert e.capabilities.requires_env == ["MY_API_KEY"]
|
||||
|
||||
def test_tier_defaults_to_community(self, catalog_dir):
|
||||
_write_entry(catalog_dir, "no-tier")
|
||||
(entry,) = load_catalog()
|
||||
assert entry.tier == "community"
|
||||
assert entry.tier in CATALOG_TIERS
|
||||
|
||||
def test_bad_sha_rejected(self, catalog_dir, caplog):
|
||||
_write_entry(catalog_dir, "bad-sha", sha="main")
|
||||
_write_entry(catalog_dir, "short-sha", sha="38fe0fb")
|
||||
_write_entry(catalog_dir, "good", sha=VALID_SHA)
|
||||
with caplog.at_level("WARNING"):
|
||||
entries = load_catalog()
|
||||
assert [e.name for e in entries] == ["good"]
|
||||
|
||||
def test_bad_name_rejected(self, catalog_dir, caplog):
|
||||
_write_entry(catalog_dir, "BadName")
|
||||
_write_entry(catalog_dir, "has spaces")
|
||||
with caplog.at_level("WARNING"):
|
||||
entries = load_catalog()
|
||||
assert entries == []
|
||||
|
||||
def test_non_https_repo_rejected(self, catalog_dir, caplog):
|
||||
_write_entry(catalog_dir, "sshrepo", repo="git@github.com:x/y.git")
|
||||
with caplog.at_level("WARNING"):
|
||||
entries = load_catalog()
|
||||
assert entries == []
|
||||
|
||||
def test_invalid_tier_rejected(self, catalog_dir, caplog):
|
||||
_write_entry(catalog_dir, "weird-tier", tier="platinum")
|
||||
with caplog.at_level("WARNING"):
|
||||
entries = load_catalog()
|
||||
assert entries == []
|
||||
|
||||
def test_removed_yaml_is_not_an_entry(self, catalog_dir):
|
||||
_write_entry(catalog_dir, "real-entry")
|
||||
_write_removed(catalog_dir, [])
|
||||
entries = load_catalog()
|
||||
assert [e.name for e in entries] == ["real-entry"]
|
||||
|
||||
def test_unparseable_yaml_skipped_without_raising(self, catalog_dir, caplog):
|
||||
(catalog_dir / "broken.yaml").write_text(
|
||||
"name: [unclosed", encoding="utf-8"
|
||||
)
|
||||
_write_entry(catalog_dir, "ok-entry")
|
||||
with caplog.at_level("WARNING"):
|
||||
entries = load_catalog()
|
||||
assert [e.name for e in entries] == ["ok-entry"]
|
||||
|
||||
def test_missing_dir_returns_empty(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv(
|
||||
"HERMES_PLUGIN_CATALOG_DIR", str(tmp_path / "does-not-exist")
|
||||
)
|
||||
assert load_catalog() == []
|
||||
|
||||
|
||||
# ── get_catalog_entry / search_catalog ─────────────────────────────────────
|
||||
|
||||
|
||||
class TestLookupAndSearch:
|
||||
def test_get_catalog_entry_by_name(self, catalog_dir):
|
||||
_write_entry(catalog_dir, "alpha")
|
||||
_write_entry(catalog_dir, "beta")
|
||||
entry = get_catalog_entry("beta")
|
||||
assert entry is not None and entry.name == "beta"
|
||||
assert get_catalog_entry("nope") is None
|
||||
|
||||
def test_search_matches_name_case_insensitive(self, catalog_dir):
|
||||
_write_entry(catalog_dir, "weather-tools")
|
||||
_write_entry(catalog_dir, "other")
|
||||
results = search_catalog("WEATHER")
|
||||
assert [e.name for e in results] == ["weather-tools"]
|
||||
|
||||
def test_search_matches_description(self, catalog_dir):
|
||||
_write_entry(catalog_dir, "abc", description="Fetches Stock Quotes.")
|
||||
results = search_catalog("stock")
|
||||
assert [e.name for e in results] == ["abc"]
|
||||
|
||||
def test_search_matches_declared_tools(self, catalog_dir):
|
||||
_write_entry(
|
||||
catalog_dir,
|
||||
"toolful",
|
||||
capabilities={"provides_tools": ["get_forecast"]},
|
||||
)
|
||||
_write_entry(catalog_dir, "toolless")
|
||||
results = search_catalog("Forecast")
|
||||
assert [e.name for e in results] == ["toolful"]
|
||||
|
||||
def test_empty_query_returns_all(self, catalog_dir):
|
||||
_write_entry(catalog_dir, "one")
|
||||
_write_entry(catalog_dir, "two")
|
||||
assert len(search_catalog("")) == 2
|
||||
|
||||
|
||||
# ── removed list ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRemovedList:
|
||||
def test_load_removed_list(self, catalog_dir):
|
||||
_write_removed(
|
||||
catalog_dir,
|
||||
[
|
||||
{
|
||||
"name": "evil-plugin",
|
||||
"repo": "https://github.com/evil/evil-plugin",
|
||||
"reason": "Exfiltrated env vars",
|
||||
"date": "2026-07-02",
|
||||
}
|
||||
],
|
||||
)
|
||||
removed = load_removed_list()
|
||||
assert len(removed) == 1
|
||||
r = removed[0]
|
||||
assert isinstance(r, RemovedEntry)
|
||||
assert r.name == "evil-plugin"
|
||||
assert r.reason == "Exfiltrated env vars"
|
||||
assert r.date == "2026-07-02"
|
||||
|
||||
def test_missing_removed_yaml_returns_empty(self, catalog_dir):
|
||||
assert load_removed_list() == []
|
||||
assert find_removed("anything") is None
|
||||
|
||||
def test_find_removed_by_name(self, catalog_dir):
|
||||
_write_removed(catalog_dir, [{"name": "evil-plugin", "reason": "bad"}])
|
||||
hit = find_removed("evil-plugin")
|
||||
assert hit is not None and hit.reason == "bad"
|
||||
|
||||
def test_find_removed_by_repo_url_with_and_without_git_suffix(
|
||||
self, catalog_dir
|
||||
):
|
||||
_write_removed(
|
||||
catalog_dir,
|
||||
[
|
||||
{
|
||||
"name": "evil-plugin",
|
||||
"repo": "https://github.com/evil/evil-plugin",
|
||||
"reason": "bad",
|
||||
}
|
||||
],
|
||||
)
|
||||
assert find_removed("https://github.com/evil/evil-plugin") is not None
|
||||
assert find_removed("https://github.com/evil/evil-plugin.git") is not None
|
||||
assert find_removed("https://github.com/good/fine.git") is None
|
||||
|
||||
|
||||
# ── entry_capability_summary ───────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCapabilitySummary:
|
||||
def test_summary_contains_declared_capabilities(self):
|
||||
entry = PluginCatalogEntry(
|
||||
name="cap-plugin",
|
||||
repo="https://github.com/example/cap-plugin",
|
||||
sha=VALID_SHA,
|
||||
description="Does capable things.",
|
||||
maintainer="Example",
|
||||
)
|
||||
entry.capabilities.provides_tools = ["tool_a", "tool_b"]
|
||||
entry.capabilities.provides_hooks = ["session_start"]
|
||||
entry.capabilities.requires_env = ["CAP_API_KEY"]
|
||||
summary = entry_capability_summary(entry)
|
||||
assert "tool_a" in summary
|
||||
assert "tool_b" in summary
|
||||
assert "session_start" in summary
|
||||
assert "CAP_API_KEY" in summary
|
||||
|
||||
def test_summary_for_empty_capabilities_mentions_none(self):
|
||||
entry = PluginCatalogEntry(
|
||||
name="plain",
|
||||
repo="https://github.com/example/plain",
|
||||
sha=VALID_SHA,
|
||||
description="Plain.",
|
||||
maintainer="Example",
|
||||
)
|
||||
summary = entry_capability_summary(entry)
|
||||
assert summary # non-empty human text
|
||||
|
||||
|
||||
# ── shipped catalog seed ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestShippedCatalog:
|
||||
def test_shipped_catalog_entries_are_valid(self, monkeypatch):
|
||||
"""Every yaml shipped in <repo>/plugin-catalog must load cleanly."""
|
||||
monkeypatch.delenv("HERMES_PLUGIN_CATALOG_DIR", raising=False)
|
||||
shipped = get_catalog_dir()
|
||||
yaml_files = [
|
||||
p for p in shipped.glob("*.yaml") if p.name != "removed.yaml"
|
||||
]
|
||||
entries = load_catalog()
|
||||
assert len(entries) == len(yaml_files)
|
||||
# removed.yaml must exist and parse
|
||||
assert (shipped / "removed.yaml").exists()
|
||||
load_removed_list()
|
||||
|
||||
|
||||
# ── _version_satisfies ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestVersionSatisfies:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _import(self):
|
||||
from hermes_cli.plugins import _version_satisfies
|
||||
|
||||
self.satisfies = _version_satisfies
|
||||
|
||||
def test_ge(self):
|
||||
assert self.satisfies(">=0.19", "0.19.0") is True
|
||||
assert self.satisfies(">=0.19", "0.20.1") is True
|
||||
assert self.satisfies(">=0.19", "0.18.2") is False
|
||||
|
||||
def test_gt_lt_le(self):
|
||||
assert self.satisfies(">0.19", "0.19.1") is True
|
||||
assert self.satisfies(">0.19", "0.19.0") is False
|
||||
assert self.satisfies("<1.0", "0.19.0") is True
|
||||
assert self.satisfies("<=0.19.0", "0.19.0") is True
|
||||
|
||||
def test_eq_ne(self):
|
||||
assert self.satisfies("==0.19.0", "0.19.0") is True
|
||||
assert self.satisfies("==0.19.0", "0.19.1") is False
|
||||
assert self.satisfies("!=0.19.0", "0.19.1") is True
|
||||
assert self.satisfies("!=0.19.0", "0.19.0") is False
|
||||
|
||||
def test_comma_separated_all_must_hold(self):
|
||||
assert self.satisfies(">=0.10, <1.0", "0.19.0") is True
|
||||
assert self.satisfies(">=0.10, <0.15", "0.19.0") is False
|
||||
|
||||
def test_bare_version_treated_as_ge(self):
|
||||
assert self.satisfies("0.10", "0.19.0") is True
|
||||
assert self.satisfies("999", "0.19.0") is False
|
||||
|
||||
def test_empty_spec_is_satisfied(self):
|
||||
assert self.satisfies("", "0.19.0") is True
|
||||
|
||||
def test_non_numeric_segments_fall_back_permissive(self):
|
||||
assert self.satisfies(">=abc.def", "0.19.0") is True
|
||||
assert self.satisfies(">=0.19", "unknown") is True
|
||||
|
||||
|
||||
# ── requires_hermes manifest gate ──────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_plugin(base: Path, name: str, *, manifest_extra: dict | None = None,
|
||||
register_body: str = "pass", enable: bool = True) -> Path:
|
||||
"""Create a plugin dir under <HERMES_HOME>/plugins and opt it in."""
|
||||
plugin_dir = base / name
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest = {"name": name, "version": "0.1.0", "description": name}
|
||||
if manifest_extra:
|
||||
manifest.update(manifest_extra)
|
||||
(plugin_dir / "plugin.yaml").write_text(yaml.safe_dump(manifest))
|
||||
(plugin_dir / "__init__.py").write_text(
|
||||
f"def register(ctx):\n {register_body}\n"
|
||||
)
|
||||
if enable:
|
||||
hermes_home = Path(os.environ["HERMES_HOME"])
|
||||
cfg_path = hermes_home / "config.yaml"
|
||||
cfg: dict = {}
|
||||
if cfg_path.exists():
|
||||
cfg = yaml.safe_load(cfg_path.read_text()) or {}
|
||||
cfg.setdefault("plugins", {}).setdefault("enabled", []).append(name)
|
||||
cfg_path.write_text(yaml.safe_dump(cfg))
|
||||
return plugin_dir
|
||||
|
||||
|
||||
class TestRequiresHermesGate:
|
||||
def test_unsatisfied_requires_hermes_skips_load(self, monkeypatch):
|
||||
from hermes_cli.plugins import PluginManager
|
||||
|
||||
hermes_home = Path(os.environ["HERMES_HOME"])
|
||||
plugins_dir = hermes_home / "plugins"
|
||||
_make_plugin(
|
||||
plugins_dir, "future_plugin",
|
||||
manifest_extra={"requires_hermes": ">=999.0"},
|
||||
)
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
loaded = mgr._plugins["future_plugin"]
|
||||
assert loaded.enabled is False
|
||||
assert loaded.error is not None
|
||||
assert "requires hermes" in loaded.error
|
||||
assert ">=999.0" in loaded.error
|
||||
assert loaded.module is None # register() never ran
|
||||
|
||||
def test_satisfied_requires_hermes_loads_normally(self, monkeypatch):
|
||||
from hermes_cli.plugins import PluginManager
|
||||
|
||||
hermes_home = Path(os.environ["HERMES_HOME"])
|
||||
plugins_dir = hermes_home / "plugins"
|
||||
_make_plugin(
|
||||
plugins_dir, "old_ok_plugin",
|
||||
manifest_extra={"requires_hermes": ">=0.1"},
|
||||
)
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
loaded = mgr._plugins["old_ok_plugin"]
|
||||
assert loaded.enabled is True
|
||||
assert loaded.error is None
|
||||
|
||||
def test_requires_hermes_parsed_onto_manifest(self):
|
||||
from hermes_cli.plugins import PluginManager
|
||||
|
||||
hermes_home = Path(os.environ["HERMES_HOME"])
|
||||
plugins_dir = hermes_home / "plugins"
|
||||
_make_plugin(
|
||||
plugins_dir, "spec_plugin",
|
||||
manifest_extra={"requires_hermes": ">=0.19"},
|
||||
enable=False,
|
||||
)
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
assert mgr._plugins["spec_plugin"].manifest.requires_hermes == ">=0.19"
|
||||
|
||||
|
||||
# ── config: spec parsing + ctx.plugin_config ───────────────────────────────
|
||||
|
||||
|
||||
class TestPluginConfig:
|
||||
def test_config_spec_parsed_onto_manifest(self):
|
||||
from hermes_cli.plugins import PluginManager
|
||||
|
||||
hermes_home = Path(os.environ["HERMES_HOME"])
|
||||
plugins_dir = hermes_home / "plugins"
|
||||
spec = [
|
||||
{"key": "api_url", "prompt": "API URL", "type": "str",
|
||||
"default": "https://api.example.com", "secret": False},
|
||||
{"key": "token", "prompt": "Token", "type": "str", "secret": True},
|
||||
]
|
||||
_make_plugin(
|
||||
plugins_dir, "cfg_plugin",
|
||||
manifest_extra={"config": spec},
|
||||
enable=False,
|
||||
)
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
manifest = mgr._plugins["cfg_plugin"].manifest
|
||||
assert isinstance(manifest.config_spec, list)
|
||||
assert manifest.config_spec[0]["key"] == "api_url"
|
||||
assert manifest.config_spec[1]["secret"] is True
|
||||
|
||||
def test_plugin_config_merges_defaults_under_config_entries(self):
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
hermes_home = Path(os.environ["HERMES_HOME"])
|
||||
cfg_path = hermes_home / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({
|
||||
"plugins": {"entries": {"merge_plugin": {"api_url": "https://override"}}}
|
||||
}))
|
||||
|
||||
manifest = PluginManifest(
|
||||
name="merge_plugin",
|
||||
key="merge_plugin",
|
||||
config_spec=[
|
||||
{"key": "api_url", "default": "https://default"},
|
||||
{"key": "retries", "type": "int", "default": 3},
|
||||
],
|
||||
)
|
||||
ctx = PluginContext(manifest, PluginManager())
|
||||
cfg = ctx.plugin_config
|
||||
assert cfg["api_url"] == "https://override" # config.yaml wins
|
||||
assert cfg["retries"] == 3 # default fills the gap
|
||||
|
||||
def test_plugin_config_empty_without_spec_or_entries(self):
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(name="bare_plugin", key="bare_plugin")
|
||||
ctx = PluginContext(manifest, PluginManager())
|
||||
assert ctx.plugin_config == {}
|
||||
|
||||
|
||||
# ── _install_plugin_core: ref checkout + removed blocklist ────────────────
|
||||
|
||||
|
||||
def _make_git_repo(tmp_path: Path) -> tuple[Path, str, str]:
|
||||
"""Create a local git repo with two commits; return (path, sha1, sha2)."""
|
||||
repo = tmp_path / "src-repo"
|
||||
repo.mkdir()
|
||||
|
||||
def git(*args):
|
||||
subprocess.run(
|
||||
["git", *args], cwd=repo, check=True, capture_output=True, text=True,
|
||||
env={**os.environ,
|
||||
"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
||||
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"},
|
||||
)
|
||||
|
||||
git("init", "-b", "main")
|
||||
(repo / "plugin.yaml").write_text(
|
||||
yaml.safe_dump({"name": "refplugin", "version": "1"})
|
||||
)
|
||||
(repo / "__init__.py").write_text("def register(ctx):\n pass\n")
|
||||
(repo / "marker.txt").write_text("first\n")
|
||||
git("add", "-A")
|
||||
git("commit", "-m", "first")
|
||||
sha1 = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"], cwd=repo, check=True,
|
||||
capture_output=True, text=True,
|
||||
).stdout.strip()
|
||||
(repo / "marker.txt").write_text("second\n")
|
||||
git("add", "-A")
|
||||
git("commit", "-m", "second")
|
||||
sha2 = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"], cwd=repo, check=True,
|
||||
capture_output=True, text=True,
|
||||
).stdout.strip()
|
||||
return repo, sha1, sha2
|
||||
|
||||
|
||||
class TestInstallPluginCore:
|
||||
def test_ref_checkout_installs_pinned_commit(self, tmp_path, catalog_dir):
|
||||
from hermes_cli.plugins_cmd import _install_plugin_core
|
||||
|
||||
repo, sha1, _sha2 = _make_git_repo(tmp_path)
|
||||
target, manifest, name = _install_plugin_core(
|
||||
f"file://{repo}", force=False, ref=sha1
|
||||
)
|
||||
assert name == "refplugin"
|
||||
assert (target / "marker.txt").read_text() == "first\n"
|
||||
|
||||
def test_default_install_gets_head(self, tmp_path, catalog_dir):
|
||||
from hermes_cli.plugins_cmd import _install_plugin_core
|
||||
|
||||
repo, _sha1, _sha2 = _make_git_repo(tmp_path)
|
||||
target, _manifest, _name = _install_plugin_core(
|
||||
f"file://{repo}", force=False
|
||||
)
|
||||
assert (target / "marker.txt").read_text() == "second\n"
|
||||
|
||||
def test_bad_ref_raises(self, tmp_path, catalog_dir):
|
||||
from hermes_cli.plugins_cmd import PluginOperationError, _install_plugin_core
|
||||
|
||||
repo, _sha1, _sha2 = _make_git_repo(tmp_path)
|
||||
with pytest.raises(PluginOperationError):
|
||||
_install_plugin_core(
|
||||
f"file://{repo}", force=False,
|
||||
ref="0000000000000000000000000000000000000000",
|
||||
)
|
||||
|
||||
def test_removed_repo_blocked(self, tmp_path, catalog_dir):
|
||||
from hermes_cli.plugins_cmd import PluginOperationError, _install_plugin_core
|
||||
|
||||
repo, _sha1, _sha2 = _make_git_repo(tmp_path)
|
||||
_write_removed(
|
||||
catalog_dir,
|
||||
[{
|
||||
"name": "refplugin",
|
||||
"repo": f"file://{repo}",
|
||||
"reason": "exfiltrated env vars",
|
||||
"date": "2026-07-02",
|
||||
}],
|
||||
)
|
||||
with pytest.raises(PluginOperationError, match="exfiltrated env vars"):
|
||||
_install_plugin_core(f"file://{repo}", force=False)
|
||||
|
||||
def test_removed_identifier_blocked_by_name(self, tmp_path, catalog_dir):
|
||||
from hermes_cli.plugins_cmd import PluginOperationError, _install_plugin_core
|
||||
|
||||
_write_removed(
|
||||
catalog_dir,
|
||||
[{"name": "evil-plugin", "reason": "malware", "date": "2026-01-01"}],
|
||||
)
|
||||
with pytest.raises(PluginOperationError, match="malware"):
|
||||
_install_plugin_core("evil-plugin", force=False)
|
||||
|
||||
def test_skip_removed_check_bypasses_block(self, tmp_path, catalog_dir):
|
||||
from hermes_cli.plugins_cmd import _install_plugin_core
|
||||
|
||||
repo, _sha1, _sha2 = _make_git_repo(tmp_path)
|
||||
_write_removed(
|
||||
catalog_dir,
|
||||
[{
|
||||
"name": "refplugin",
|
||||
"repo": f"file://{repo}",
|
||||
"reason": "bad",
|
||||
"date": "2026-07-02",
|
||||
}],
|
||||
)
|
||||
target, _manifest, name = _install_plugin_core(
|
||||
f"file://{repo}", force=False, skip_removed_check=True
|
||||
)
|
||||
assert name == "refplugin"
|
||||
assert target.exists()
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Tests for ``hermes plugins validate`` (hermes_cli/plugin_validate.py).
|
||||
|
||||
Static manifest checks + subprocess-isolated capability probing against a
|
||||
recording stub context.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
import hermes_cli.plugins_cmd as plugins_cmd
|
||||
from hermes_cli.plugin_validate import validate_plugin_dir
|
||||
|
||||
|
||||
def _make_plugin(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
manifest: dict,
|
||||
init_py: str = "def register(ctx):\n pass\n",
|
||||
) -> Path:
|
||||
d = tmp_path / manifest.get("name", "fixture-plugin")
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / "plugin.yaml").write_text(yaml.safe_dump(manifest), encoding="utf-8")
|
||||
(d / "__init__.py").write_text(init_py, encoding="utf-8")
|
||||
return d
|
||||
|
||||
|
||||
BASE_MANIFEST = {
|
||||
"name": "fixture-plugin",
|
||||
"version": "1.0.0",
|
||||
"description": "A fixture plugin.",
|
||||
}
|
||||
|
||||
|
||||
class TestStaticChecks:
|
||||
def test_valid_plugin_passes(self, tmp_path):
|
||||
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST))
|
||||
report = validate_plugin_dir(d)
|
||||
assert report.ok
|
||||
assert report.exit_code == 0
|
||||
|
||||
def test_missing_manifest_fails(self, tmp_path):
|
||||
d = tmp_path / "empty-plugin"
|
||||
d.mkdir()
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
assert report.exit_code == 1
|
||||
assert any("plugin.yaml" in f for f in report.failures)
|
||||
|
||||
def test_missing_required_fields_fail(self, tmp_path):
|
||||
d = _make_plugin(tmp_path, manifest={"name": "fixture-plugin"})
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
joined = " ".join(report.failures)
|
||||
assert "version" in joined
|
||||
assert "description" in joined
|
||||
|
||||
def test_bad_requires_hermes_spec_fails(self, tmp_path):
|
||||
manifest = dict(BASE_MANIFEST, requires_hermes=">=not.a.version")
|
||||
d = _make_plugin(tmp_path, manifest=manifest)
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
assert any("requires_hermes" in f for f in report.failures)
|
||||
|
||||
def test_good_requires_hermes_spec_passes(self, tmp_path):
|
||||
manifest = dict(BASE_MANIFEST, requires_hermes=">=0.1, <99")
|
||||
d = _make_plugin(tmp_path, manifest=manifest)
|
||||
report = validate_plugin_dir(d)
|
||||
assert report.ok
|
||||
|
||||
def test_invalid_config_section_fails(self, tmp_path):
|
||||
manifest = dict(BASE_MANIFEST, config=[{"prompt": "no key here"}])
|
||||
d = _make_plugin(tmp_path, manifest=manifest)
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
assert any("config" in f for f in report.failures)
|
||||
|
||||
def test_valid_config_section_passes(self, tmp_path):
|
||||
manifest = dict(
|
||||
BASE_MANIFEST,
|
||||
config=[
|
||||
{"key": "endpoint", "prompt": "Endpoint?", "type": "str"},
|
||||
{"key": "token", "secret": True, "type": "str"},
|
||||
],
|
||||
)
|
||||
d = _make_plugin(tmp_path, manifest=manifest)
|
||||
report = validate_plugin_dir(d)
|
||||
assert report.ok
|
||||
|
||||
def test_lower_snake_requires_env_fails(self, tmp_path):
|
||||
manifest = dict(BASE_MANIFEST, requires_env=["lower_case_bad"])
|
||||
d = _make_plugin(tmp_path, manifest=manifest)
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
assert any("requires_env" in f for f in report.failures)
|
||||
|
||||
def test_upper_snake_requires_env_passes(self, tmp_path):
|
||||
manifest = dict(BASE_MANIFEST, requires_env=["MY_API_KEY_2"])
|
||||
d = _make_plugin(tmp_path, manifest=manifest)
|
||||
report = validate_plugin_dir(d)
|
||||
assert report.ok
|
||||
|
||||
def test_rich_requires_env_dict_entries_accepted(self, tmp_path):
|
||||
manifest = dict(
|
||||
BASE_MANIFEST,
|
||||
requires_env=[{"name": "MY_KEY", "description": "key"}],
|
||||
)
|
||||
d = _make_plugin(tmp_path, manifest=manifest)
|
||||
report = validate_plugin_dir(d)
|
||||
assert report.ok
|
||||
|
||||
|
||||
class TestCapabilityProbe:
|
||||
def test_undeclared_tool_registration_fails_with_diff(self, tmp_path):
|
||||
init = (
|
||||
"def register(ctx):\n"
|
||||
" ctx.register_tool('sneaky_tool', 'sneaky', {}, lambda a: '')\n"
|
||||
)
|
||||
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST), init_py=init)
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
joined = " ".join(report.failures)
|
||||
assert "sneaky_tool" in joined
|
||||
assert "undeclared" in joined.lower()
|
||||
|
||||
def test_declared_and_registered_passes(self, tmp_path):
|
||||
manifest = dict(BASE_MANIFEST, provides_tools=["good_tool"])
|
||||
init = (
|
||||
"def register(ctx):\n"
|
||||
" ctx.register_tool('good_tool', 'good', {}, lambda a: '')\n"
|
||||
)
|
||||
d = _make_plugin(tmp_path, manifest=manifest, init_py=init)
|
||||
report = validate_plugin_dir(d)
|
||||
assert report.ok
|
||||
|
||||
def test_declared_but_not_registered_warns(self, tmp_path):
|
||||
manifest = dict(BASE_MANIFEST, provides_tools=["phantom_tool"])
|
||||
d = _make_plugin(tmp_path, manifest=manifest)
|
||||
report = validate_plugin_dir(d)
|
||||
assert report.ok # warn, not fail
|
||||
assert any("phantom_tool" in w for w in report.warnings)
|
||||
|
||||
def test_undeclared_hook_registration_fails(self, tmp_path):
|
||||
init = (
|
||||
"def register(ctx):\n"
|
||||
" ctx.register_hook('pre_tool_call', lambda **kw: None)\n"
|
||||
)
|
||||
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST), init_py=init)
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
assert any("pre_tool_call" in f for f in report.failures)
|
||||
|
||||
def test_crashing_register_is_contained(self, tmp_path):
|
||||
init = "def register(ctx):\n raise RuntimeError('boom')\n"
|
||||
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST), init_py=init)
|
||||
report = validate_plugin_dir(d) # must not raise / kill the CLI
|
||||
assert not report.ok
|
||||
assert any("boom" in f or "register()" in f for f in report.failures)
|
||||
|
||||
def test_import_time_os_exit_is_contained(self, tmp_path):
|
||||
init = "import os\nos._exit(7)\n"
|
||||
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST), init_py=init)
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
|
||||
def test_builtin_tool_collision_fails(self, tmp_path):
|
||||
manifest = dict(BASE_MANIFEST, provides_tools=["terminal"])
|
||||
init = (
|
||||
"def register(ctx):\n"
|
||||
" ctx.register_tool('terminal', 'shadow', {}, lambda a: '')\n"
|
||||
)
|
||||
d = _make_plugin(tmp_path, manifest=manifest, init_py=init)
|
||||
report = validate_plugin_dir(d)
|
||||
assert not report.ok
|
||||
joined = " ".join(report.failures)
|
||||
assert "terminal" in joined
|
||||
assert "built-in" in joined
|
||||
|
||||
|
||||
class TestCmdValidate:
|
||||
def test_cmd_validate_exit_zero_on_pass(self, tmp_path, capsys):
|
||||
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST))
|
||||
with pytest.raises(SystemExit) as e:
|
||||
plugins_cmd.cmd_validate(str(d))
|
||||
assert e.value.code == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "✓" in out
|
||||
|
||||
def test_cmd_validate_exit_one_on_fail(self, tmp_path, capsys):
|
||||
d = tmp_path / "not-a-plugin"
|
||||
d.mkdir()
|
||||
with pytest.raises(SystemExit) as e:
|
||||
plugins_cmd.cmd_validate(str(d))
|
||||
assert e.value.code == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "✗" in out
|
||||
|
||||
def test_cmd_validate_json_output(self, tmp_path, capsys):
|
||||
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST))
|
||||
with pytest.raises(SystemExit) as e:
|
||||
plugins_cmd.cmd_validate(str(d), as_json=True)
|
||||
assert e.value.code == 0
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
assert payload["ok"] is True
|
||||
assert "checks" in payload
|
||||
|
||||
def test_cmd_validate_missing_dir_fails(self, tmp_path, capsys):
|
||||
with pytest.raises(SystemExit) as e:
|
||||
plugins_cmd.cmd_validate(str(tmp_path / "ghost"))
|
||||
assert e.value.code == 1
|
||||
@@ -0,0 +1,626 @@
|
||||
"""Tests for the catalog-driven ``hermes plugins`` CLI surface.
|
||||
|
||||
Covers: catalog-name install resolution (pinned ref + provenance sidecar),
|
||||
custom-URL banner, --allow-removed wiring, catalog-pin updates, list
|
||||
annotations, live-index fetch/fallback/TTL, search/browse/info rendering,
|
||||
doctor, and argparse dispatch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
import hermes_cli.plugin_catalog as plugin_catalog
|
||||
import hermes_cli.plugins_cmd as plugins_cmd
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
SHA_A = "a" * 40
|
||||
SHA_B = "b" * 40
|
||||
|
||||
|
||||
# ── Helpers / fixtures ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _write_entry(catalog_dir: Path, name: str, **overrides) -> Path:
|
||||
data = {
|
||||
"name": name,
|
||||
"repo": f"https://github.com/example/{name}",
|
||||
"sha": SHA_A,
|
||||
"description": f"Test entry {name}.",
|
||||
"maintainer": "Example",
|
||||
}
|
||||
data.update(overrides)
|
||||
catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = catalog_dir / f"{name}.yaml"
|
||||
path.write_text(yaml.safe_dump(data), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _write_removed(catalog_dir: Path, removed: list) -> Path:
|
||||
catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = catalog_dir / "removed.yaml"
|
||||
path.write_text(yaml.safe_dump({"removed": removed}), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _install_user_plugin(name: str, *, sidecar: dict | None = None) -> Path:
|
||||
"""Create a fake installed plugin under the per-test HERMES_HOME."""
|
||||
d = get_hermes_home() / "plugins" / name
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / "plugin.yaml").write_text(
|
||||
yaml.safe_dump(
|
||||
{"name": name, "version": "1.0.0", "description": f"{name} plugin"}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
if sidecar is not None:
|
||||
(d / ".hermes-catalog.json").write_text(
|
||||
json.dumps(sidecar), encoding="utf-8"
|
||||
)
|
||||
return d
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def catalog_dir(tmp_path, monkeypatch):
|
||||
d = tmp_path / "catalog"
|
||||
d.mkdir()
|
||||
monkeypatch.setenv("HERMES_PLUGIN_CATALOG_DIR", str(d))
|
||||
return d
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def offline(monkeypatch):
|
||||
"""Force the live-index path to fall back to the in-tree catalog."""
|
||||
monkeypatch.setattr(plugin_catalog, "fetch_live_catalog", lambda **kw: None)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def fake_core(monkeypatch, tmp_path):
|
||||
"""Replace _install_plugin_core with a recording fake."""
|
||||
calls: list[dict] = []
|
||||
target = tmp_path / "fake-installed"
|
||||
|
||||
def fake(identifier, *, force, ref=None, skip_removed_check=False):
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
calls.append(
|
||||
{
|
||||
"identifier": identifier,
|
||||
"force": force,
|
||||
"ref": ref,
|
||||
"skip_removed_check": skip_removed_check,
|
||||
}
|
||||
)
|
||||
return target, {"name": "my-entry"}, "my-entry"
|
||||
|
||||
monkeypatch.setattr(plugins_cmd, "_install_plugin_core", fake)
|
||||
return types.SimpleNamespace(calls=calls, target=target)
|
||||
|
||||
|
||||
# ── Catalog-name install ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCatalogInstall:
|
||||
def test_catalog_name_resolves_to_pinned_repo(
|
||||
self, catalog_dir, offline, fake_core
|
||||
):
|
||||
_write_entry(catalog_dir, "my-entry", sha=SHA_A)
|
||||
plugins_cmd.cmd_install("my-entry", enable=False)
|
||||
assert len(fake_core.calls) == 1
|
||||
call = fake_core.calls[0]
|
||||
assert call["identifier"] == "https://github.com/example/my-entry"
|
||||
assert call["ref"] == SHA_A
|
||||
assert call["skip_removed_check"] is False
|
||||
|
||||
def test_subdir_entry_uses_fragment_identifier(
|
||||
self, catalog_dir, offline, fake_core
|
||||
):
|
||||
_write_entry(catalog_dir, "my-entry", subdir="plugins/inner")
|
||||
plugins_cmd.cmd_install("my-entry", enable=False)
|
||||
assert fake_core.calls[0]["identifier"] == (
|
||||
"https://github.com/example/my-entry#plugins/inner"
|
||||
)
|
||||
|
||||
def test_sidecar_written_with_provenance(
|
||||
self, catalog_dir, offline, fake_core
|
||||
):
|
||||
_write_entry(catalog_dir, "my-entry", tier="official")
|
||||
plugins_cmd.cmd_install("my-entry", enable=False)
|
||||
sidecar_path = fake_core.target / ".hermes-catalog.json"
|
||||
assert sidecar_path.is_file()
|
||||
sidecar = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||
assert sidecar["catalog_name"] == "my-entry"
|
||||
assert sidecar["repo"] == "https://github.com/example/my-entry"
|
||||
assert sidecar["sha"] == SHA_A
|
||||
assert sidecar["tier"] == "official"
|
||||
assert sidecar["installed_at"]
|
||||
|
||||
def test_capability_summary_and_tier_shown(
|
||||
self, catalog_dir, offline, fake_core, capsys
|
||||
):
|
||||
_write_entry(
|
||||
catalog_dir,
|
||||
"my-entry",
|
||||
tier="official",
|
||||
capabilities={"provides_tools": ["cool_tool"]},
|
||||
)
|
||||
plugins_cmd.cmd_install("my-entry", enable=False)
|
||||
out = capsys.readouterr().out
|
||||
assert "official" in out
|
||||
assert "cool_tool" in out
|
||||
|
||||
def test_unknown_catalog_like_name_errors(
|
||||
self, catalog_dir, offline, fake_core, capsys
|
||||
):
|
||||
with pytest.raises(SystemExit):
|
||||
plugins_cmd.cmd_install("nonexistent-entry", enable=False)
|
||||
out = capsys.readouterr().out
|
||||
assert "search" in out
|
||||
assert not fake_core.calls
|
||||
|
||||
def test_custom_url_gets_unreviewed_banner(
|
||||
self, catalog_dir, offline, fake_core, capsys
|
||||
):
|
||||
plugins_cmd.cmd_install(
|
||||
"https://github.com/foo/bar.git", enable=False
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
assert "custom (unreviewed) source" in out
|
||||
# Custom installs never get a ref pin.
|
||||
assert fake_core.calls[0]["ref"] is None
|
||||
|
||||
def test_allow_removed_passes_skip_flag_and_warns(
|
||||
self, catalog_dir, offline, fake_core, capsys
|
||||
):
|
||||
plugins_cmd.cmd_install(
|
||||
"https://github.com/foo/bar.git",
|
||||
enable=False,
|
||||
allow_removed=True,
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
assert fake_core.calls[0]["skip_removed_check"] is True
|
||||
assert "removed" in out.lower()
|
||||
|
||||
|
||||
# ── Catalog update ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCatalogUpdate:
|
||||
def test_update_reinstalls_at_new_pin(
|
||||
self, catalog_dir, offline, fake_core, capsys
|
||||
):
|
||||
_write_entry(catalog_dir, "my-entry", sha=SHA_B)
|
||||
target = _install_user_plugin(
|
||||
"my-entry",
|
||||
sidecar={
|
||||
"catalog_name": "my-entry",
|
||||
"repo": "https://github.com/example/my-entry",
|
||||
"sha": SHA_A,
|
||||
"tier": "community",
|
||||
"installed_at": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
)
|
||||
plugins_cmd.cmd_update("my-entry")
|
||||
assert len(fake_core.calls) == 1
|
||||
call = fake_core.calls[0]
|
||||
assert call["ref"] == SHA_B
|
||||
assert call["force"] is True
|
||||
out = capsys.readouterr().out
|
||||
assert SHA_A[:8] in out
|
||||
assert SHA_B[:8] in out
|
||||
# Sidecar refreshed to the new pin (written into the reinstall target).
|
||||
sidecar = json.loads(
|
||||
(fake_core.target / ".hermes-catalog.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
assert sidecar["sha"] == SHA_B
|
||||
assert target.exists() or True # target replaced by reinstall
|
||||
|
||||
def test_update_already_at_pin_is_noop(
|
||||
self, catalog_dir, offline, fake_core, capsys
|
||||
):
|
||||
_write_entry(catalog_dir, "my-entry", sha=SHA_A)
|
||||
_install_user_plugin(
|
||||
"my-entry",
|
||||
sidecar={
|
||||
"catalog_name": "my-entry",
|
||||
"repo": "https://github.com/example/my-entry",
|
||||
"sha": SHA_A,
|
||||
"tier": "community",
|
||||
"installed_at": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
)
|
||||
plugins_cmd.cmd_update("my-entry")
|
||||
out = capsys.readouterr().out
|
||||
assert "already at catalog pin" in out
|
||||
assert not fake_core.calls
|
||||
|
||||
def test_update_preserves_enabled_state(
|
||||
self, catalog_dir, offline, fake_core
|
||||
):
|
||||
_write_entry(catalog_dir, "my-entry", sha=SHA_B)
|
||||
_install_user_plugin(
|
||||
"my-entry",
|
||||
sidecar={
|
||||
"catalog_name": "my-entry",
|
||||
"repo": "https://github.com/example/my-entry",
|
||||
"sha": SHA_A,
|
||||
"tier": "community",
|
||||
"installed_at": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
)
|
||||
plugins_cmd._save_enabled_set({"my-entry"})
|
||||
plugins_cmd.cmd_update("my-entry")
|
||||
assert "my-entry" in plugins_cmd._get_enabled_set()
|
||||
|
||||
def test_update_without_sidecar_keeps_git_flow(
|
||||
self, catalog_dir, offline, fake_core, capsys
|
||||
):
|
||||
_install_user_plugin("plain-git-plugin") # no sidecar, no .git
|
||||
with pytest.raises(SystemExit):
|
||||
plugins_cmd.cmd_update("plain-git-plugin")
|
||||
out = capsys.readouterr().out
|
||||
assert "not installed from git" in out
|
||||
assert not fake_core.calls
|
||||
|
||||
def test_update_entry_gone_from_catalog_errors(
|
||||
self, catalog_dir, offline, fake_core, capsys
|
||||
):
|
||||
_install_user_plugin(
|
||||
"my-entry",
|
||||
sidecar={
|
||||
"catalog_name": "my-entry",
|
||||
"repo": "https://github.com/example/my-entry",
|
||||
"sha": SHA_A,
|
||||
"tier": "community",
|
||||
"installed_at": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
plugins_cmd.cmd_update("my-entry")
|
||||
out = capsys.readouterr().out
|
||||
assert "no longer in the catalog" in out
|
||||
|
||||
|
||||
# ── List annotations ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestListAnnotations:
|
||||
def test_json_includes_catalog_annotation(
|
||||
self, catalog_dir, offline, capsys
|
||||
):
|
||||
_install_user_plugin(
|
||||
"cat-plugin",
|
||||
sidecar={
|
||||
"catalog_name": "cat-plugin",
|
||||
"repo": "https://github.com/example/cat-plugin",
|
||||
"sha": SHA_A,
|
||||
"tier": "official",
|
||||
"installed_at": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
)
|
||||
args = argparse.Namespace(json=True)
|
||||
plugins_cmd.cmd_list(args)
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
row = next(p for p in payload if p["name"] == "cat-plugin")
|
||||
assert row["catalog"] == f"catalog:official@{SHA_A[:8]}"
|
||||
|
||||
def test_removed_plugin_flagged(self, catalog_dir, offline, capsys):
|
||||
_install_user_plugin("evil-plugin")
|
||||
_write_removed(
|
||||
catalog_dir,
|
||||
[{"name": "evil-plugin", "reason": "exfiltrated env vars"}],
|
||||
)
|
||||
plugins_cmd.cmd_list(argparse.Namespace())
|
||||
out = capsys.readouterr().out
|
||||
assert "REMOVED from catalog" in out
|
||||
assert "exfiltrated env vars" in out
|
||||
|
||||
|
||||
# ── Live index fetch ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _FakeResp:
|
||||
def __init__(self, *, json_data=None, text=""):
|
||||
self._json = json_data
|
||||
self.text = text
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def json(self):
|
||||
return self._json
|
||||
|
||||
|
||||
def _fake_httpx_get(listing, files, counter):
|
||||
def fake_get(url, **kwargs):
|
||||
counter.append(url)
|
||||
if "api.github.com" in url:
|
||||
return _FakeResp(json_data=listing)
|
||||
fname = url.rsplit("/", 1)[-1]
|
||||
return _FakeResp(text=files[fname])
|
||||
|
||||
return fake_get
|
||||
|
||||
|
||||
class TestLiveIndex:
|
||||
def _remote_entry_yaml(self, name, sha=SHA_B):
|
||||
return yaml.safe_dump(
|
||||
{
|
||||
"name": name,
|
||||
"repo": f"https://github.com/example/{name}",
|
||||
"sha": sha,
|
||||
"description": f"Remote entry {name}.",
|
||||
"maintainer": "Example",
|
||||
}
|
||||
)
|
||||
|
||||
def test_live_fetch_populates_cache_and_entries(
|
||||
self, catalog_dir, monkeypatch
|
||||
):
|
||||
_write_entry(catalog_dir, "local-entry")
|
||||
listing = [
|
||||
{
|
||||
"name": "remote-entry.yaml",
|
||||
"download_url": "https://raw.example/remote-entry.yaml",
|
||||
},
|
||||
]
|
||||
files = {"remote-entry.yaml": self._remote_entry_yaml("remote-entry")}
|
||||
counter: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"httpx.get", _fake_httpx_get(listing, files, counter)
|
||||
)
|
||||
entries = plugin_catalog.load_catalog_live()
|
||||
names = [e.name for e in entries]
|
||||
assert names == ["remote-entry"]
|
||||
cache = get_hermes_home() / "cache" / "plugin-catalog"
|
||||
assert (cache / "remote-entry.yaml").is_file()
|
||||
|
||||
def test_network_failure_falls_back_to_in_tree(
|
||||
self, catalog_dir, monkeypatch
|
||||
):
|
||||
_write_entry(catalog_dir, "local-entry")
|
||||
|
||||
def boom(url, **kwargs):
|
||||
raise OSError("no network")
|
||||
|
||||
monkeypatch.setattr("httpx.get", boom)
|
||||
entries = plugin_catalog.load_catalog_live()
|
||||
assert [e.name for e in entries] == ["local-entry"]
|
||||
|
||||
def test_ttl_cache_skips_refetch(self, catalog_dir, monkeypatch):
|
||||
listing = [
|
||||
{
|
||||
"name": "remote-entry.yaml",
|
||||
"download_url": "https://raw.example/remote-entry.yaml",
|
||||
},
|
||||
]
|
||||
files = {"remote-entry.yaml": self._remote_entry_yaml("remote-entry")}
|
||||
counter: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"httpx.get", _fake_httpx_get(listing, files, counter)
|
||||
)
|
||||
plugin_catalog.load_catalog_live()
|
||||
first_count = len(counter)
|
||||
assert first_count >= 2 # listing + file
|
||||
|
||||
# Second call within TTL must not hit the network at all — even if
|
||||
# the network is now broken.
|
||||
def boom(url, **kwargs):
|
||||
raise AssertionError("network hit despite fresh cache")
|
||||
|
||||
monkeypatch.setattr("httpx.get", boom)
|
||||
entries = plugin_catalog.load_catalog_live()
|
||||
assert [e.name for e in entries] == ["remote-entry"]
|
||||
|
||||
|
||||
# ── search / browse / info ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSearchBrowseInfo:
|
||||
def test_search_filters_entries(self, catalog_dir, offline, capsys):
|
||||
_write_entry(catalog_dir, "alpha-entry")
|
||||
_write_entry(catalog_dir, "beta-entry")
|
||||
plugins_cmd.cmd_search("alpha")
|
||||
out = capsys.readouterr().out
|
||||
assert "alpha-entry" in out
|
||||
assert "beta-entry" not in out
|
||||
|
||||
def test_browse_lists_all(self, catalog_dir, offline, capsys):
|
||||
_write_entry(catalog_dir, "alpha-entry")
|
||||
_write_entry(catalog_dir, "beta-entry")
|
||||
plugins_cmd.cmd_browse()
|
||||
out = capsys.readouterr().out
|
||||
assert "alpha-entry" in out
|
||||
assert "beta-entry" in out
|
||||
|
||||
def test_search_no_results_message(self, catalog_dir, offline, capsys):
|
||||
plugins_cmd.cmd_search("zzz-nothing")
|
||||
out = capsys.readouterr().out
|
||||
assert "No catalog entries" in out
|
||||
|
||||
def test_info_shows_full_detail(self, catalog_dir, offline, capsys):
|
||||
_write_entry(
|
||||
catalog_dir,
|
||||
"alpha-entry",
|
||||
tier="official",
|
||||
requires_hermes=">=0.19",
|
||||
docs_url="https://example.com/docs",
|
||||
platforms=["linux"],
|
||||
capabilities={
|
||||
"provides_tools": ["cool_tool"],
|
||||
"requires_env": ["ALPHA_KEY"],
|
||||
},
|
||||
)
|
||||
plugins_cmd.cmd_info("alpha-entry")
|
||||
out = capsys.readouterr().out
|
||||
assert SHA_A in out
|
||||
assert "official" in out
|
||||
assert "cool_tool" in out
|
||||
assert "ALPHA_KEY" in out
|
||||
assert ">=0.19" in out
|
||||
assert "hermes plugins install alpha-entry" in out
|
||||
|
||||
def test_info_unknown_entry_exits(self, catalog_dir, offline, capsys):
|
||||
with pytest.raises(SystemExit):
|
||||
plugins_cmd.cmd_info("ghost-entry")
|
||||
|
||||
def test_info_warns_when_removed(self, catalog_dir, offline, capsys):
|
||||
_write_entry(catalog_dir, "alpha-entry")
|
||||
_write_removed(
|
||||
catalog_dir,
|
||||
[{"name": "alpha-entry", "reason": "bad actor"}],
|
||||
)
|
||||
plugins_cmd.cmd_info("alpha-entry")
|
||||
out = capsys.readouterr().out
|
||||
assert "REMOVED" in out
|
||||
assert "bad actor" in out
|
||||
|
||||
|
||||
# ── doctor ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestDoctor:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_runtime_scan(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
plugins_cmd, "_runtime_load_errors", lambda: {}
|
||||
)
|
||||
|
||||
def test_doctor_table_lists_installed_plugin(
|
||||
self, catalog_dir, offline, capsys
|
||||
):
|
||||
_install_user_plugin(
|
||||
"doc-plugin",
|
||||
sidecar={
|
||||
"catalog_name": "doc-plugin",
|
||||
"repo": "https://github.com/example/doc-plugin",
|
||||
"sha": SHA_A,
|
||||
"tier": "official",
|
||||
"installed_at": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
)
|
||||
_write_entry(catalog_dir, "doc-plugin", sha=SHA_A, tier="official")
|
||||
plugins_cmd.cmd_doctor()
|
||||
out = capsys.readouterr().out
|
||||
assert "doc-plugin" in out
|
||||
assert "official" in out
|
||||
|
||||
def test_doctor_detail_flags_pin_mismatch(
|
||||
self, catalog_dir, offline, capsys
|
||||
):
|
||||
_install_user_plugin(
|
||||
"doc-plugin",
|
||||
sidecar={
|
||||
"catalog_name": "doc-plugin",
|
||||
"repo": "https://github.com/example/doc-plugin",
|
||||
"sha": SHA_A,
|
||||
"tier": "official",
|
||||
"installed_at": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
)
|
||||
_write_entry(catalog_dir, "doc-plugin", sha=SHA_B)
|
||||
plugins_cmd.cmd_doctor("doc-plugin")
|
||||
out = capsys.readouterr().out
|
||||
assert "doc-plugin" in out
|
||||
assert "behind catalog pin" in out or "pin mismatch" in out
|
||||
|
||||
def test_doctor_flags_removed_plugin(self, catalog_dir, offline, capsys):
|
||||
_install_user_plugin("evil-plugin")
|
||||
_write_removed(
|
||||
catalog_dir,
|
||||
[{"name": "evil-plugin", "reason": "exfiltrated env vars"}],
|
||||
)
|
||||
plugins_cmd.cmd_doctor("evil-plugin")
|
||||
out = capsys.readouterr().out
|
||||
assert "REMOVED" in out
|
||||
|
||||
def test_doctor_unknown_plugin_exits(self, catalog_dir, offline, capsys):
|
||||
with pytest.raises(SystemExit):
|
||||
plugins_cmd.cmd_doctor("no-such-plugin")
|
||||
|
||||
|
||||
# ── argparse dispatch ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestDispatch:
|
||||
def _dispatch(self, monkeypatch, action, **attrs):
|
||||
recorded = {}
|
||||
|
||||
def record(fn_name):
|
||||
def _rec(*args, **kwargs):
|
||||
recorded["fn"] = fn_name
|
||||
recorded["args"] = args
|
||||
recorded["kwargs"] = kwargs
|
||||
|
||||
return _rec
|
||||
|
||||
for fn in (
|
||||
"cmd_search",
|
||||
"cmd_browse",
|
||||
"cmd_info",
|
||||
"cmd_validate",
|
||||
"cmd_doctor",
|
||||
"cmd_install",
|
||||
):
|
||||
monkeypatch.setattr(plugins_cmd, fn, record(fn))
|
||||
ns = argparse.Namespace(plugins_action=action, **attrs)
|
||||
plugins_cmd.plugins_command(ns)
|
||||
return recorded
|
||||
|
||||
def test_search_dispatch(self, monkeypatch):
|
||||
rec = self._dispatch(monkeypatch, "search", query="foo")
|
||||
assert rec["fn"] == "cmd_search"
|
||||
assert "foo" in rec["args"] or rec["kwargs"].get("query") == "foo"
|
||||
|
||||
def test_browse_dispatch(self, monkeypatch):
|
||||
rec = self._dispatch(monkeypatch, "browse")
|
||||
assert rec["fn"] == "cmd_browse"
|
||||
|
||||
def test_info_dispatch(self, monkeypatch):
|
||||
rec = self._dispatch(monkeypatch, "info", name="foo")
|
||||
assert rec["fn"] == "cmd_info"
|
||||
|
||||
def test_validate_dispatch(self, monkeypatch):
|
||||
rec = self._dispatch(monkeypatch, "validate", path="/tmp/x", json=True)
|
||||
assert rec["fn"] == "cmd_validate"
|
||||
|
||||
def test_doctor_dispatch(self, monkeypatch):
|
||||
rec = self._dispatch(monkeypatch, "doctor", name=None)
|
||||
assert rec["fn"] == "cmd_doctor"
|
||||
|
||||
def test_install_allow_removed_dispatch(self, monkeypatch):
|
||||
rec = self._dispatch(
|
||||
monkeypatch,
|
||||
"install",
|
||||
identifier="x",
|
||||
force=False,
|
||||
enable=False,
|
||||
no_enable=True,
|
||||
allow_removed=True,
|
||||
)
|
||||
assert rec["fn"] == "cmd_install"
|
||||
assert rec["kwargs"].get("allow_removed") is True
|
||||
|
||||
def test_parser_wires_new_subcommands(self):
|
||||
from hermes_cli.subcommands.plugins import build_plugins_parser
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="command")
|
||||
build_plugins_parser(sub, cmd_plugins=lambda args: None)
|
||||
for argv in (
|
||||
["plugins", "search", "foo"],
|
||||
["plugins", "browse"],
|
||||
["plugins", "info", "foo"],
|
||||
["plugins", "validate", "/tmp/x", "--json"],
|
||||
["plugins", "doctor"],
|
||||
["plugins", "install", "foo", "--allow-removed"],
|
||||
):
|
||||
args = parser.parse_args(argv)
|
||||
assert args.plugins_action == argv[1]
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Tests for the dashboard plugin-catalog surface in hermes_cli.web_server.
|
||||
|
||||
Covers:
|
||||
- GET /api/dashboard/plugins/catalog — entry serialization, installed-state
|
||||
merge (via the ``.hermes-catalog.json`` sidecar), removed list exposure.
|
||||
- POST /api/dashboard/agent-plugins/install — removed-blocklist refusal for
|
||||
raw identifiers AND catalog names, catalog_name resolution to a pinned-ref
|
||||
install, sidecar write.
|
||||
- /api/dashboard/plugins/hub — ``removed_reason`` annotation on rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
VALID_SHA = "38fe0fb53eff98d477f807432e965429e665ca33"
|
||||
OTHER_SHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
|
||||
def _write_entry(catalog_dir: Path, name: str, **overrides) -> dict:
|
||||
data = {
|
||||
"name": name,
|
||||
"repo": f"https://github.com/example/{name}",
|
||||
"sha": VALID_SHA,
|
||||
"description": f"Test entry {name}.",
|
||||
"maintainer": "Example",
|
||||
"tier": "official",
|
||||
"docs_url": f"https://example.com/docs/{name}",
|
||||
"capabilities": {
|
||||
"provides_tools": ["tool_a"],
|
||||
"provides_hooks": ["hook_b"],
|
||||
"provides_middleware": [],
|
||||
"requires_env": ["EXAMPLE_API_KEY"],
|
||||
},
|
||||
}
|
||||
data.update(overrides)
|
||||
catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
(catalog_dir / f"{name}.yaml").write_text(
|
||||
yaml.safe_dump(data), encoding="utf-8"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def _write_removed(catalog_dir: Path, removed: list) -> None:
|
||||
catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
(catalog_dir / "removed.yaml").write_text(
|
||||
yaml.safe_dump({"removed": removed}), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def _make_installed_plugin(name: str, sidecar: dict | None = None) -> Path:
|
||||
"""Drop a minimal plugin dir under the isolated HERMES_HOME."""
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
plugin_dir = get_hermes_home() / "plugins" / name
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_dir / "plugin.yaml").write_text(
|
||||
yaml.safe_dump({"name": name, "version": "1.0", "description": "x"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
if sidecar is not None:
|
||||
(plugin_dir / ".hermes-catalog.json").write_text(
|
||||
json.dumps(sidecar), encoding="utf-8"
|
||||
)
|
||||
return plugin_dir
|
||||
|
||||
|
||||
class TestDashboardPluginCatalog:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _setup(self, monkeypatch, tmp_path, _isolate_hermes_home):
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip("fastapi/starlette not installed")
|
||||
|
||||
import hermes_state
|
||||
from hermes_constants import get_hermes_home
|
||||
from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN
|
||||
|
||||
monkeypatch.setattr(
|
||||
hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db"
|
||||
)
|
||||
|
||||
self.catalog_dir = tmp_path / "catalog"
|
||||
self.catalog_dir.mkdir()
|
||||
monkeypatch.setenv("HERMES_PLUGIN_CATALOG_DIR", str(self.catalog_dir))
|
||||
|
||||
self.client = TestClient(app)
|
||||
self.client.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN
|
||||
|
||||
# ── GET /api/dashboard/plugins/catalog ──────────────────────────────
|
||||
|
||||
def test_catalog_endpoint_requires_token(self):
|
||||
from starlette.testclient import TestClient
|
||||
from hermes_cli.web_server import app
|
||||
|
||||
unauth = TestClient(app)
|
||||
resp = unauth.get("/api/dashboard/plugins/catalog")
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_catalog_endpoint_shape(self):
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
_write_removed(
|
||||
self.catalog_dir,
|
||||
[{"name": "bad-plugin", "repo": "https://github.com/evil/bad-plugin",
|
||||
"reason": "exfiltrated env vars", "date": "2026-07-02"}],
|
||||
)
|
||||
|
||||
resp = self.client.get("/api/dashboard/plugins/catalog")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
|
||||
assert "generated_at" in data
|
||||
assert isinstance(data["entries"], list) and len(data["entries"]) == 1
|
||||
entry = data["entries"][0]
|
||||
assert entry["name"] == "alpha-plugin"
|
||||
assert entry["repo"] == "https://github.com/example/alpha-plugin"
|
||||
assert entry["sha"] == VALID_SHA
|
||||
assert entry["sha_short"] == VALID_SHA[:7]
|
||||
assert entry["tier"] == "official"
|
||||
assert entry["maintainer"] == "Example"
|
||||
assert entry["docs_url"] == "https://example.com/docs/alpha-plugin"
|
||||
assert entry["capabilities"]["provides_tools"] == ["tool_a"]
|
||||
assert entry["capabilities"]["requires_env"] == ["EXAMPLE_API_KEY"]
|
||||
assert "tool_a" in entry["capability_summary"]
|
||||
# Not installed → degraded install state.
|
||||
assert entry["installed"] is False
|
||||
assert entry["installed_sha"] is None
|
||||
assert entry["update_available"] is False
|
||||
assert entry["runtime_status"] is None
|
||||
|
||||
assert len(data["removed"]) == 1
|
||||
removed = data["removed"][0]
|
||||
assert removed["name"] == "bad-plugin"
|
||||
assert removed["reason"] == "exfiltrated env vars"
|
||||
|
||||
def test_catalog_installed_state_merge_with_sidecar(self):
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
_make_installed_plugin(
|
||||
"alpha-plugin",
|
||||
sidecar={
|
||||
"catalog_name": "alpha-plugin",
|
||||
"repo": "https://github.com/example/alpha-plugin",
|
||||
"sha": OTHER_SHA,
|
||||
"installed_at": "2026-07-01T00:00:00Z",
|
||||
"tier": "official",
|
||||
},
|
||||
)
|
||||
|
||||
resp = self.client.get("/api/dashboard/plugins/catalog")
|
||||
assert resp.status_code == 200
|
||||
entry = resp.json()["entries"][0]
|
||||
assert entry["installed"] is True
|
||||
assert entry["installed_sha"] == OTHER_SHA
|
||||
assert entry["update_available"] is True
|
||||
assert entry["runtime_status"] == "inactive"
|
||||
|
||||
def test_catalog_installed_no_sidecar_degrades_to_null_sha(self):
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
_make_installed_plugin("alpha-plugin", sidecar=None)
|
||||
|
||||
resp = self.client.get("/api/dashboard/plugins/catalog")
|
||||
entry = resp.json()["entries"][0]
|
||||
assert entry["installed"] is True
|
||||
assert entry["installed_sha"] is None
|
||||
assert entry["update_available"] is False
|
||||
|
||||
def test_catalog_installed_same_sha_no_update(self):
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
_make_installed_plugin(
|
||||
"alpha-plugin",
|
||||
sidecar={
|
||||
"catalog_name": "alpha-plugin",
|
||||
"repo": "https://github.com/example/alpha-plugin",
|
||||
"sha": VALID_SHA,
|
||||
"installed_at": "2026-07-01T00:00:00Z",
|
||||
"tier": "official",
|
||||
},
|
||||
)
|
||||
|
||||
entry = self.client.get("/api/dashboard/plugins/catalog").json()["entries"][0]
|
||||
assert entry["installed"] is True
|
||||
assert entry["installed_sha"] == VALID_SHA
|
||||
assert entry["update_available"] is False
|
||||
|
||||
# ── POST /api/dashboard/agent-plugins/install ───────────────────────
|
||||
|
||||
def test_install_refuses_removed_raw_identifier(self):
|
||||
_write_removed(
|
||||
self.catalog_dir,
|
||||
[{"name": "bad-plugin", "repo": "https://github.com/evil/bad-plugin",
|
||||
"reason": "exfiltrated env vars", "date": "2026-07-02"}],
|
||||
)
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": "https://github.com/evil/bad-plugin"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "exfiltrated env vars" in resp.json()["detail"]
|
||||
|
||||
def test_install_refuses_removed_catalog_name(self):
|
||||
_write_removed(
|
||||
self.catalog_dir,
|
||||
[{"name": "bad-plugin", "reason": "policy violation",
|
||||
"date": "2026-07-02"}],
|
||||
)
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": "", "catalog_name": "bad-plugin"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "policy violation" in resp.json()["detail"]
|
||||
|
||||
def test_install_unknown_catalog_name_is_400(self):
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": "", "catalog_name": "does-not-exist"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "does-not-exist" in resp.json()["detail"]
|
||||
|
||||
def test_install_missing_identifier_and_catalog_name_is_400(self):
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": ""},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_catalog_name_install_resolves_pinned_ref_and_writes_sidecar(
|
||||
self, monkeypatch, tmp_path
|
||||
):
|
||||
from hermes_constants import get_hermes_home
|
||||
import hermes_cli.plugins_cmd as plugins_cmd
|
||||
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_core(identifier, *, force, ref=None, skip_removed_check=False):
|
||||
captured["identifier"] = identifier
|
||||
captured["ref"] = ref
|
||||
target = get_hermes_home() / "plugins" / "alpha-plugin"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
(target / "plugin.yaml").write_text(
|
||||
yaml.safe_dump({"name": "alpha-plugin"}), encoding="utf-8"
|
||||
)
|
||||
return target, {"name": "alpha-plugin"}, "alpha-plugin"
|
||||
|
||||
monkeypatch.setattr(plugins_cmd, "_install_plugin_core", fake_core)
|
||||
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": "", "catalog_name": "alpha-plugin",
|
||||
"enable": False},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["ok"] is True
|
||||
assert body["plugin_name"] == "alpha-plugin"
|
||||
|
||||
assert captured["ref"] == VALID_SHA
|
||||
assert captured["identifier"].startswith(
|
||||
"https://github.com/example/alpha-plugin"
|
||||
)
|
||||
|
||||
sidecar_path = (
|
||||
get_hermes_home() / "plugins" / "alpha-plugin" / ".hermes-catalog.json"
|
||||
)
|
||||
assert sidecar_path.is_file()
|
||||
sidecar = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||
assert sidecar["catalog_name"] == "alpha-plugin"
|
||||
assert sidecar["repo"] == "https://github.com/example/alpha-plugin"
|
||||
assert sidecar["sha"] == VALID_SHA
|
||||
assert sidecar["tier"] == "official"
|
||||
assert sidecar["installed_at"]
|
||||
|
||||
# ── /api/dashboard/plugins/hub removed_reason annotation ─────────────
|
||||
|
||||
def test_hub_rows_annotated_with_removed_reason(self):
|
||||
_write_removed(
|
||||
self.catalog_dir,
|
||||
[{"name": "bad-plugin", "reason": "supply chain incident",
|
||||
"date": "2026-07-02"}],
|
||||
)
|
||||
_make_installed_plugin("bad-plugin")
|
||||
_make_installed_plugin("good-plugin")
|
||||
|
||||
resp = self.client.get("/api/dashboard/plugins/hub")
|
||||
assert resp.status_code == 200
|
||||
rows = {r["name"]: r for r in resp.json()["plugins"]}
|
||||
assert rows["bad-plugin"]["removed_reason"] == "supply chain incident"
|
||||
assert rows["good-plugin"]["removed_reason"] is None
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Behavior tests for scripts/validate_plugin_catalog.py.
|
||||
|
||||
The script is the no-install structural validator used by the plugin-catalog
|
||||
admission CI: it must run with only stdlib + pyyaml, take file paths or a
|
||||
directory, exit 0/1, and support --json machine output. These tests exercise
|
||||
the CLI contract via subprocess (the same way CI invokes it).
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = REPO_ROOT / "scripts" / "validate_plugin_catalog.py"
|
||||
|
||||
VALID_ENTRY = {
|
||||
"name": "example-plugin",
|
||||
"repo": "https://github.com/NousResearch/hermes-example-plugins",
|
||||
"sha": "38fe0fb53eff98d477f807432e965429e665ca33",
|
||||
"subdir": "",
|
||||
"description": "One-line description.",
|
||||
"maintainer": "NousResearch",
|
||||
"tier": "official",
|
||||
"requires_hermes": ">=0.19",
|
||||
"docs_url": "",
|
||||
"platforms": [],
|
||||
"capabilities": {
|
||||
"provides_tools": ["example_tool"],
|
||||
"provides_hooks": [],
|
||||
"provides_middleware": [],
|
||||
"requires_env": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_entry(tmp_path: Path, data: dict, filename: str | None = None) -> Path:
|
||||
name = filename or f"{data.get('name', 'entry')}.yaml"
|
||||
path = tmp_path / name
|
||||
path.write_text(yaml.safe_dump(data), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def run_validator(*args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(SCRIPT), *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
# ── valid input ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_valid_entry_passes(tmp_path):
|
||||
path = write_entry(tmp_path, VALID_ENTRY)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_valid_entry_without_optional_fields_passes(tmp_path):
|
||||
entry = {
|
||||
"name": "minimal-plugin",
|
||||
"repo": "https://github.com/example/minimal",
|
||||
"sha": "a" * 40,
|
||||
"description": "Minimal.",
|
||||
"maintainer": "someone",
|
||||
}
|
||||
path = write_entry(tmp_path, entry)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
# ── each malformed field fails with a pointed error ────────────────────
|
||||
|
||||
|
||||
def _expect_error(tmp_path, mutation: dict, expected_substring: str, drop: str = ""):
|
||||
entry = {**VALID_ENTRY, **mutation}
|
||||
if drop:
|
||||
entry.pop(drop, None)
|
||||
path = write_entry(tmp_path, entry, filename="entry.yaml")
|
||||
result = run_validator(str(path))
|
||||
combined = result.stdout + result.stderr
|
||||
assert result.returncode == 1, combined
|
||||
assert expected_substring in combined, combined
|
||||
assert "entry.yaml" in combined, combined
|
||||
|
||||
|
||||
def test_bad_name_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"name": "Bad Name!"}, "name")
|
||||
|
||||
|
||||
def test_name_too_long_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"name": "x" * 65}, "name")
|
||||
|
||||
|
||||
def test_non_https_repo_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"repo": "git@github.com:evil/x.git"}, "repo")
|
||||
|
||||
|
||||
def test_short_sha_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"sha": "abc123"}, "sha")
|
||||
|
||||
|
||||
def test_non_hex_sha_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"sha": "z" * 40}, "sha")
|
||||
|
||||
|
||||
def test_bad_tier_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"tier": "platinum"}, "tier")
|
||||
|
||||
|
||||
def test_empty_description_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"description": ""}, "description")
|
||||
|
||||
|
||||
def test_empty_maintainer_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"maintainer": ""}, "maintainer")
|
||||
|
||||
|
||||
def test_missing_required_field_fails(tmp_path):
|
||||
_expect_error(tmp_path, {}, "sha", drop="sha")
|
||||
|
||||
|
||||
def test_capabilities_value_not_a_list_fails(tmp_path):
|
||||
_expect_error(
|
||||
tmp_path,
|
||||
{"capabilities": {"provides_tools": "not-a-list"}},
|
||||
"provides_tools",
|
||||
)
|
||||
|
||||
|
||||
def test_capabilities_list_of_non_strings_fails(tmp_path):
|
||||
_expect_error(
|
||||
tmp_path,
|
||||
{"capabilities": {"requires_env": [1, 2]}},
|
||||
"requires_env",
|
||||
)
|
||||
|
||||
|
||||
def test_bad_requires_hermes_spec_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"requires_hermes": "banana"}, "requires_hermes")
|
||||
|
||||
|
||||
def test_comma_separated_requires_hermes_passes(tmp_path):
|
||||
entry = {**VALID_ENTRY, "requires_hermes": ">=0.19, <2.0"}
|
||||
path = write_entry(tmp_path, entry)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_unknown_platform_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"platforms": ["linux", "amiga"]}, "platforms")
|
||||
|
||||
|
||||
def test_entry_not_a_mapping_fails(tmp_path):
|
||||
path = tmp_path / "entry.yaml"
|
||||
path.write_text("- just\n- a\n- list\n", encoding="utf-8")
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 1
|
||||
assert "mapping" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
# ── unknown top-level keys warn but do not fail ────────────────────────
|
||||
|
||||
|
||||
def test_unknown_key_warns_but_passes(tmp_path):
|
||||
entry = {**VALID_ENTRY, "future_field": "hello"}
|
||||
path = write_entry(tmp_path, entry)
|
||||
result = run_validator(str(path))
|
||||
combined = result.stdout + result.stderr
|
||||
assert result.returncode == 0, combined
|
||||
assert "future_field" in combined
|
||||
assert "warning" in combined.lower()
|
||||
|
||||
|
||||
# ── removed.yaml shape ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_valid_removed_yaml_passes(tmp_path):
|
||||
path = tmp_path / "removed.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"removed": [
|
||||
{
|
||||
"name": "some-plugin",
|
||||
"repo": "https://github.com/evil/some-plugin",
|
||||
"reason": "Exfiltrated env vars",
|
||||
"date": "2026-07-02",
|
||||
}
|
||||
]
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_removed_yaml_not_a_list_fails(tmp_path):
|
||||
path = tmp_path / "removed.yaml"
|
||||
path.write_text(yaml.safe_dump({"removed": "nope"}), encoding="utf-8")
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 1
|
||||
assert "removed" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
def test_removed_item_missing_name_fails(tmp_path):
|
||||
path = tmp_path / "removed.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump({"removed": [{"reason": "bad", "date": "2026-01-01"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 1
|
||||
assert "name" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
# ── --json machine output ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_json_output_shape_on_failure(tmp_path):
|
||||
bad = write_entry(tmp_path, {**VALID_ENTRY, "sha": "short"}, filename="bad.yaml")
|
||||
result = run_validator("--json", str(bad))
|
||||
assert result.returncode == 1
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["ok"] is False
|
||||
assert isinstance(payload["files"], list)
|
||||
entry = next(f for f in payload["files"] if f["path"].endswith("bad.yaml"))
|
||||
assert entry["ok"] is False
|
||||
assert any("sha" in e for e in entry["errors"])
|
||||
|
||||
|
||||
def test_json_output_shape_on_success_with_warning(tmp_path):
|
||||
good = write_entry(tmp_path, {**VALID_ENTRY, "future_field": 1})
|
||||
result = run_validator("--json", str(good))
|
||||
assert result.returncode == 0
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["ok"] is True
|
||||
(entry,) = payload["files"]
|
||||
assert entry["ok"] is True
|
||||
assert entry["errors"] == []
|
||||
assert any("future_field" in w for w in entry["warnings"])
|
||||
|
||||
|
||||
# ── directory mode ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_directory_mode_validates_all_entries_and_removed(tmp_path):
|
||||
write_entry(tmp_path, VALID_ENTRY)
|
||||
write_entry(tmp_path, {**VALID_ENTRY, "name": "bad-one", "sha": "nope"})
|
||||
(tmp_path / "removed.yaml").write_text(
|
||||
yaml.safe_dump({"removed": [{"name": "gone", "reason": "test"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = run_validator(str(tmp_path))
|
||||
combined = result.stdout + result.stderr
|
||||
assert result.returncode == 1
|
||||
assert "bad-one.yaml" in combined
|
||||
# the valid entry and removed.yaml must not produce errors
|
||||
assert combined.count("ERROR") == combined.count("bad-one.yaml: ERROR")
|
||||
|
||||
|
||||
def test_directory_mode_all_valid_exits_zero(tmp_path):
|
||||
write_entry(tmp_path, VALID_ENTRY)
|
||||
(tmp_path / "removed.yaml").write_text(
|
||||
yaml.safe_dump({"removed": []}), encoding="utf-8"
|
||||
)
|
||||
result = run_validator(str(tmp_path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
@@ -0,0 +1,325 @@
|
||||
"""Tests for the messages_fts_cjk CJK-bigram index (salvaged from PR #65544).
|
||||
|
||||
Builds the loadable tokenizer from native/fts5_cjk/fts5_cjk.c on the fly;
|
||||
skips when no C toolchain / extension loading is available.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import FTS_CJK_STALE_KEY, SessionDB
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SRC = REPO / "native" / "fts5_cjk" / "fts5_cjk.c"
|
||||
VENDOR = REPO / "native" / "fts5_cjk" / "vendor"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cjk_so(tmp_path_factory):
|
||||
if shutil.which("gcc") is None or not SRC.exists():
|
||||
pytest.skip("no C toolchain / tokenizer source")
|
||||
out = tmp_path_factory.mktemp("fts5cjk") / "libfts5_cjk.so"
|
||||
try:
|
||||
subprocess.run(
|
||||
["gcc", "-shared", "-fPIC", "-O2", f"-I{VENDOR}", str(SRC),
|
||||
"-o", str(out)],
|
||||
check=True, capture_output=True, text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
pytest.skip(f"tokenizer build failed: {e.stderr[:200]}")
|
||||
# Loadability probe (extension loading may be disabled in this build).
|
||||
probe = sqlite3.connect(":memory:")
|
||||
try:
|
||||
probe.enable_load_extension(True)
|
||||
probe.load_extension(str(out))
|
||||
except Exception as e:
|
||||
pytest.skip(f"extension loading unavailable: {e}")
|
||||
finally:
|
||||
probe.close()
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(cjk_so, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
assert d._fts_cjk_loaded, "tokenizer must load on the writer connection"
|
||||
assert d._fts_cjk_available, "fresh DB must be born with the cjk index"
|
||||
d.create_session(session_id="s1", source="cli", model="m")
|
||||
d.append_message("s1", role="user", content="웅기가 shared default 프로필을 요청했다")
|
||||
d.append_message("s1", role="assistant", content="일본 MCP 후보 우선순위 정리했습니다")
|
||||
d.append_message("s1", role="user", content="graphiti daemon looks healthy")
|
||||
d.append_message("s1", role="tool", content="일본 tool output blob", tool_name="terminal")
|
||||
yield d
|
||||
d.close()
|
||||
|
||||
|
||||
def test_two_char_korean_hits_cjk_index(db):
|
||||
rows = db.search_messages("웅기", limit=10)
|
||||
assert rows and "웅기" in rows[0]["snippet"]
|
||||
rows = db.search_messages("일본", limit=10)
|
||||
assert rows
|
||||
|
||||
|
||||
def test_mixed_and_ascii_queries(db):
|
||||
assert db.search_messages("graphiti", limit=10)
|
||||
assert db.search_messages('"shared default" AND 웅기', limit=10)
|
||||
assert db.search_messages("우선순위", limit=10)
|
||||
|
||||
|
||||
def test_no_false_positive_across_words(db):
|
||||
# 기가/했다 exist inside runs; a bigram crossing a word boundary must not.
|
||||
assert db.search_messages("다프로", limit=10) == []
|
||||
|
||||
|
||||
def test_lone_single_cjk_char_routes_like(db):
|
||||
# 1-char CJK terms keep LIKE substring semantics (bigram index only
|
||||
# holds unigrams for isolated chars). "가" appears inside 웅기가.
|
||||
assert db._describe_search_path("가") == "like_scan"
|
||||
rows = db.search_messages("가", limit=10)
|
||||
assert rows, "LIKE fallback must still find substring matches"
|
||||
|
||||
|
||||
def test_tool_role_filter_routes_like(db):
|
||||
# Tool rows are excluded from the cjk index; role_filter=['tool'] CJK
|
||||
# queries take the LIKE route and still find tool output.
|
||||
rows = db.search_messages("일본", role_filter=["tool"], limit=10)
|
||||
assert rows and all(r["role"] == "tool" for r in rows)
|
||||
|
||||
|
||||
def test_triggers_mirror_updates_and_deletes(db):
|
||||
db.append_message("s1", role="user", content="자바스크립트 리팩토링")
|
||||
assert db.search_messages("리팩토링", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"UPDATE messages SET content = '파이썬 리라이트' WHERE content LIKE '%리팩토링%'"
|
||||
)
|
||||
db._conn.commit()
|
||||
assert db.search_messages("리팩토링", limit=10) == []
|
||||
assert db.search_messages("리라이트", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute("DELETE FROM messages WHERE content = '파이썬 리라이트'")
|
||||
db._conn.commit()
|
||||
assert db.search_messages("리라이트", limit=10) == []
|
||||
|
||||
|
||||
def test_rewound_rows_hidden_from_cjk_search(db):
|
||||
db.append_message("s1", role="user", content="되돌리기 대상 메시지")
|
||||
assert db.search_messages("되돌리기", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"UPDATE messages SET active = 0, compacted = 0 "
|
||||
"WHERE content LIKE '%되돌리기%'"
|
||||
)
|
||||
db._conn.commit()
|
||||
assert db.search_messages("되돌리기", limit=10) == []
|
||||
assert db.search_messages("되돌리기", include_inactive=True, limit=10)
|
||||
|
||||
|
||||
def test_config_toggle_disables_cjk(cjk_so, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "0")
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
try:
|
||||
assert not d._fts_cjk_loaded
|
||||
assert not d._fts_cjk_available
|
||||
# No cjk objects created at all.
|
||||
with d._lock:
|
||||
row = d._conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name = 'messages_fts_cjk'"
|
||||
).fetchone()
|
||||
assert row is None
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_no_extension_no_cjk_objects(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "nonexistent.so"))
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
try:
|
||||
assert not d._fts_cjk_loaded
|
||||
assert not d._fts_cjk_available
|
||||
d.create_session(session_id="s1", source="cli", model="m")
|
||||
d.append_message("s1", role="user", content="일본 MCP 정리")
|
||||
# Trigram/LIKE routing still answers.
|
||||
assert d.search_messages("일본", limit=10)
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_tokenizer_loss_self_heals_and_optimize_rebuilds(cjk_so, tmp_path, monkeypatch):
|
||||
"""Full stale lifecycle: capable open → tokenizer-less open (drops
|
||||
triggers, breadcrumbs) → rows written in the gap → capable open again
|
||||
(index NOT served) → optimize-storage rebuilds → search complete."""
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
db_path = tmp_path / "state.db"
|
||||
|
||||
d1 = SessionDB(db_path=db_path)
|
||||
assert d1._fts_cjk_available
|
||||
d1.create_session(session_id="s1", source="cli", model="m")
|
||||
d1.append_message("s1", role="user", content="첫번째 메시지")
|
||||
d1.close()
|
||||
|
||||
# Tokenizer-less open: triggers dropped, breadcrumb set, writes fine.
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "gone.so"))
|
||||
d2 = SessionDB(db_path=db_path)
|
||||
assert not d2._fts_cjk_loaded
|
||||
assert not d2._fts_cjk_available
|
||||
d2.append_message("s1", role="user", content="틈새에 쓰인 메시지")
|
||||
assert d2.get_meta(FTS_CJK_STALE_KEY) == "1"
|
||||
with d2._lock:
|
||||
trigs = d2._conn.execute(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='trigger' "
|
||||
"AND name LIKE 'messages_fts_cjk%'"
|
||||
).fetchone()[0]
|
||||
assert trigs == 0
|
||||
# Search still answers via trigram/LIKE.
|
||||
assert d2.search_messages("틈새", limit=10)
|
||||
d2.close()
|
||||
|
||||
# Capable open again: stale index must NOT be served.
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d3 = SessionDB(db_path=db_path)
|
||||
assert d3._fts_cjk_loaded
|
||||
assert not d3._fts_cjk_available, "stale index must not serve reads"
|
||||
assert d3.fts_optimize_available(), "optimize must offer the rebuild"
|
||||
# Search still complete through the legacy routes meanwhile.
|
||||
assert d3.search_messages("틈새", limit=10)
|
||||
|
||||
result = d3.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d3._fts_cjk_available
|
||||
assert d3.get_meta(FTS_CJK_STALE_KEY) is None
|
||||
assert d3.fts_cjk_rebuild_status() is None
|
||||
# Both the pre-gap and in-gap rows are now searchable via the index.
|
||||
assert d3._describe_search_path("틈새") == "fts_cjk"
|
||||
assert d3.search_messages("첫번째", limit=10)
|
||||
assert d3.search_messages("틈새", limit=10)
|
||||
d3.close()
|
||||
|
||||
|
||||
def test_existing_v23_db_gains_cjk_via_optimize(cjk_so, tmp_path, monkeypatch):
|
||||
"""A v23 DB created BEFORE the extension existed: next capable open
|
||||
creates the index with backfill markers; optimize-storage backfills."""
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "absent.so"))
|
||||
db_path = tmp_path / "state.db"
|
||||
d1 = SessionDB(db_path=db_path)
|
||||
d1.create_session(session_id="s1", source="cli", model="m")
|
||||
for i in range(10):
|
||||
d1.append_message("s1", role="user", content=f"기존 메시지 {i}")
|
||||
d1.close()
|
||||
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d2 = SessionDB(db_path=db_path)
|
||||
assert d2._fts_cjk_loaded
|
||||
# Backfill pending — index not served yet, old rows not indexed.
|
||||
assert not d2._fts_cjk_available
|
||||
st = d2.fts_cjk_rebuild_status()
|
||||
assert st is not None and st["pending"]
|
||||
assert d2.fts_optimize_available()
|
||||
# NEW rows are indexed live by the id-gated triggers even mid-backfill.
|
||||
d2.append_message("s1", role="user", content="새로운 메시지")
|
||||
# Search answers via legacy routes meanwhile.
|
||||
assert d2.search_messages("기존", limit=10)
|
||||
|
||||
result = d2.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d2._fts_cjk_available
|
||||
assert d2.fts_cjk_rebuild_status() is None
|
||||
assert d2._describe_search_path("기존") == "fts_cjk"
|
||||
rows = d2.search_messages("기존", limit=20)
|
||||
assert len(rows) == 10
|
||||
assert d2.search_messages("새로운", limit=10)
|
||||
d2.close()
|
||||
|
||||
|
||||
def test_legacy_v22_optimize_lands_on_cjk(cjk_so, tmp_path, monkeypatch):
|
||||
"""A legacy inline-FTS (pre-v23) DB optimized on a tokenizer-capable
|
||||
host comes out with BOTH the v23 external-content layout AND a complete
|
||||
cjk index in the same run."""
|
||||
import time as _time
|
||||
|
||||
from hermes_state import SCHEMA_SQL
|
||||
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
db_path = tmp_path / "state.db"
|
||||
|
||||
# Hand-build a genuine legacy inline DB (single-column messages_fts).
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
conn.executescript("""
|
||||
DROP TABLE IF EXISTS messages_fts;
|
||||
DROP TABLE IF EXISTS messages_fts_trigram;
|
||||
DROP VIEW IF EXISTS messages_fts_trigram_src;
|
||||
CREATE VIRTUAL TABLE messages_fts USING fts5(content);
|
||||
CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, content) VALUES (new.id, COALESCE(new.content,''));
|
||||
END;
|
||||
""")
|
||||
conn.execute("DELETE FROM schema_version")
|
||||
conn.execute("INSERT INTO schema_version (version) VALUES (10)")
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (id, source, started_at) VALUES ('s1', 'cli', ?)",
|
||||
(_time.time(),),
|
||||
)
|
||||
for role, content in (
|
||||
("user", "레거시 일본 메시지"),
|
||||
("assistant", "legacy english reply"),
|
||||
("tool", "레거시 tool output"),
|
||||
):
|
||||
conn.execute(
|
||||
"INSERT INTO messages (session_id, timestamp, role, content) "
|
||||
"VALUES ('s1', ?, ?, ?)",
|
||||
(_time.time(), role, content),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
d = SessionDB(db_path=db_path)
|
||||
try:
|
||||
assert d.fts_optimize_available()
|
||||
# Legacy DB: cjk index deliberately not created at open (the legacy
|
||||
# branch of _init_schema doesn't touch v23 surfaces).
|
||||
assert not d._fts_cjk_available
|
||||
|
||||
result = d.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d._fts_cjk_available
|
||||
assert d.fts_cjk_rebuild_status() is None
|
||||
assert d._describe_search_path("일본") == "fts_cjk"
|
||||
assert d.search_messages("일본", limit=10)
|
||||
assert d.search_messages("legacy english", limit=10)
|
||||
with d._lock:
|
||||
idx = d._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages_fts_cjk"
|
||||
).fetchone()[0]
|
||||
non_tool = d._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
assert idx == non_tool
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_fresh_db_index_counts_exclude_tool_rows(db):
|
||||
with db._lock:
|
||||
idx = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages_fts_cjk"
|
||||
).fetchone()[0]
|
||||
non_tool = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
assert idx == non_tool
|
||||
|
||||
|
||||
def test_integrity_after_lifecycle(db):
|
||||
db.append_message("s1", role="user", content="무결성 검사")
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(messages_fts_cjk) "
|
||||
"VALUES('integrity-check')"
|
||||
)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Tests for the session-search slow-query log (salvaged from PR #65544)."""
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import SessionDB
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(tmp_path):
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
d.create_session(session_id="s1", source="cli", model="m")
|
||||
d.append_message("s1", role="user", content="hello graphiti 일본 MCP 정리")
|
||||
yield d
|
||||
d.close()
|
||||
|
||||
|
||||
def test_slow_log_emitted_at_zero_threshold(db, monkeypatch, caplog):
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "0")
|
||||
with caplog.at_level(logging.INFO, logger="hermes_state"):
|
||||
rows = db.search_messages("graphiti", limit=5)
|
||||
assert rows
|
||||
slow = [r for r in caplog.records if "slow session search" in r.getMessage()]
|
||||
assert slow, "threshold 0 must log every search"
|
||||
msg = slow[0].getMessage()
|
||||
assert "path=" in msg and "rows=1" in msg
|
||||
|
||||
|
||||
def test_no_log_under_threshold(db, monkeypatch, caplog):
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "60000")
|
||||
with caplog.at_level(logging.INFO, logger="hermes_state"):
|
||||
db.search_messages("graphiti", limit=5)
|
||||
assert not [r for r in caplog.records if "slow session search" in r.getMessage()]
|
||||
|
||||
|
||||
def test_path_attribution(db):
|
||||
# Without the cjk tokenizer loaded, routing matches the pre-cjk shape.
|
||||
assert db._describe_search_path("graphiti OR neo4j") == "fts5"
|
||||
assert db._describe_search_path("우선순위 캘린더") == "trigram"
|
||||
assert db._describe_search_path("일본 MCP") == "like_scan"
|
||||
|
||||
|
||||
def test_path_attribution_cjk_available(db):
|
||||
# With the bigram index available, CJK queries (including 2-char terms)
|
||||
# route to fts_cjk; lone 1-char CJK runs keep the LIKE route.
|
||||
db._fts_cjk_available = True
|
||||
try:
|
||||
assert db._describe_search_path("일본 MCP") == "fts_cjk"
|
||||
assert db._describe_search_path("우선순위 캘린더") == "fts_cjk"
|
||||
assert db._describe_search_path("가 alone") == "like_scan"
|
||||
assert db._describe_search_path("graphiti OR neo4j") == "fts5"
|
||||
finally:
|
||||
db._fts_cjk_available = False
|
||||
|
||||
|
||||
def test_results_unchanged_by_wrapper(db, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "0")
|
||||
rows = db.search_messages("graphiti", limit=5)
|
||||
assert rows and rows[0]["session_id"] == "s1"
|
||||
@@ -402,6 +402,21 @@ export const en: Translations = {
|
||||
versionBadge: "Version",
|
||||
showInSidebar: "Show in sidebar",
|
||||
hideFromSidebar: "Hide from sidebar",
|
||||
catalogHeading: "Plugin catalog",
|
||||
catalogHint:
|
||||
"Curated, Nous-reviewed plugins pinned to exact commits. Install from here for supply-chain-safe versions.",
|
||||
catalogSearchPlaceholder: "Search catalog...",
|
||||
catalogEmpty: "No catalog entries match.",
|
||||
catalogEmptyDocsLink: "Learn about Hermes plugins",
|
||||
catalogInstallBtn: "Install",
|
||||
catalogInstalledBadge: "Installed ✓",
|
||||
catalogUpdateBtn: "Update available",
|
||||
catalogRemovedBadge: "Removed",
|
||||
catalogConfirmTitle: "Install this plugin?",
|
||||
catalogConfirmInstallNote:
|
||||
"Plugins install disabled; enable it after install to activate.",
|
||||
catalogRequiresEnv: "Requires env",
|
||||
removedFromCatalog: "Removed from catalog",
|
||||
},
|
||||
|
||||
skills: {
|
||||
|
||||
@@ -351,6 +351,20 @@ export interface Translations {
|
||||
versionBadge: string;
|
||||
showInSidebar: string;
|
||||
hideFromSidebar: string;
|
||||
// Catalog section (en-only fallback convention — optional keys).
|
||||
catalogHeading?: string;
|
||||
catalogHint?: string;
|
||||
catalogSearchPlaceholder?: string;
|
||||
catalogEmpty?: string;
|
||||
catalogEmptyDocsLink?: string;
|
||||
catalogInstallBtn?: string;
|
||||
catalogInstalledBadge?: string;
|
||||
catalogUpdateBtn?: string;
|
||||
catalogRemovedBadge?: string;
|
||||
catalogConfirmTitle?: string;
|
||||
catalogConfirmInstallNote?: string;
|
||||
catalogRequiresEnv?: string;
|
||||
removedFromCatalog?: string;
|
||||
};
|
||||
|
||||
// ── Profiles page ──
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { api } from "./api";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function jsonFetchMock(body: unknown = { ok: true }) {
|
||||
return vi.fn<typeof fetch>(
|
||||
async () =>
|
||||
new Response(JSON.stringify(body), {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
status: 200,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe("api.getPluginsCatalog", () => {
|
||||
it("fetches the dashboard plugins catalog endpoint", async () => {
|
||||
vi.stubGlobal("window", {});
|
||||
|
||||
const fetchMock = jsonFetchMock({ entries: [], removed: [], generated_at: "" });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const result = await api.getPluginsCatalog();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"/api/dashboard/plugins/catalog",
|
||||
expect.objectContaining({ credentials: "include" }),
|
||||
);
|
||||
expect(result.entries).toEqual([]);
|
||||
expect(result.removed).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("api.installAgentPlugin with catalog_name", () => {
|
||||
it("posts catalog_name through to the install endpoint", async () => {
|
||||
vi.stubGlobal("window", {});
|
||||
|
||||
const fetchMock = jsonFetchMock({ ok: true, plugin_name: "alpha-plugin" });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await api.installAgentPlugin({
|
||||
identifier: "",
|
||||
catalog_name: "alpha-plugin",
|
||||
enable: false,
|
||||
});
|
||||
|
||||
const [url, init] = fetchMock.mock.calls[0]!;
|
||||
expect(url).toBe("/api/dashboard/agent-plugins/install");
|
||||
const body = JSON.parse(String((init as RequestInit).body));
|
||||
expect(body.catalog_name).toBe("alpha-plugin");
|
||||
expect(body.identifier).toBe("");
|
||||
expect(body.enable).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -921,6 +921,9 @@ export const api = {
|
||||
|
||||
getPluginsHub: () => fetchJSON<PluginsHubResponse>("/api/dashboard/plugins/hub"),
|
||||
|
||||
getPluginsCatalog: () =>
|
||||
fetchJSON<CatalogResponse>("/api/dashboard/plugins/catalog"),
|
||||
|
||||
installAgentPlugin: (body: AgentPluginInstallRequest) =>
|
||||
fetchJSON<AgentPluginInstallResponse>("/api/dashboard/agent-plugins/install", {
|
||||
method: "POST",
|
||||
@@ -2506,6 +2509,8 @@ export interface HubAgentPluginRow {
|
||||
auth_required: boolean;
|
||||
auth_command: string;
|
||||
user_hidden: boolean;
|
||||
/** Reason string when this plugin is on the catalog removed blocklist. */
|
||||
removed_reason?: string | null;
|
||||
}
|
||||
|
||||
export interface PluginsHubProviders {
|
||||
@@ -2525,6 +2530,8 @@ export interface AgentPluginInstallRequest {
|
||||
identifier: string;
|
||||
force?: boolean;
|
||||
enable?: boolean;
|
||||
/** Install by curated-catalog name (resolves repo + pinned SHA server-side). */
|
||||
catalog_name?: string;
|
||||
}
|
||||
|
||||
export interface AgentPluginInstallResponse {
|
||||
@@ -2537,6 +2544,48 @@ export interface AgentPluginInstallResponse {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ── Plugin catalog types ───────────────────────────────────────────────
|
||||
|
||||
export interface CatalogCapabilities {
|
||||
provides_tools: string[];
|
||||
provides_hooks: string[];
|
||||
provides_middleware: string[];
|
||||
requires_env: string[];
|
||||
}
|
||||
|
||||
export interface CatalogEntry {
|
||||
name: string;
|
||||
description: string;
|
||||
repo: string;
|
||||
sha: string;
|
||||
sha_short: string;
|
||||
tier: "official" | "community";
|
||||
maintainer: string;
|
||||
requires_hermes: string;
|
||||
platforms: string[];
|
||||
capabilities: CatalogCapabilities;
|
||||
docs_url: string;
|
||||
capability_summary: string;
|
||||
/** Installed-state merge (computed server-side). */
|
||||
installed: boolean;
|
||||
installed_sha: string | null;
|
||||
update_available: boolean;
|
||||
runtime_status: "disabled" | "enabled" | "inactive" | null;
|
||||
}
|
||||
|
||||
export interface CatalogRemovedEntry {
|
||||
name: string;
|
||||
repo: string;
|
||||
reason: string;
|
||||
date: string;
|
||||
}
|
||||
|
||||
export interface CatalogResponse {
|
||||
entries: CatalogEntry[];
|
||||
removed: CatalogRemovedEntry[];
|
||||
generated_at: string;
|
||||
}
|
||||
|
||||
export interface AgentPluginUpdateResponse {
|
||||
ok: boolean;
|
||||
name?: string;
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { ExternalLink, RefreshCw, Trash2, Eye, EyeOff } from "lucide-react";
|
||||
import type { Translations } from "@/i18n/types";
|
||||
import { Link } from "react-router-dom";
|
||||
import { api } from "@/lib/api";
|
||||
import type {
|
||||
CatalogEntry,
|
||||
CatalogRemovedEntry,
|
||||
CatalogResponse,
|
||||
HubAgentPluginRow,
|
||||
MemoryProviderConfig,
|
||||
MemoryProviderField,
|
||||
@@ -281,6 +284,11 @@ function MemoryProviderSetupHint({
|
||||
export default function PluginsPage() {
|
||||
const [hub, setHub] = useState<PluginsHubResponse | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [catalog, setCatalog] = useState<CatalogResponse | null>(null);
|
||||
const [catalogLoading, setCatalogLoading] = useState(true);
|
||||
const [catalogSearch, setCatalogSearch] = useState("");
|
||||
const [catalogConfirm, setCatalogConfirm] = useState<CatalogEntry | null>(null);
|
||||
const [catalogBusy, setCatalogBusy] = useState<string | null>(null);
|
||||
const [installId, setInstallId] = useState("");
|
||||
const [installForce, setInstallForce] = useState(false);
|
||||
const [installEnable, setInstallEnable] = useState(true);
|
||||
@@ -316,9 +324,17 @@ export default function PluginsPage() {
|
||||
.catch(() => showToast(t.common.loading, "error"));
|
||||
}, [showToast, t.common.loading]);
|
||||
|
||||
const loadCatalog = useCallback(() => {
|
||||
return api
|
||||
.getPluginsCatalog()
|
||||
.then(setCatalog)
|
||||
.catch(() => setCatalog(null));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void loadHub().finally(() => setLoading(false));
|
||||
}, [loadHub]);
|
||||
void loadCatalog().finally(() => setCatalogLoading(false));
|
||||
}, [loadHub, loadCatalog]);
|
||||
|
||||
useEffect(() => {
|
||||
const provider = memorySel === MEMORY_PROVIDER_BUILTIN ? "" : memorySel;
|
||||
@@ -391,6 +407,27 @@ export default function PluginsPage() {
|
||||
}
|
||||
};
|
||||
|
||||
const onCatalogInstall = async (entry: CatalogEntry) => {
|
||||
setCatalogConfirm(null);
|
||||
setCatalogBusy(entry.name);
|
||||
try {
|
||||
const r = await api.installAgentPlugin({
|
||||
identifier: "",
|
||||
catalog_name: entry.name,
|
||||
force: entry.installed,
|
||||
enable: false,
|
||||
});
|
||||
showToast(`${r.plugin_name ?? entry.name} installed`, "success");
|
||||
if ((r.missing_env?.length ?? 0) > 0)
|
||||
showToast(`${t.pluginsPage.missingEnvWarn} ${r.missing_env!.join(", ")}`, "error");
|
||||
await Promise.all([loadHub(), loadCatalog()]);
|
||||
} catch (e) {
|
||||
showToast(e instanceof Error ? e.message : "Install failed", "error");
|
||||
} finally {
|
||||
setCatalogBusy(null);
|
||||
}
|
||||
};
|
||||
|
||||
const onRescan = useCallback(async () => {
|
||||
setRescanBusy(true);
|
||||
try {
|
||||
@@ -506,6 +543,27 @@ export default function PluginsPage() {
|
||||
|
||||
const rows = hub?.plugins ?? [];
|
||||
const providers = hub?.providers;
|
||||
|
||||
const catalogEntries = useMemo(() => {
|
||||
const entries = catalog?.entries ?? [];
|
||||
const q = catalogSearch.trim().toLowerCase();
|
||||
if (!q) return entries;
|
||||
return entries.filter((entry) =>
|
||||
[
|
||||
entry.name,
|
||||
entry.description,
|
||||
entry.maintainer,
|
||||
...entry.capabilities.provides_tools,
|
||||
].some((haystack) => haystack.toLowerCase().includes(q)),
|
||||
);
|
||||
}, [catalog, catalogSearch]);
|
||||
|
||||
const removedByName = useMemo(() => {
|
||||
const map = new Map<string, CatalogRemovedEntry>();
|
||||
for (const r of catalog?.removed ?? []) map.set(r.name, r);
|
||||
return map;
|
||||
}, [catalog]);
|
||||
|
||||
const selectedMemoryName = memorySel === MEMORY_PROVIDER_BUILTIN ? "" : memorySel;
|
||||
const selectedMemoryInfo = selectedMemoryName
|
||||
? providers?.memory_options.find((provider) => provider.name === selectedMemoryName)
|
||||
@@ -822,6 +880,59 @@ export default function PluginsPage() {
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<div className="flex flex-col gap-3" data-testid="plugin-catalog-section">
|
||||
|
||||
<h3 className="font-mondwest text-display text-xs tracking-[0.12em] text-text-secondary">
|
||||
{t.pluginsPage.catalogHeading ?? "Plugin catalog"}
|
||||
</h3>
|
||||
|
||||
<p className="text-xs tracking-[0.06em] text-text-tertiary">
|
||||
{t.pluginsPage.catalogHint ??
|
||||
"Curated, Nous-reviewed plugins pinned to exact commits."}
|
||||
</p>
|
||||
|
||||
<Input
|
||||
className="max-w-md"
|
||||
placeholder={t.pluginsPage.catalogSearchPlaceholder ?? "Search catalog..."}
|
||||
value={catalogSearch}
|
||||
onChange={(e) => setCatalogSearch(e.target.value)}
|
||||
aria-label={t.pluginsPage.catalogSearchPlaceholder ?? "Search catalog..."}
|
||||
/>
|
||||
|
||||
{catalogLoading ? (
|
||||
<div className="flex items-center gap-2 py-4 text-xs text-text-tertiary">
|
||||
<Spinner />
|
||||
<span>{t.common.loading}</span>
|
||||
</div>
|
||||
) : catalogEntries.length === 0 ? (
|
||||
<p className="text-xs text-text-tertiary">
|
||||
{t.pluginsPage.catalogEmpty ?? "No catalog entries match."}{" "}
|
||||
<a
|
||||
className="underline"
|
||||
href="https://hermes-agent.nousresearch.com/docs/plugins"
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
{t.pluginsPage.catalogEmptyDocsLink ?? "Learn about Hermes plugins"}
|
||||
</a>
|
||||
</p>
|
||||
) : (
|
||||
<ul className="flex flex-col gap-3">
|
||||
{catalogEntries.map((entry) => (
|
||||
<li key={entry.name}>
|
||||
<CatalogEntryCard
|
||||
busy={catalogBusy === entry.name}
|
||||
entry={entry}
|
||||
onInstall={() => setCatalogConfirm(entry)}
|
||||
removed={removedByName.get(entry.name) ?? null}
|
||||
t={t}
|
||||
/>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-3">
|
||||
|
||||
<h3 className="font-mondwest text-display text-xs tracking-[0.12em] text-text-secondary">
|
||||
@@ -896,6 +1007,30 @@ export default function PluginsPage() {
|
||||
|
||||
<Toast toast={toast} />
|
||||
<PluginSlot name="plugins:bottom" />
|
||||
|
||||
<ConfirmDialog
|
||||
open={catalogConfirm !== null}
|
||||
onCancel={() => setCatalogConfirm(null)}
|
||||
onConfirm={() => {
|
||||
if (catalogConfirm) void onCatalogInstall(catalogConfirm);
|
||||
}}
|
||||
title={t.pluginsPage.catalogConfirmTitle ?? "Install this plugin?"}
|
||||
description={
|
||||
catalogConfirm
|
||||
? [
|
||||
catalogConfirm.capability_summary,
|
||||
catalogConfirm.capabilities.requires_env.length
|
||||
? `${t.pluginsPage.catalogRequiresEnv ?? "Requires env"}: ${catalogConfirm.capabilities.requires_env.join(", ")}`
|
||||
: "",
|
||||
t.pluginsPage.catalogConfirmInstallNote ??
|
||||
"Plugins install disabled; enable it after install to activate.",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
: ""
|
||||
}
|
||||
confirmLabel={t.pluginsPage.catalogInstallBtn ?? "Install"}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -961,6 +1096,12 @@ function PluginRowCard(props: PluginRowCardProps) {
|
||||
{row.auth_required ? (
|
||||
<Badge tone="destructive">{t.pluginsPage.authRequired}</Badge>
|
||||
) : null}
|
||||
|
||||
{row.removed_reason ? (
|
||||
<Badge tone="destructive">
|
||||
{t.pluginsPage.catalogRemovedBadge ?? "Removed"}
|
||||
</Badge>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2 shrink-0">
|
||||
@@ -1070,6 +1211,12 @@ function PluginRowCard(props: PluginRowCardProps) {
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{row.removed_reason ? (
|
||||
<p className="border border-destructive/50 px-3 py-2 text-xs text-destructive">
|
||||
{t.pluginsPage.removedFromCatalog ?? "Removed from catalog"}: {row.removed_reason}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{dm?.slots?.length ? (
|
||||
|
||||
<p className="text-xs tracking-[0.05em] text-text-tertiary">
|
||||
@@ -1111,3 +1258,127 @@ function PluginRowCard(props: PluginRowCardProps) {
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
interface CatalogEntryCardProps {
|
||||
busy: boolean;
|
||||
entry: CatalogEntry;
|
||||
onInstall: () => void;
|
||||
removed: CatalogRemovedEntry | null;
|
||||
t: Translations;
|
||||
}
|
||||
|
||||
function CatalogEntryCard(props: CatalogEntryCardProps) {
|
||||
const { busy, entry, onInstall, removed, t } = props;
|
||||
|
||||
const caps = entry.capabilities;
|
||||
const chips: string[] = [];
|
||||
if (caps.provides_tools.length) chips.push(`${caps.provides_tools.length} tools`);
|
||||
if (caps.provides_hooks.length) chips.push(`${caps.provides_hooks.length} hooks`);
|
||||
if (caps.provides_middleware.length)
|
||||
chips.push(`${caps.provides_middleware.length} middleware`);
|
||||
if (caps.requires_env.length) chips.push(`env: ${caps.requires_env.join(", ")}`);
|
||||
|
||||
const isRemoved = removed !== null;
|
||||
|
||||
return (
|
||||
<Card className={cn(busy ? "opacity-70" : undefined)}>
|
||||
<CardContent className="flex flex-col gap-3 px-6 py-4">
|
||||
<div className="flex flex-wrap items-start justify-between gap-4">
|
||||
<div className="flex min-w-0 flex-1 flex-wrap items-center gap-3">
|
||||
<span className="truncate font-semibold">{entry.name}</span>
|
||||
|
||||
<Badge tone={entry.tier === "official" ? "success" : "secondary"}>
|
||||
{entry.tier}
|
||||
</Badge>
|
||||
|
||||
{entry.installed && entry.runtime_status ? (
|
||||
<Badge tone="outline">{entry.runtime_status}</Badge>
|
||||
) : null}
|
||||
|
||||
{isRemoved ? (
|
||||
<Badge tone="destructive">
|
||||
{t.pluginsPage.catalogRemovedBadge ?? "Removed"}
|
||||
</Badge>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2 shrink-0">
|
||||
{isRemoved ? null : entry.installed && !entry.update_available ? (
|
||||
<Badge tone="success">
|
||||
{t.pluginsPage.catalogInstalledBadge ?? "Installed ✓"}
|
||||
</Badge>
|
||||
) : (
|
||||
<Button disabled={busy} ghost size="sm" onClick={onInstall}>
|
||||
{busy ? <Spinner /> : null}
|
||||
{entry.update_available
|
||||
? t.pluginsPage.catalogUpdateBtn ?? "Update available"
|
||||
: t.pluginsPage.catalogInstallBtn ?? "Install"}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isRemoved ? (
|
||||
<p className="border border-destructive/50 px-3 py-2 text-xs text-destructive">
|
||||
{t.pluginsPage.removedFromCatalog ?? "Removed from catalog"}
|
||||
{removed.reason ? `: ${removed.reason}` : ""}
|
||||
{removed.date ? ` (${removed.date})` : ""}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{entry.description ? (
|
||||
<p className="min-w-0 w-full text-xs tracking-[0.06em] text-text-secondary break-words">
|
||||
{entry.description}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{chips.length ? (
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{chips.map((chip) => (
|
||||
<code
|
||||
key={chip}
|
||||
className="border border-border bg-background/40 px-2 py-1 font-mono text-[0.6875rem]"
|
||||
>
|
||||
{chip}
|
||||
</code>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="flex flex-wrap items-center gap-3 text-xs text-text-tertiary">
|
||||
<span>{entry.maintainer}</span>
|
||||
|
||||
<a
|
||||
className="inline-flex items-center gap-1 font-mono underline"
|
||||
href={`${entry.repo.replace(/\.git$/, "")}/tree/${entry.sha}`}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
{entry.sha_short}
|
||||
<ExternalLink className="h-3 w-3 opacity-65" />
|
||||
</a>
|
||||
|
||||
{entry.docs_url ? (
|
||||
<a
|
||||
className="inline-flex items-center gap-1 underline"
|
||||
href={entry.docs_url}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
docs
|
||||
<ExternalLink className="h-3 w-3 opacity-65" />
|
||||
</a>
|
||||
) : null}
|
||||
|
||||
{entry.requires_hermes ? (
|
||||
<span>hermes {entry.requires_hermes}</span>
|
||||
) : null}
|
||||
|
||||
{entry.platforms.length ? (
|
||||
<span>{entry.platforms.join(", ")}</span>
|
||||
) : null}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -744,6 +744,7 @@ All compression settings live in `config.yaml` (no environment variables).
|
||||
compression:
|
||||
enabled: true # Toggle compression on/off
|
||||
threshold: 0.50 # Compress at this % of context limit
|
||||
threshold_tokens: null # Absolute token cap (optional) — takes lower of ratio vs absolute
|
||||
target_ratio: 0.20 # Fraction of threshold to preserve as recent tail
|
||||
protect_last_n: 20 # Min recent messages to keep uncompressed
|
||||
protect_first_n: 3 # Non-system head messages pinned across compactions (0 = pin nothing)
|
||||
@@ -765,6 +766,8 @@ Older configs with `compression.summary_model`, `compression.summary_provider`,
|
||||
|
||||
`protect_first_n` controls how many **non-system** head messages are pinned across every compaction. Default `3` — the opening user/assistant exchange survives every summarizer pass so the original goal stays visible. On long-running rolling-compaction sessions where the opening turn is no longer relevant, set `protect_first_n: 0` to pin nothing but the system prompt + summary + tail. The system prompt itself is always preserved regardless of this setting.
|
||||
|
||||
`threshold_tokens` sets an optional **absolute token cap** for the compression trigger. When set, compression fires at the lower of the ratio-based `threshold` and this absolute count — so compression never fires later than the user's preferred token number regardless of which model is active. This solves the problem where switching between models with different context windows (e.g. 1M → 400K) shifts the absolute trigger point. The cap is clamped to the model's context length, so setting it higher than the model supports is safe — the ratio-based threshold is used instead. Default `null` (disabled — ratio-based threshold only). The cap survives model switches and fallback activations.
|
||||
|
||||
:::tip Gateway hot-reload of compression and context length
|
||||
As of recent releases, editing `model.context_length` or any `compression.*` key in `config.yaml` on a running gateway takes effect on the next message — no gateway restart, no `/reset`, no session rotation required. The cached-agent signature includes these keys, so the gateway transparently rebuilds the agent when it sees a change. API keys and tool/skill config still require the usual reload paths.
|
||||
:::
|
||||
|
||||
Reference in New Issue
Block a user