Compare commits
40
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f3c6282dae | ||
|
|
ab1d9f4c15 | ||
|
|
7880be4580 | ||
|
|
741d06445d | ||
|
|
1c3d6e59dd | ||
|
|
ae68edcfe3 | ||
|
|
355a8e4076 | ||
|
|
cbc1054e23 | ||
|
|
356ff99030 | ||
|
|
5a3ee3c537 | ||
|
|
2b84ed921c | ||
|
|
020bd1ba0a | ||
|
|
e5078e3152 | ||
|
|
0acdf1d8c8 | ||
|
|
2ee50c69d3 | ||
|
|
97cd0d98f1 | ||
|
|
bc4824167d | ||
|
|
fe9dc06071 | ||
|
|
f13f845116 | ||
|
|
8364576e33 | ||
|
|
b10952e9c6 | ||
|
|
fb40a768fc | ||
|
|
17155e3ae0 | ||
|
|
5d747a91c4 | ||
|
|
503c0c0e51 | ||
|
|
73d5c896ee | ||
|
|
fee392fee1 | ||
|
|
fc0009b9ba | ||
|
|
ad4034711d | ||
|
|
fd433e046a | ||
|
|
c8089dabcd | ||
|
|
2d71e9e9bc | ||
|
|
8dd07bd517 | ||
|
|
40c3b62b30 | ||
|
|
3c5c389f18 | ||
|
|
45556b71ce | ||
|
|
caf8e2f214 | ||
|
|
7bbdabbef2 | ||
|
|
54a0f07101 | ||
|
|
77beb6a085 |
@@ -0,0 +1,56 @@
|
||||
name: Hermes Plugin Validate
|
||||
description: >-
|
||||
Validate a Hermes Agent plugin (plugin.yaml manifest schema AND
|
||||
declared-vs-actually-registered capabilities) using
|
||||
`hermes plugins validate`. Drop this into your plugin repo's CI:
|
||||
|
||||
- uses: actions/checkout@<sha>
|
||||
- uses: NousResearch/hermes-agent/.github/actions/plugin-validate@main
|
||||
with:
|
||||
path: .
|
||||
|
||||
The caller's job owns checkout; this action installs Python + hermes-agent
|
||||
(git install — a supported CI-context install route) and runs the
|
||||
validator against your plugin directory.
|
||||
|
||||
inputs:
|
||||
path:
|
||||
description: Path to the plugin directory (containing plugin.yaml).
|
||||
default: "."
|
||||
hermes-ref:
|
||||
description: hermes-agent git ref (branch/tag/sha) to install and validate with.
|
||||
default: "main"
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install hermes-agent
|
||||
shell: bash
|
||||
env:
|
||||
_HERMES_REF: ${{ inputs.hermes-ref }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# CI-context install from git; the ref lets plugin authors validate
|
||||
# against a pinned hermes release instead of main.
|
||||
pip install "git+https://github.com/NousResearch/hermes-agent@${_HERMES_REF}"
|
||||
|
||||
- name: Validate plugin
|
||||
shell: bash
|
||||
env:
|
||||
_PLUGIN_PATH: ${{ inputs.path }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# `hermes plugins validate` checks the plugin.yaml manifest schema
|
||||
# and loads the plugin in a scratch subprocess to verify that the
|
||||
# capabilities it DECLARES match what it actually registers.
|
||||
if hermes plugins validate "$_PLUGIN_PATH"; then
|
||||
echo "✅ PASS: plugin at '$_PLUGIN_PATH' validated cleanly"
|
||||
else
|
||||
echo "❌ FAIL: plugin at '$_PLUGIN_PATH' failed validation (see output above)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -157,6 +157,9 @@ jobs:
|
||||
- name: Extract skill metadata for dashboard
|
||||
run: python3 website/scripts/extract-skills.py
|
||||
|
||||
- name: Extract plugin catalog for the Plugins page
|
||||
run: python3 website/scripts/extract-plugins.py
|
||||
|
||||
- name: Regenerate per-skill docs pages + catalogs
|
||||
run: python3 website/scripts/generate-skill-docs.py
|
||||
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
name: Plugin Catalog CI
|
||||
|
||||
# Admission gate for plugin-catalog entries. Fires ONLY on PRs touching
|
||||
# plugin-catalog/** so it can never go red on unrelated PRs.
|
||||
#
|
||||
# Two gates:
|
||||
# structural — cheap schema check, no hermes install needed
|
||||
# pinned-source-validate — supply-chain gate: the pinned sha MUST be
|
||||
# reachable in the entry's repo, and the plugin
|
||||
# at that exact commit must pass
|
||||
# `hermes plugins validate`.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "plugin-catalog/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
structural:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install PyYAML
|
||||
uses: ./.github/actions/retry
|
||||
with:
|
||||
command: pip install pyyaml==6.0.2
|
||||
|
||||
- name: Validate catalog files (structural)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Validating the whole directory is simpler than diffing and keeps
|
||||
# the invariant that EVERYTHING in plugin-catalog/ stays valid.
|
||||
python3 scripts/validate_plugin_catalog.py plugin-catalog/
|
||||
|
||||
pinned-source-validate:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0 # need the merge-base to diff changed catalog files
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Find changed catalog entries
|
||||
id: changed
|
||||
run: |
|
||||
set -euo pipefail
|
||||
MERGE_BASE=$(git merge-base "origin/${{ github.base_ref }}" HEAD)
|
||||
# Added + modified entry files only; deletions and removed.yaml
|
||||
# have nothing to clone.
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$MERGE_BASE"...HEAD \
|
||||
-- 'plugin-catalog/*.yaml' 'plugin-catalog/*.yml' \
|
||||
| grep -v '/removed\.yaml$' || true)
|
||||
echo "Changed catalog entries:"
|
||||
echo "${CHANGED:-<none>}"
|
||||
{
|
||||
echo 'files<<__EOF__'
|
||||
echo "$CHANGED"
|
||||
echo '__EOF__'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Install hermes-agent from the PR's own checkout
|
||||
if: steps.changed.outputs.files != ''
|
||||
uses: ./.github/actions/retry
|
||||
with:
|
||||
command: pip install -e .
|
||||
|
||||
- name: Clone each entry at its pinned sha and validate
|
||||
if: steps.changed.outputs.files != ''
|
||||
env:
|
||||
CHANGED_FILES: ${{ steps.changed.outputs.files }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
FAILED=0
|
||||
while IFS= read -r entry; do
|
||||
[ -z "$entry" ] && continue
|
||||
echo "::group::validate $entry"
|
||||
|
||||
# Parse repo / sha / subdir from the entry yaml.
|
||||
eval "$(python3 - "$entry" <<'PYEOF'
|
||||
import shlex
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as fh:
|
||||
data = yaml.safe_load(fh) or {}
|
||||
print(f"REPO={shlex.quote(str(data.get('repo', '')))}")
|
||||
print(f"SHA={shlex.quote(str(data.get('sha', '')))}")
|
||||
print(f"SUBDIR={shlex.quote(str(data.get('subdir', '') or ''))}")
|
||||
PYEOF
|
||||
)"
|
||||
echo "repo=$REPO sha=$SHA subdir=$SUBDIR"
|
||||
|
||||
CLONE_DIR=$(mktemp -d)
|
||||
# Full clone (no --depth 1): the pinned sha may not be the branch tip.
|
||||
if ! git clone "$REPO" "$CLONE_DIR"; then
|
||||
echo "::error file=$entry::clone failed for $REPO"
|
||||
FAILED=1; echo "::endgroup::"; continue
|
||||
fi
|
||||
|
||||
# SUPPLY-CHAIN GATE: the pinned sha must be reachable in the repo.
|
||||
if ! git -C "$CLONE_DIR" checkout --detach "$SHA"; then
|
||||
echo "::error file=$entry::pinned sha $SHA is not reachable in $REPO"
|
||||
FAILED=1; echo "::endgroup::"; continue
|
||||
fi
|
||||
|
||||
PLUGIN_DIR="$CLONE_DIR${SUBDIR:+/$SUBDIR}"
|
||||
if [ ! -f "$PLUGIN_DIR/plugin.yaml" ]; then
|
||||
echo "::error file=$entry::no plugin.yaml at subdir '$SUBDIR' of $REPO@$SHA"
|
||||
FAILED=1; echo "::endgroup::"; continue
|
||||
fi
|
||||
|
||||
# Manifest schema + declared-vs-registered capability check.
|
||||
if hermes plugins validate "$PLUGIN_DIR"; then
|
||||
echo "✅ PASS: $entry"
|
||||
else
|
||||
echo "::error file=$entry::hermes plugins validate failed"
|
||||
FAILED=1
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done <<< "$CHANGED_FILES"
|
||||
|
||||
if [ "$FAILED" -ne 0 ]; then
|
||||
echo "❌ FAIL: one or more catalog entries failed pinned-source validation"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ PASS: all changed catalog entries validated at their pinned shas"
|
||||
@@ -113,6 +113,10 @@ website/static/api/skills-index.json
|
||||
# every build).
|
||||
website/static/api/skills.json
|
||||
website/static/api/skills-meta.json
|
||||
# plugins.json + plugins-meta.json are build artifacts emitted by
|
||||
# website/scripts/extract-plugins.py during prebuild (Plugin Catalog page).
|
||||
website/static/api/plugins.json
|
||||
website/static/api/plugins-meta.json
|
||||
# automation-blueprints-index.json is a build artifact emitted by
|
||||
# website/scripts/extract-automation-blueprints.py during prebuild.
|
||||
website/static/api/automation-blueprints-index.json
|
||||
@@ -169,3 +173,4 @@ apps/desktop/demo/
|
||||
# PR body is the archive. See the hermes-agent-dev skill's
|
||||
# pr-infographic-workflow reference (storage rule + lapse #8 / #COMMIT-1).
|
||||
infographic/
|
||||
native/fts5_cjk/*.so
|
||||
|
||||
+18
-1
@@ -1839,6 +1839,18 @@ def init_agent(
|
||||
}
|
||||
else:
|
||||
compression_model_thresholds = {}
|
||||
# Absolute token cap: when set, compression triggers at the lower of
|
||||
# the ratio-based threshold and this absolute count. Clamped to the
|
||||
# model's context length at apply-time so a cap above the window is
|
||||
# a no-op (ratio-based threshold wins).
|
||||
compression_threshold_tokens = _compression_cfg.get("threshold_tokens")
|
||||
if compression_threshold_tokens is not None:
|
||||
try:
|
||||
compression_threshold_tokens = int(compression_threshold_tokens)
|
||||
if compression_threshold_tokens <= 0:
|
||||
compression_threshold_tokens = None
|
||||
except (TypeError, ValueError):
|
||||
compression_threshold_tokens = None
|
||||
# In-place compaction: when True, compress_context() rewrites the message
|
||||
# list + rebuilds the system prompt WITHOUT rotating the session id (no
|
||||
# parent_session_id chain, no `name #N` renumber). See #38763 and
|
||||
@@ -2271,6 +2283,7 @@ def init_agent(
|
||||
abort_on_summary_failure=compression_abort_on_summary_failure,
|
||||
max_tokens=agent.max_tokens,
|
||||
model_thresholds=compression_model_thresholds,
|
||||
threshold_tokens_cap=compression_threshold_tokens,
|
||||
)
|
||||
_bind_session_state = getattr(agent.context_compressor, "bind_session_state", None)
|
||||
if callable(_bind_session_state):
|
||||
@@ -2482,7 +2495,11 @@ def init_agent(
|
||||
_active_threshold_pct = getattr(
|
||||
agent.context_compressor, "threshold_percent", compression_threshold
|
||||
)
|
||||
print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(_active_threshold_pct*100)}% = {agent.context_compressor.threshold_tokens:,})")
|
||||
_cap_note = ""
|
||||
_cap = getattr(agent.context_compressor, "threshold_tokens_cap", None)
|
||||
if _cap and _cap > 0:
|
||||
_cap_note = f" (capped at {_cap:,} tokens)"
|
||||
print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(_active_threshold_pct*100)}% = {agent.context_compressor.threshold_tokens:,}{_cap_note})")
|
||||
else:
|
||||
print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (auto-compression disabled)")
|
||||
# Notice with the exact opt-back-out command. Printed inline at startup
|
||||
|
||||
+505
-43
@@ -22,6 +22,7 @@ import logging
|
||||
import sqlite3
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.auxiliary_client import call_llm, _is_connection_error, aux_interrupt_protection
|
||||
@@ -40,6 +41,14 @@ from tools.todo_tool import TODO_INJECTION_HEADER
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
"""Best-effort integer coercion for telemetry fields."""
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
_SUMMARY_PERMANENT_QUOTA_MARKERS: tuple[str, ...] = (
|
||||
"insufficient_quota",
|
||||
"quota exceeded",
|
||||
@@ -342,6 +351,27 @@ _HISTORICAL_TASK_SECTION_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def _redact_compaction_text(text: Any) -> str:
|
||||
"""Redact text that crosses a compaction summary boundary.
|
||||
|
||||
Compaction summaries persist across sessions and are re-injected into
|
||||
every subsequent summarizer prompt, so this boundary uses strict mode:
|
||||
|
||||
- ``force=True`` — deliberately overrides ``security.redact_secrets:
|
||||
false``. That opt-out targets *live tool output* (e.g. working on the
|
||||
redactor itself); a summary is a persistence boundary where a leaked
|
||||
credential keeps re-entering prompts indefinitely.
|
||||
- ``redact_url_credentials=True`` — OAuth callback codes, magic-link
|
||||
tokens, and URL userinfo never need to survive summarization the way
|
||||
they must survive live navigation flows.
|
||||
"""
|
||||
return redact_sensitive_text(
|
||||
text or "",
|
||||
force=True,
|
||||
redact_url_credentials=True,
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_append(items: list[str], value: str, *, limit: int) -> None:
|
||||
value = value.strip()
|
||||
if value and value not in items and len(items) < limit:
|
||||
@@ -941,6 +971,102 @@ class ContextCompressor(ContextEngine):
|
||||
self.last_compression_rough_tokens = 0
|
||||
self.last_rough_tokens_when_real_prompt_fit = 0
|
||||
self.awaiting_real_usage_after_compression = False
|
||||
self._last_compression_telemetry = None
|
||||
self._active_compression_telemetry = None
|
||||
self._compression_telemetry_seed = None
|
||||
|
||||
def _begin_compression_telemetry(
|
||||
self,
|
||||
*,
|
||||
current_tokens: int | None,
|
||||
attempt_id: str | None = None,
|
||||
session_id: str | None = None,
|
||||
trigger_source: str | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Initialize content-free per-attempt compression telemetry."""
|
||||
seed = getattr(self, "_compression_telemetry_seed", None)
|
||||
if isinstance(seed, dict):
|
||||
attempt_id = attempt_id or seed.get("attempt_id")
|
||||
session_id = session_id or seed.get("session_id")
|
||||
trigger_source = trigger_source or seed.get("trigger_source")
|
||||
telemetry: Dict[str, Any] = {
|
||||
"event": "compression_attempt",
|
||||
"attempt_id": attempt_id or uuid.uuid4().hex,
|
||||
"session_id": session_id or "",
|
||||
"trigger_source": trigger_source or "unknown",
|
||||
"main_provider": self.provider or "",
|
||||
"main_model": self.model or "",
|
||||
"main_context_limit": _safe_int(self.context_length),
|
||||
"current_estimated_tokens": _safe_int(current_tokens),
|
||||
"effective_threshold": _safe_int(self.threshold_tokens),
|
||||
"protected_head_tokens": None,
|
||||
"protected_tail_tokens": None,
|
||||
"middle_window_tokens": None,
|
||||
"aux_prompt_tokens": None,
|
||||
"aux_output_reservation": None,
|
||||
"aux_provider": "",
|
||||
"aux_model": "",
|
||||
"effective_aux_context": None,
|
||||
"fit_margin": None,
|
||||
"chunking": False,
|
||||
"chunk_count": 0,
|
||||
"total_duration_ms": None,
|
||||
"aux_call_duration_ms": None,
|
||||
"fallback_used": False,
|
||||
"commit_status": "unknown",
|
||||
"split_status": "unknown",
|
||||
"failure_class": None,
|
||||
}
|
||||
self._active_compression_telemetry = telemetry
|
||||
self._last_compression_telemetry = telemetry
|
||||
return telemetry
|
||||
|
||||
def _record_compression_regions(
|
||||
self,
|
||||
*,
|
||||
head_messages: List[Dict[str, Any]],
|
||||
middle_messages: List[Dict[str, Any]],
|
||||
tail_messages: List[Dict[str, Any]],
|
||||
) -> None:
|
||||
telemetry = getattr(self, "_active_compression_telemetry", None)
|
||||
if not isinstance(telemetry, dict):
|
||||
return
|
||||
telemetry["protected_head_tokens"] = estimate_messages_tokens_rough(head_messages)
|
||||
telemetry["middle_window_tokens"] = estimate_messages_tokens_rough(middle_messages)
|
||||
telemetry["protected_tail_tokens"] = estimate_messages_tokens_rough(tail_messages)
|
||||
|
||||
def _record_aux_compression_call(
|
||||
self,
|
||||
*,
|
||||
prompt_messages: List[Dict[str, Any]],
|
||||
max_tokens: int | None,
|
||||
duration_ms: int,
|
||||
aux_provider: str | None = None,
|
||||
aux_model: str | None = None,
|
||||
effective_aux_context: int | None = None,
|
||||
) -> None:
|
||||
telemetry = getattr(self, "_active_compression_telemetry", None)
|
||||
if not isinstance(telemetry, dict):
|
||||
return
|
||||
telemetry["aux_prompt_tokens"] = estimate_messages_tokens_rough(prompt_messages)
|
||||
telemetry["aux_output_reservation"] = _safe_int(max_tokens)
|
||||
if aux_provider:
|
||||
telemetry["aux_provider"] = aux_provider
|
||||
if aux_model:
|
||||
telemetry["aux_model"] = aux_model
|
||||
if effective_aux_context is not None:
|
||||
telemetry["effective_aux_context"] = _safe_int(effective_aux_context)
|
||||
if (
|
||||
telemetry["effective_aux_context"] is not None
|
||||
and telemetry["aux_prompt_tokens"] is not None
|
||||
):
|
||||
telemetry["fit_margin"] = (
|
||||
telemetry["effective_aux_context"]
|
||||
- telemetry["aux_prompt_tokens"]
|
||||
- (telemetry["aux_output_reservation"] or 0)
|
||||
)
|
||||
previous = telemetry.get("aux_call_duration_ms") or 0
|
||||
telemetry["aux_call_duration_ms"] = previous + max(0, int(duration_ms))
|
||||
|
||||
def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> None:
|
||||
"""Clear all per-session compaction state at a real session boundary.
|
||||
@@ -983,6 +1109,9 @@ class ContextCompressor(ContextEngine):
|
||||
self.last_compression_rough_tokens = 0
|
||||
self.last_rough_tokens_when_real_prompt_fit = 0
|
||||
self.awaiting_real_usage_after_compression = False
|
||||
self._last_compression_telemetry = None
|
||||
self._active_compression_telemetry = None
|
||||
self._compression_telemetry_seed = None
|
||||
|
||||
def bind_session_state(self, session_db: Any = None, session_id: str = "") -> None:
|
||||
"""Bind the current session row so durable cooldowns can round-trip."""
|
||||
@@ -1231,6 +1360,11 @@ class ContextCompressor(ContextEngine):
|
||||
self.threshold_tokens = self._compute_threshold_tokens(
|
||||
context_length, self.threshold_percent, self.max_tokens,
|
||||
)
|
||||
# Re-apply the absolute token cap so it survives model switches
|
||||
# and fallback activations. The cap is a first-class config value
|
||||
# stored on the compressor instance, not a one-time post-construction
|
||||
# patch — this is why update_model() must re-apply it.
|
||||
self._apply_threshold_tokens_cap()
|
||||
# Recalculate token budgets for the new context length so the
|
||||
# compressor stays calibrated after a model switch (e.g. 200K → 32K).
|
||||
target_tokens = int(self.threshold_tokens * self.summary_target_ratio)
|
||||
@@ -1293,6 +1427,36 @@ class ContextCompressor(ContextEngine):
|
||||
return None
|
||||
return ivalue if ivalue > 0 else None
|
||||
|
||||
@staticmethod
|
||||
def _coerce_threshold_tokens_cap(value: Any) -> int | None:
|
||||
"""Normalize a threshold_tokens cap to a positive int or None.
|
||||
|
||||
None means "no absolute cap — use the ratio-based threshold only".
|
||||
Non-numeric or non-positive values are treated as None so a bad
|
||||
config value never silently caps the threshold at zero.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
ivalue = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return ivalue if ivalue > 0 else None
|
||||
|
||||
def _apply_threshold_tokens_cap(self) -> None:
|
||||
"""Apply the absolute token cap if configured.
|
||||
|
||||
After ``threshold_tokens`` is (re)computed from the ratio-based
|
||||
percent, clamp it to the cap so compression never fires later
|
||||
than the user's preferred absolute token count. The cap itself
|
||||
is clamped to the current context length so a cap larger than
|
||||
the model's window is a no-op (the ratio-based threshold wins).
|
||||
"""
|
||||
if self.threshold_tokens_cap is not None and self.threshold_tokens_cap > 0:
|
||||
_effective_cap = min(self.threshold_tokens_cap, self.context_length)
|
||||
if _effective_cap < self.threshold_tokens:
|
||||
self.threshold_tokens = _effective_cap
|
||||
|
||||
@staticmethod
|
||||
def _effective_threshold_percent(
|
||||
context_length: int, threshold_percent: float,
|
||||
@@ -1368,6 +1532,7 @@ class ContextCompressor(ContextEngine):
|
||||
abort_on_summary_failure: bool = False,
|
||||
max_tokens: int | None = None,
|
||||
model_thresholds: dict[str, float] | None = None,
|
||||
threshold_tokens_cap: Any = None,
|
||||
):
|
||||
self.model = model
|
||||
self.base_url = base_url
|
||||
@@ -1387,6 +1552,14 @@ class ContextCompressor(ContextEngine):
|
||||
model, self.model_thresholds, threshold_percent,
|
||||
)
|
||||
self.threshold_percent = self._base_threshold_percent
|
||||
# Absolute token cap from config (compression.threshold_tokens). When
|
||||
# set, the effective trigger point is min(ratio-based threshold, cap)
|
||||
# so compression never fires later than the user's preferred token
|
||||
# count regardless of which model is active. Applied in __init__ and
|
||||
# re-applied in update_model() so it survives model switches/fallbacks.
|
||||
self.threshold_tokens_cap = self._coerce_threshold_tokens_cap(
|
||||
threshold_tokens_cap,
|
||||
)
|
||||
self.protect_first_n = protect_first_n
|
||||
self.protect_last_n = protect_last_n
|
||||
self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80))
|
||||
@@ -1432,6 +1605,9 @@ class ContextCompressor(ContextEngine):
|
||||
self.threshold_tokens = self._compute_threshold_tokens(
|
||||
self.context_length, threshold_percent, self.max_tokens,
|
||||
)
|
||||
# Apply absolute token cap (compression.threshold_tokens) — takes
|
||||
# the lower of the ratio-based threshold and the cap.
|
||||
self._apply_threshold_tokens_cap()
|
||||
self.compression_count = 0
|
||||
|
||||
# Derive token budgets: ratio is relative to the threshold, not total context
|
||||
@@ -1522,6 +1698,9 @@ class ContextCompressor(ContextEngine):
|
||||
# succeeded. Silent recovery would hide the broken config.
|
||||
self._last_aux_model_failure_error: Optional[str] = None
|
||||
self._last_aux_model_failure_model: Optional[str] = None
|
||||
self._last_compression_telemetry: Optional[Dict[str, Any]] = None
|
||||
self._active_compression_telemetry: Optional[Dict[str, Any]] = None
|
||||
self._compression_telemetry_seed: Optional[Dict[str, Any]] = None
|
||||
|
||||
def update_from_response(self, usage: Dict[str, Any]):
|
||||
"""Update tracked token usage from API response."""
|
||||
@@ -1934,7 +2113,7 @@ class ContextCompressor(ContextEngine):
|
||||
elif isinstance(part, str):
|
||||
text_parts.append(part)
|
||||
content = "\n".join(text_parts)
|
||||
content = redact_sensitive_text(content or "")
|
||||
content = _redact_compaction_text(content or "")
|
||||
content = _MEDIA_DIRECTIVE_RE.sub("[media attachment]", content)
|
||||
# Strip inline reasoning blocks (<think>, <reasoning>, etc.) from
|
||||
# assistant content before it reaches the summarizer. Reasoning
|
||||
@@ -1967,7 +2146,7 @@ class ContextCompressor(ContextEngine):
|
||||
if isinstance(tc, dict):
|
||||
fn = tc.get("function", {})
|
||||
name = fn.get("name", "?")
|
||||
args = redact_sensitive_text(fn.get("arguments", ""))
|
||||
args = _redact_compaction_text(fn.get("arguments", ""))
|
||||
# Truncate long arguments but keep enough for context
|
||||
if len(args) > self._TOOL_ARGS_MAX:
|
||||
args = args[:self._TOOL_ARGS_HEAD] + "..."
|
||||
@@ -2010,7 +2189,7 @@ class ContextCompressor(ContextEngine):
|
||||
last_dropped_turns: list[str] = []
|
||||
|
||||
def _compact_fallback_turn(value: Any) -> str:
|
||||
text = redact_sensitive_text(_content_text_for_contains(value))
|
||||
text = _redact_compaction_text(_content_text_for_contains(value))
|
||||
text = re.sub(r"\bgh[pousr]_[A-Za-z0-9_]{8,}\b", "[REDACTED]", text)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
if len(text) > _FALLBACK_TURN_MAX_CHARS:
|
||||
@@ -2042,7 +2221,7 @@ class ContextCompressor(ContextEngine):
|
||||
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
for tc in msg.get("tool_calls") or []:
|
||||
name, raw_args = _extract_tool_call_name_and_args(tc)
|
||||
args = redact_sensitive_text(raw_args)
|
||||
args = _redact_compaction_text(raw_args)
|
||||
call_id = _extract_tool_call_id(tc)
|
||||
if call_id:
|
||||
call_id_to_tool[call_id] = (name, args)
|
||||
@@ -2180,7 +2359,7 @@ Continue from the most recent unfulfilled user ask and protected tail messages.
|
||||
|
||||
## Critical Context
|
||||
Summary generation was unavailable, so this is a best-effort deterministic fallback for {len(turns_to_summarize)} compacted message(s).{reason_text}"""
|
||||
summary = self._with_summary_prefix(redact_sensitive_text(body.strip()))
|
||||
summary = self._with_summary_prefix(_redact_compaction_text(body.strip()))
|
||||
if len(summary) > _FALLBACK_SUMMARY_MAX_CHARS:
|
||||
summary = summary[: _FALLBACK_SUMMARY_MAX_CHARS - 42].rstrip() + "\n...[fallback summary truncated]"
|
||||
return summary
|
||||
@@ -2209,6 +2388,10 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb
|
||||
_err_text = _err_text[:217].rstrip() + "..."
|
||||
self._last_aux_model_failure_error = _err_text
|
||||
self._last_aux_model_failure_model = self.summary_model
|
||||
telemetry = getattr(self, "_active_compression_telemetry", None)
|
||||
if isinstance(telemetry, dict):
|
||||
telemetry["fallback_used"] = True
|
||||
telemetry["failure_class"] = telemetry.get("failure_class") or "aux_model_fallback"
|
||||
self.summary_model = "" # empty = use main model
|
||||
self._clear_compression_failure_cooldown() # no cooldown — retry immediately
|
||||
|
||||
@@ -2243,6 +2426,15 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb
|
||||
)
|
||||
return None
|
||||
|
||||
# Strict-redact prompt inputs that bypass _serialize_for_summary:
|
||||
# a manual `/compress <focus>` string, and a previous summary that
|
||||
# may predate compaction redaction (resumed from a persisted
|
||||
# handoff message written before this boundary existed).
|
||||
if focus_topic:
|
||||
focus_topic = _redact_compaction_text(focus_topic)
|
||||
if self._previous_summary:
|
||||
self._previous_summary = _redact_compaction_text(self._previous_summary)
|
||||
|
||||
summary_budget = self._compute_summary_budget(turns_to_summarize)
|
||||
content_to_summarize = self._serialize_for_summary(turns_to_summarize)
|
||||
_sanitized_memory_context = sanitize_memory_context(memory_context)
|
||||
@@ -2500,13 +2692,43 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
}
|
||||
if self.summary_model:
|
||||
call_kwargs["model"] = self.summary_model
|
||||
_aux_provider = ""
|
||||
_aux_model = self.summary_model or ""
|
||||
_aux_context = None
|
||||
try:
|
||||
from agent.auxiliary_client import _resolve_task_provider_model
|
||||
|
||||
_resolved_provider, _resolved_model, _, _, _ = _resolve_task_provider_model(
|
||||
"compression",
|
||||
model=(self.summary_model or ""),
|
||||
)
|
||||
_aux_provider = _resolved_provider or ""
|
||||
_aux_model = _resolved_model or _aux_model or self.model or ""
|
||||
if _aux_model == self.model:
|
||||
_aux_context = self.context_length
|
||||
except Exception:
|
||||
pass
|
||||
# Compression is atomic: protect the in-flight summary call from a
|
||||
# mid-turn gateway interrupt. Without this, an incoming user message
|
||||
# aborts the summary and compression falls back to a degraded static
|
||||
# marker, losing the real handoff (#23975). Re-entrant: a main-model
|
||||
# retry (_generate_summary recursion) re-enters harmlessly.
|
||||
with aux_interrupt_protection():
|
||||
response = call_llm(**call_kwargs)
|
||||
_aux_call_start = time.monotonic()
|
||||
try:
|
||||
with aux_interrupt_protection():
|
||||
response = call_llm(**call_kwargs)
|
||||
finally:
|
||||
self._record_aux_compression_call(
|
||||
prompt_messages=call_kwargs["messages"],
|
||||
# Current main intentionally omits max_tokens from the aux
|
||||
# call (summary_budget is prompt-level guidance only) —
|
||||
# use .get() so the telemetry hook never breaks the call.
|
||||
max_tokens=call_kwargs.get("max_tokens"),
|
||||
duration_ms=int((time.monotonic() - _aux_call_start) * 1000),
|
||||
aux_provider=_aux_provider,
|
||||
aux_model=_aux_model,
|
||||
effective_aux_context=_aux_context,
|
||||
)
|
||||
# ``_validate_llm_response`` only guarantees ``choices[0].message``
|
||||
# exists, not that it's an object with ``.content``. Some
|
||||
# OpenAI-compatible proxies / local backends return a dict- or
|
||||
@@ -2547,7 +2769,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
content = stripped
|
||||
# Redact the summary output as well — the summarizer LLM may
|
||||
# ignore prompt instructions and echo back secrets verbatim.
|
||||
summary = redact_sensitive_text(content.strip())
|
||||
summary = _redact_compaction_text(content.strip())
|
||||
summary = self._ground_historical_task_snapshot(summary, turns_to_summarize)
|
||||
self._validate_summary_user_provenance(summary, has_user_turn)
|
||||
# Store for iterative updates on next compaction
|
||||
@@ -2859,6 +3081,68 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
"session with no user-authored turns"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _is_blank_user_turn(cls, message: Any) -> bool:
|
||||
"""Return whether *message* is an empty, non-summary user-role echo."""
|
||||
if not isinstance(message, dict) or message.get("role") != "user":
|
||||
return False
|
||||
if cls._has_compressed_summary_metadata(message):
|
||||
return False
|
||||
content = message.get("content")
|
||||
if cls._is_context_summary_content(content):
|
||||
return False
|
||||
if content is None or (isinstance(content, str) and not content.strip()):
|
||||
return True
|
||||
if not isinstance(content, list):
|
||||
return False
|
||||
if not content:
|
||||
return True
|
||||
for part in content:
|
||||
if isinstance(part, str):
|
||||
if part.strip():
|
||||
return False
|
||||
continue
|
||||
if isinstance(part, dict) and part.get("type") in {"text", "input_text"}:
|
||||
text = part.get("text")
|
||||
if isinstance(text, str) and not text.strip():
|
||||
continue
|
||||
# Images, audio, and unknown structured blocks are user input.
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _is_actionable_user_turn(cls, message: Any) -> bool:
|
||||
"""Return whether *message* contains user input worth anchoring."""
|
||||
if not isinstance(message, dict) or message.get("role") != "user":
|
||||
return False
|
||||
if cls._has_compressed_summary_metadata(message):
|
||||
return False
|
||||
content = message.get("content")
|
||||
if cls._is_context_summary_content(content):
|
||||
return False
|
||||
return not cls._is_blank_user_turn(message)
|
||||
|
||||
@classmethod
|
||||
def _blank_echo_indices_after(
|
||||
cls, messages: List[Dict[str, Any]], user_idx: int
|
||||
) -> set[int]:
|
||||
"""Return contiguous blank echoes safe to remove after a user event.
|
||||
|
||||
A blank user row is only a removable platform echo when an assistant turn
|
||||
immediately follows it. Otherwise it may be an intentional alternation
|
||||
placeholder for a transcript still being assembled.
|
||||
"""
|
||||
indices: set[int] = set()
|
||||
if user_idx < 0:
|
||||
return indices
|
||||
idx = user_idx + 1
|
||||
while idx < len(messages) and cls._is_blank_user_turn(messages[idx]):
|
||||
indices.add(idx)
|
||||
idx += 1
|
||||
if not indices or idx >= len(messages):
|
||||
return set()
|
||||
return indices if messages[idx].get("role") == "assistant" else set()
|
||||
|
||||
@classmethod
|
||||
def _derive_auto_focus_topic(
|
||||
cls,
|
||||
@@ -2873,7 +3157,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
if cls._is_synthetic_compression_user_turn(msg):
|
||||
continue
|
||||
content = msg.get("content")
|
||||
text = redact_sensitive_text(_content_text_for_contains(content).strip())
|
||||
text = _redact_compaction_text(_content_text_for_contains(content).strip())
|
||||
if not text:
|
||||
continue
|
||||
text = " ".join(text.split())
|
||||
@@ -2917,7 +3201,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
if not _is_real_user_message(msg):
|
||||
continue
|
||||
content = msg.get("content")
|
||||
text = redact_sensitive_text(_content_text_for_contains(content).strip())
|
||||
text = _redact_compaction_text(_content_text_for_contains(content).strip())
|
||||
if not text:
|
||||
continue
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
@@ -2969,6 +3253,125 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
return idx, cls._strip_summary_prefix(_content_text_for_contains(content))
|
||||
return None, ""
|
||||
|
||||
@classmethod
|
||||
def _strip_context_summary_handoff_message(
|
||||
cls,
|
||||
message: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Drop stale handoff data while preserving merged prior-tail content."""
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
|
||||
content = message.get("content")
|
||||
is_summary = (
|
||||
cls._is_context_summary_content(content)
|
||||
or cls._has_compressed_summary_metadata(message)
|
||||
)
|
||||
if not is_summary:
|
||||
return message.copy()
|
||||
|
||||
if isinstance(content, str):
|
||||
if _MERGED_SUMMARY_DELIMITER in content:
|
||||
prior = content.split(_MERGED_SUMMARY_DELIMITER, 1)[0].strip()
|
||||
if prior.startswith(_MERGED_PRIOR_CONTEXT_HEADER):
|
||||
prior = prior[len(_MERGED_PRIOR_CONTEXT_HEADER):].lstrip()
|
||||
if prior:
|
||||
unwrapped = message.copy()
|
||||
unwrapped["content"] = prior
|
||||
unwrapped.pop(COMPRESSED_SUMMARY_METADATA_KEY, None)
|
||||
return unwrapped
|
||||
else:
|
||||
marker_idx = content.find(_SUMMARY_END_MARKER)
|
||||
if marker_idx >= 0:
|
||||
remainder = content[marker_idx + len(_SUMMARY_END_MARKER):].lstrip()
|
||||
if remainder:
|
||||
unwrapped = message.copy()
|
||||
unwrapped["content"] = remainder
|
||||
unwrapped.pop(COMPRESSED_SUMMARY_METADATA_KEY, None)
|
||||
return unwrapped
|
||||
|
||||
if isinstance(content, list):
|
||||
prior_blocks: list[Any] = []
|
||||
found_delimiter = False
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
if _MERGED_SUMMARY_DELIMITER in item:
|
||||
before = item.split(_MERGED_SUMMARY_DELIMITER, 1)[0]
|
||||
if before.strip():
|
||||
prior_blocks.append(before)
|
||||
found_delimiter = True
|
||||
break
|
||||
prior_blocks.append(item)
|
||||
continue
|
||||
if isinstance(item, dict):
|
||||
text = item.get("text")
|
||||
if isinstance(text, str) and _MERGED_SUMMARY_DELIMITER in text:
|
||||
before = text.split(_MERGED_SUMMARY_DELIMITER, 1)[0]
|
||||
if before.strip():
|
||||
copied = item.copy()
|
||||
copied["text"] = before
|
||||
prior_blocks.append(copied)
|
||||
found_delimiter = True
|
||||
break
|
||||
prior_blocks.append(item.copy())
|
||||
continue
|
||||
prior_blocks.append(item)
|
||||
|
||||
if not found_delimiter:
|
||||
legacy_blocks: list[Any] = []
|
||||
found_marker = False
|
||||
for index, item in enumerate(content):
|
||||
text = item if isinstance(item, str) else item.get("text") if isinstance(item, dict) else None
|
||||
if not isinstance(text, str) or _SUMMARY_END_MARKER not in text:
|
||||
continue
|
||||
remainder = text.split(_SUMMARY_END_MARKER, 1)[1].lstrip()
|
||||
if remainder:
|
||||
if isinstance(item, dict):
|
||||
copied = item.copy()
|
||||
copied["text"] = remainder
|
||||
legacy_blocks.append(copied)
|
||||
else:
|
||||
legacy_blocks.append(remainder)
|
||||
for later in content[index + 1:]:
|
||||
legacy_blocks.append(later.copy() if isinstance(later, dict) else later)
|
||||
found_marker = True
|
||||
break
|
||||
if found_marker and legacy_blocks:
|
||||
unwrapped = message.copy()
|
||||
unwrapped["content"] = legacy_blocks
|
||||
unwrapped.pop(COMPRESSED_SUMMARY_METADATA_KEY, None)
|
||||
return unwrapped
|
||||
|
||||
if found_delimiter:
|
||||
for index, item in enumerate(prior_blocks):
|
||||
if isinstance(item, str):
|
||||
if item.lstrip().startswith(_MERGED_PRIOR_CONTEXT_HEADER):
|
||||
leading = item.lstrip()[len(_MERGED_PRIOR_CONTEXT_HEADER):].lstrip()
|
||||
if leading:
|
||||
prior_blocks[index] = leading
|
||||
else:
|
||||
prior_blocks.pop(index)
|
||||
break
|
||||
elif isinstance(item, dict) and isinstance(item.get("text"), str):
|
||||
text = item["text"]
|
||||
if text.lstrip().startswith(_MERGED_PRIOR_CONTEXT_HEADER):
|
||||
leading = text.lstrip()[len(_MERGED_PRIOR_CONTEXT_HEADER):].lstrip()
|
||||
if leading:
|
||||
copied = item.copy()
|
||||
copied["text"] = leading
|
||||
prior_blocks[index] = copied
|
||||
else:
|
||||
prior_blocks.pop(index)
|
||||
break
|
||||
|
||||
if prior_blocks:
|
||||
unwrapped = message.copy()
|
||||
unwrapped["content"] = prior_blocks
|
||||
unwrapped.pop(COMPRESSED_SUMMARY_METADATA_KEY, None)
|
||||
return unwrapped
|
||||
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tool-call / tool-result pair integrity helpers
|
||||
# ------------------------------------------------------------------
|
||||
@@ -3141,20 +3544,16 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
def _find_last_user_message_idx(
|
||||
self, messages: List[Dict[str, Any]], head_end: int
|
||||
) -> int:
|
||||
"""Return the index of the last user-role message at or after *head_end*, or -1.
|
||||
"""Return the latest actionable user turn at or after *head_end*, or -1.
|
||||
|
||||
A context-compaction handoff banner can be inserted as a ``role="user"``
|
||||
message (see the summary-role selection in ``compress``). It is internal
|
||||
continuity state, not a real user turn, so it must not be picked as the
|
||||
tail anchor — otherwise ``_ensure_last_user_message_in_tail`` protects
|
||||
the summary and rolls the genuine last user message into the next
|
||||
compaction, re-triggering the active-task loss the anchor exists to
|
||||
prevent.
|
||||
Compaction handoffs and empty platform echoes are continuity artifacts;
|
||||
neither may displace the request, correction, or completion that the tail
|
||||
anchor exists to preserve.
|
||||
"""
|
||||
for i in range(len(messages) - 1, head_end - 1, -1):
|
||||
msg = messages[i]
|
||||
if (
|
||||
msg.get("role") == "user"
|
||||
self._is_actionable_user_turn(msg)
|
||||
and not self._is_synthetic_compression_user_turn(msg)
|
||||
):
|
||||
return i
|
||||
@@ -3520,6 +3919,13 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
4. Summarize middle turns with structured LLM prompt
|
||||
5. On re-compression, iteratively update the previous summary
|
||||
|
||||
Blank platform-echo user rows trailing the latest actionable user
|
||||
turn are removed in the same cheap pre-pass phase as tool-result
|
||||
pruning — i.e. BEFORE any summary-abort early return. An aborted
|
||||
compression can therefore still hand back a modified list (echoes
|
||||
stripped, no turns summarized); this mirrors the long-standing
|
||||
Phase-1 pruning behavior, which likewise survives an abort.
|
||||
|
||||
After compression, orphaned tool_call / tool_result pairs are cleaned
|
||||
up so the API never receives mismatched IDs.
|
||||
|
||||
@@ -3553,6 +3959,8 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# static-fallback — the exact data-loss #29559 describes. Letting them
|
||||
# persist across compress() calls is safe because a successful summary
|
||||
# always clears both.
|
||||
telemetry = self._begin_compression_telemetry(current_tokens=current_tokens)
|
||||
telemetry["chunk_count"] = 0
|
||||
|
||||
# Manual /compress (force=True) bypasses the failure cooldown so the
|
||||
# user can retry immediately after an auto-compress abort. Without
|
||||
@@ -3572,6 +3980,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# returns here unchanged, and the CLI appears frozen.
|
||||
self._ineffective_compression_count += 1
|
||||
self._last_compression_savings_pct = 0.0
|
||||
telemetry["failure_class"] = "insufficient_messages"
|
||||
if not self.quiet_mode:
|
||||
logger.warning(
|
||||
"Cannot compress: only %d messages (need > %d). "
|
||||
@@ -3591,6 +4000,19 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
if pruned_count and not self.quiet_mode:
|
||||
logger.info("Pre-compression: pruned %d old tool result(s)", pruned_count)
|
||||
|
||||
latest_actionable_idx = self._find_last_user_message_idx(messages, 0)
|
||||
blank_echo_indices = self._blank_echo_indices_after(
|
||||
messages, latest_actionable_idx
|
||||
)
|
||||
if blank_echo_indices:
|
||||
messages = [
|
||||
message
|
||||
for idx, message in enumerate(messages)
|
||||
if idx not in blank_echo_indices
|
||||
]
|
||||
n_messages = len(messages)
|
||||
latest_actionable_idx = self._find_last_user_message_idx(messages, 0)
|
||||
|
||||
# Phase 2: Determine boundaries
|
||||
compress_start = self._protect_head_size(messages)
|
||||
compress_start = self._align_boundary_forward(messages, compress_start)
|
||||
@@ -3598,7 +4020,27 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# Use token-budget tail protection instead of fixed message count
|
||||
compress_end = self._find_tail_cut_by_tokens(messages, compress_start)
|
||||
|
||||
# A double role collision can merge the summary into the first tail
|
||||
# row. Keep an actionable user event out of that position by retaining
|
||||
# the genuinely older assistant/tool bridge when one exists.
|
||||
if compress_end == latest_actionable_idx:
|
||||
bridge_idx = latest_actionable_idx - 1
|
||||
if bridge_idx >= 0 and messages[bridge_idx].get("role") == "tool":
|
||||
bridge_idx = self._align_boundary_backward(
|
||||
messages, latest_actionable_idx
|
||||
)
|
||||
elif bridge_idx < 0 or messages[bridge_idx].get("role") != "assistant":
|
||||
bridge_idx = -1
|
||||
if bridge_idx > compress_start:
|
||||
compress_end = bridge_idx
|
||||
|
||||
if compress_start >= compress_end:
|
||||
self._record_compression_regions(
|
||||
head_messages=messages[:compress_start],
|
||||
middle_messages=[],
|
||||
tail_messages=messages[compress_end:],
|
||||
)
|
||||
telemetry["failure_class"] = "no_compressible_window"
|
||||
# No compressable window — the entire transcript fits within
|
||||
# the tail budget (soft_ceiling). Without recording this as
|
||||
# an ineffective compression the anti-thrashing guard in
|
||||
@@ -3660,6 +4102,13 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
else:
|
||||
self._summary_has_user_turn = real_user_present
|
||||
|
||||
self._record_compression_regions(
|
||||
head_messages=messages[:compress_start],
|
||||
middle_messages=turns_to_summarize,
|
||||
tail_messages=messages[compress_end:],
|
||||
)
|
||||
telemetry["chunk_count"] = 1 if turns_to_summarize else 0
|
||||
|
||||
if not self.quiet_mode:
|
||||
logger.info(
|
||||
"Context compression triggered (%d tokens >= %d threshold)",
|
||||
@@ -3719,6 +4168,12 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
self._last_summary_dropped_count = 0 # nothing actually dropped
|
||||
self._last_summary_fallback_used = False
|
||||
self._last_compress_aborted = True
|
||||
if self._last_summary_auth_failure:
|
||||
telemetry["failure_class"] = "summary_auth_failure"
|
||||
elif self._last_summary_network_failure:
|
||||
telemetry["failure_class"] = "summary_network_failure"
|
||||
else:
|
||||
telemetry["failure_class"] = "summary_generation_aborted"
|
||||
if not self.quiet_mode:
|
||||
if self._last_summary_auth_failure:
|
||||
logger.warning(
|
||||
@@ -3752,19 +4207,15 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# Phase 4: Assemble compressed message list
|
||||
compressed = []
|
||||
for i in range(compress_start):
|
||||
# If an earlier compaction handoff is in the protected head
|
||||
# (common after resume / in-place compaction), do not carry it
|
||||
# forward verbatim. It has already been rehydrated into
|
||||
# _previous_summary above and _generate_summary() will emit the
|
||||
# updated replacement below. Keeping both makes repeated
|
||||
# compactions accumulate old summaries and prevents the live prompt
|
||||
# from actually shrinking.
|
||||
if (
|
||||
summary_idx is not None
|
||||
and i == summary_idx
|
||||
and self._is_context_summary_content(messages[i].get("content"))
|
||||
):
|
||||
continue
|
||||
# An earlier compaction handoff in the protected head (common
|
||||
# after resume / in-place compaction) must not be carried forward
|
||||
# verbatim — it is already rehydrated into _previous_summary and
|
||||
# _generate_summary() emits the updated replacement below.
|
||||
# _strip_context_summary_handoff_message() handles both shapes:
|
||||
# standalone handoffs strip to None (dropped), merged handoffs
|
||||
# unwrap to their genuine prior-tail content (preserved). Do NOT
|
||||
# short-circuit on summary_idx here: a merged handoff carries real
|
||||
# user content that a blanket skip would silently delete.
|
||||
msg = _fresh_compaction_message_copy(messages[i])
|
||||
if i == 0 and msg.get("role") == "system":
|
||||
existing = msg.get("content")
|
||||
@@ -3774,7 +4225,9 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
existing,
|
||||
"\n\n" + _compression_note if isinstance(existing, str) and existing else _compression_note,
|
||||
)
|
||||
compressed.append(msg)
|
||||
stripped = self._strip_context_summary_handoff_message(msg)
|
||||
if stripped is not None:
|
||||
compressed.append(stripped)
|
||||
|
||||
# If LLM summary failed, insert a deterministic fallback so the model
|
||||
# gets at least locally recoverable continuity anchors instead of a
|
||||
@@ -3785,14 +4238,26 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
n_dropped = compress_end - compress_start
|
||||
self._last_summary_dropped_count = n_dropped
|
||||
self._last_summary_fallback_used = True
|
||||
telemetry["fallback_used"] = True
|
||||
telemetry["failure_class"] = telemetry.get("failure_class") or "summary_generation_failed"
|
||||
summary = self._build_static_fallback_summary(
|
||||
turns_to_summarize,
|
||||
reason=self._last_summary_error,
|
||||
)
|
||||
|
||||
tail_messages: List[Dict[str, Any]] = []
|
||||
for i in range(compress_end, n_messages):
|
||||
msg = _fresh_compaction_message_copy(messages[i])
|
||||
stripped = self._strip_context_summary_handoff_message(msg)
|
||||
if stripped is not None:
|
||||
tail_messages.append(stripped)
|
||||
|
||||
_merge_summary_into_tail = False
|
||||
last_head_role = compressed[-1].get("role", "user") if compressed else "user"
|
||||
first_tail_role = messages[compress_end].get("role", "user") if compress_end < n_messages else "user"
|
||||
# NOTE: derive the tail's leading role from tail_messages (post
|
||||
# handoff-strip), not messages[compress_end] — a stripped stale
|
||||
# handoff must not influence alternation-safe role selection.
|
||||
first_tail_role = tail_messages[0].get("role", "user") if tail_messages else None
|
||||
# When the only protected head message is the system prompt, the
|
||||
# summary becomes the first *visible* message in the API request
|
||||
# (most adapters — Anthropic, Bedrock — send the system prompt as
|
||||
@@ -3820,11 +4285,9 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# always has at least one user turn.
|
||||
if not _force_user_leading:
|
||||
_user_survives = any(
|
||||
messages[i].get("role") == "user"
|
||||
for i in range(0, compress_start)
|
||||
message.get("role") == "user" for message in compressed
|
||||
) or any(
|
||||
messages[i].get("role") == "user"
|
||||
for i in range(compress_end, n_messages)
|
||||
message.get("role") == "user" for message in tail_messages
|
||||
)
|
||||
if not _user_survives:
|
||||
_force_user_leading = True
|
||||
@@ -3836,7 +4299,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
summary_role = "assistant"
|
||||
# If the chosen role collides with the tail AND flipping wouldn't
|
||||
# collide with the head, flip it.
|
||||
if summary_role == first_tail_role:
|
||||
if first_tail_role and summary_role == first_tail_role:
|
||||
flipped = "assistant" if summary_role == "user" else "user"
|
||||
if flipped != last_head_role and not _force_user_leading:
|
||||
summary_role = flipped
|
||||
@@ -3845,7 +4308,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
# (e.g. head=assistant, tail=user — neither role works).
|
||||
# Merge the summary into the first tail message instead
|
||||
# of inserting a standalone message that breaks alternation.
|
||||
_merge_summary_into_tail = True
|
||||
_merge_summary_into_tail = bool(tail_messages)
|
||||
|
||||
# When the summary lands as a standalone role="user" message,
|
||||
# weak models read the verbatim "## Active Task" quote of a past
|
||||
@@ -3867,9 +4330,8 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
),
|
||||
})
|
||||
|
||||
for i in range(compress_end, n_messages):
|
||||
msg = _fresh_compaction_message_copy(messages[i])
|
||||
if _merge_summary_into_tail and i == compress_end:
|
||||
for tail_idx, msg in enumerate(tail_messages):
|
||||
if _merge_summary_into_tail and tail_idx == 0:
|
||||
# Merge the summary into the first tail message, but place
|
||||
# the END MARKER at the very end so the model sees an
|
||||
# unambiguous boundary. Old tail content is preserved as
|
||||
|
||||
@@ -30,10 +30,12 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
import threading
|
||||
from datetime import datetime
|
||||
@@ -173,6 +175,43 @@ def _session_was_rotated_by_compression(session_db: Any, session_id: str) -> boo
|
||||
)
|
||||
|
||||
|
||||
def _emit_compression_attempt_telemetry(
|
||||
agent: Any,
|
||||
*,
|
||||
started_at: float,
|
||||
commit_status: str,
|
||||
split_status: str,
|
||||
failure_class: str | None = None,
|
||||
) -> None:
|
||||
"""Emit one content-free JSON log line for a compression attempt."""
|
||||
try:
|
||||
telemetry = getattr(agent.context_compressor, "_last_compression_telemetry", None)
|
||||
if not isinstance(telemetry, dict):
|
||||
telemetry = {}
|
||||
payload = dict(telemetry)
|
||||
payload.setdefault("event", "compression_attempt")
|
||||
payload.setdefault("attempt_id", getattr(agent, "_compression_attempt_id", "") or uuid.uuid4().hex)
|
||||
payload.setdefault("session_id", getattr(agent, "session_id", "") or "")
|
||||
payload["total_duration_ms"] = int((time.monotonic() - started_at) * 1000)
|
||||
payload["commit_status"] = commit_status
|
||||
payload["split_status"] = split_status
|
||||
if failure_class:
|
||||
payload["failure_class"] = failure_class
|
||||
payload.setdefault("chunking", False)
|
||||
payload.setdefault("chunk_count", 0)
|
||||
payload["fallback_used"] = bool(
|
||||
payload.get("fallback_used")
|
||||
or getattr(agent.context_compressor, "_last_summary_fallback_used", False)
|
||||
or getattr(agent.context_compressor, "_last_aux_model_failure_model", None)
|
||||
)
|
||||
logger.info(
|
||||
"context compression attempt telemetry: %s",
|
||||
json.dumps(payload, sort_keys=True, separators=(",", ":")),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("failed to emit compression attempt telemetry: %s", exc)
|
||||
|
||||
|
||||
def _compression_lock_holder(agent: Any) -> str:
|
||||
"""Build a unique holder id for the lock: pid:tid:agent-instance:uuid.
|
||||
|
||||
@@ -904,6 +943,19 @@ def compress_context(
|
||||
):
|
||||
raise RuntimeError("a compression notification is already pending")
|
||||
|
||||
_attempt_started_at = time.monotonic()
|
||||
_attempt_id = uuid.uuid4().hex
|
||||
_trigger_source = "manual" if force else "auto"
|
||||
try:
|
||||
agent._compression_attempt_id = _attempt_id
|
||||
setattr(agent.context_compressor, "_compression_telemetry_seed", {
|
||||
"attempt_id": _attempt_id,
|
||||
"session_id": agent.session_id or "",
|
||||
"trigger_source": _trigger_source,
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Codex app-server sessions: the codex agent owns the real thread context;
|
||||
# Hermes' summarizer would only rewrite a local mirror without shrinking
|
||||
# the actual thread (#36801). Route compaction to the app server's own
|
||||
@@ -1110,6 +1162,18 @@ def compress_context(
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
try:
|
||||
if hasattr(agent.context_compressor, "_begin_compression_telemetry"):
|
||||
agent.context_compressor._begin_compression_telemetry(current_tokens=approx_tokens)
|
||||
except Exception:
|
||||
pass
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class="lock_contended",
|
||||
)
|
||||
return messages, _existing_sp
|
||||
_lock_released = False
|
||||
|
||||
@@ -1235,7 +1299,7 @@ def compress_context(
|
||||
messages_before_compression = copy.deepcopy(messages)
|
||||
_activity_heartbeat = _CompressionActivityHeartbeat(agent).start()
|
||||
compressed = compress_fn(messages, **compress_kwargs)
|
||||
except BaseException:
|
||||
except BaseException as _compress_exc:
|
||||
# ANY exception after lock acquisition — memory hook, capability
|
||||
# inspection, engine lookup, or compress() — must release the lock so
|
||||
# the session isn't permanently blocked from future compression.
|
||||
@@ -1243,6 +1307,13 @@ def compress_context(
|
||||
_activity_heartbeat.stop("context compression failed")
|
||||
_activity_heartbeat = None
|
||||
_release_lock()
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class=f"exception:{type(_compress_exc).__name__}",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
if _activity_heartbeat is not None:
|
||||
@@ -1278,6 +1349,16 @@ def compress_context(
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class=(
|
||||
getattr(agent.context_compressor, "_last_summary_error", None)
|
||||
and "summary_generation_aborted"
|
||||
),
|
||||
)
|
||||
return messages, _existing_sp
|
||||
finally:
|
||||
_release_lock()
|
||||
@@ -1296,6 +1377,13 @@ def compress_context(
|
||||
_existing_sp = getattr(agent, "_cached_system_prompt", None)
|
||||
if not _existing_sp:
|
||||
_existing_sp = agent._build_system_prompt(system_message)
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status="aborted",
|
||||
split_status="aborted",
|
||||
failure_class="no_progress",
|
||||
)
|
||||
_release_lock()
|
||||
return messages, _existing_sp
|
||||
|
||||
@@ -1377,7 +1465,9 @@ def compress_context(
|
||||
agent._cached_system_prompt = new_system_prompt
|
||||
|
||||
_session_commit_succeeded = False
|
||||
split_status = "not_applicable"
|
||||
if agent._session_db:
|
||||
split_status = "pending"
|
||||
try:
|
||||
# Trigger memory extraction on the current session before the
|
||||
# transcript is rewritten (runs in BOTH modes — the logical
|
||||
@@ -1405,6 +1495,7 @@ def compress_context(
|
||||
# WITHOUT destroying history, unlike a hard replace_messages).
|
||||
# See #38763.
|
||||
agent._session_db.archive_and_compact(agent.session_id, compressed)
|
||||
split_status = "in_place_committed"
|
||||
# Reset the flush identity set so the next turn's appends are
|
||||
# diffed against the COMPACTED transcript: the compacted dicts
|
||||
# are passed as conversation_history next turn and skipped by
|
||||
@@ -1521,6 +1612,7 @@ def compress_context(
|
||||
agent._session_db_created = True
|
||||
raise
|
||||
agent._session_db_created = True
|
||||
split_status = "rotated_committed"
|
||||
# Carry a persistent /goal onto the continuation session.
|
||||
# Compression mints a fresh child id; load_goal does a flat
|
||||
# per-session lookup with no parent walk, so without this an
|
||||
@@ -1559,6 +1651,7 @@ def compress_context(
|
||||
}
|
||||
_session_commit_succeeded = True
|
||||
except Exception as e:
|
||||
split_status = "aborted" if locals().get("old_session_id") is None and not in_place else "failed_not_indexed"
|
||||
# If the rotation rolled back to the parent (orphan-avoidance
|
||||
# above), agent.session_id is the still-indexed parent and
|
||||
# old_session_id was cleared — so this is recovery, not an
|
||||
@@ -1701,6 +1794,18 @@ def compress_context(
|
||||
agent.session_id or "none", _pre_msg_count, len(compressed),
|
||||
f"{_compressed_est:,}",
|
||||
)
|
||||
_commit_status = "committed" if split_status in {"not_applicable", "in_place_committed", "rotated_committed"} else "aborted"
|
||||
_emit_compression_attempt_telemetry(
|
||||
agent,
|
||||
started_at=_attempt_started_at,
|
||||
commit_status=_commit_status,
|
||||
split_status=split_status,
|
||||
failure_class=(
|
||||
"session_split_failed"
|
||||
if split_status in {"failed_not_indexed", "aborted"}
|
||||
else None
|
||||
),
|
||||
)
|
||||
return compressed, new_system_prompt
|
||||
finally:
|
||||
# Release the lock on the OLD session_id only AFTER rotation completed
|
||||
|
||||
@@ -424,6 +424,15 @@ compression:
|
||||
# "claude-sonnet": 0.35
|
||||
# "gpt-5": 0.30
|
||||
|
||||
# Optional absolute token cap for the compression trigger (default: null = disabled).
|
||||
# When set, compression fires at the LOWER of the ratio-based threshold and this
|
||||
# absolute token count — first-fires-wins. It never fires later than this count
|
||||
# regardless of which model is active (useful when switching between models with
|
||||
# very different context windows). Clamped to the model's context length at
|
||||
# apply-time, so a cap above the window is a no-op (ratio-based threshold wins).
|
||||
# Survives model switches and fallback activations.
|
||||
# threshold_tokens: 200000
|
||||
|
||||
# Existing Codex gpt-5.5 behavior: raise Hermes' compaction trigger to 85%
|
||||
# for the ChatGPT Codex OAuth route. Set false to opt back down to threshold.
|
||||
codex_gpt55_autoraise: true
|
||||
|
||||
@@ -772,6 +772,16 @@ def load_cli_config() -> Dict[str, Any]:
|
||||
if redact is not None:
|
||||
os.environ["HERMES_REDACT_SECRETS"] = str(redact).lower()
|
||||
|
||||
# Session-search index knobs (hermes_state reads the env carriers).
|
||||
sessions_config = defaults.get("sessions", {})
|
||||
if isinstance(sessions_config, dict):
|
||||
if "cjk_fts" in sessions_config:
|
||||
os.environ["HERMES_CJK_FTS"] = str(sessions_config["cjk_fts"])
|
||||
if "search_slow_ms" in sessions_config:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(
|
||||
sessions_config["search_slow_ms"]
|
||||
)
|
||||
|
||||
return defaults
|
||||
|
||||
# Load configuration at module startup
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
knoal
|
||||
# PR #64067 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
vexclawx31
|
||||
# PR #33215 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
LevSky22
|
||||
# PR #66069 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
87degrees
|
||||
# PR #52923 salvage (slack ping/pong staleness)
|
||||
@@ -0,0 +1,2 @@
|
||||
kaiyisg
|
||||
# PR #24848 salvage
|
||||
@@ -0,0 +1 @@
|
||||
DanielMaly
|
||||
@@ -0,0 +1,2 @@
|
||||
MrAbsaroka
|
||||
# PR #40064 salvage (slack dedup TTL)
|
||||
@@ -0,0 +1 @@
|
||||
AndrewMoryakov
|
||||
@@ -1481,6 +1481,14 @@ def _bridge_max_turns_from_config(home: "Path") -> None:
|
||||
agent_cfg = cfg.get("agent", {})
|
||||
if isinstance(agent_cfg, dict) and "max_turns" in agent_cfg:
|
||||
os.environ["HERMES_MAX_ITERATIONS"] = str(agent_cfg["max_turns"])
|
||||
# config-authoritative knobs for the session-search index (config.yaml
|
||||
# sessions.* wins over stale env; env stays the cross-process carrier).
|
||||
sessions_cfg = cfg.get("sessions", {})
|
||||
if isinstance(sessions_cfg, dict):
|
||||
if "cjk_fts" in sessions_cfg:
|
||||
os.environ["HERMES_CJK_FTS"] = str(sessions_cfg["cjk_fts"])
|
||||
if "search_slow_ms" in sessions_cfg:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(sessions_cfg["search_slow_ms"])
|
||||
|
||||
|
||||
def _current_max_iterations() -> int:
|
||||
@@ -1763,6 +1771,16 @@ if _config_path.exists():
|
||||
os.environ["HERMES_AUTO_CONTINUE_FRESHNESS"] = str(
|
||||
_agent_cfg["gateway_auto_continue_freshness"]
|
||||
)
|
||||
# config-authoritative knobs for the session-search index; same
|
||||
# bridge semantics as the agent settings above.
|
||||
_sessions_cfg = _cfg.get("sessions", {})
|
||||
if _sessions_cfg and isinstance(_sessions_cfg, dict):
|
||||
if "cjk_fts" in _sessions_cfg:
|
||||
os.environ["HERMES_CJK_FTS"] = str(_sessions_cfg["cjk_fts"])
|
||||
if "search_slow_ms" in _sessions_cfg:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(
|
||||
_sessions_cfg["search_slow_ms"]
|
||||
)
|
||||
_display_cfg = _cfg.get("display", {})
|
||||
if _display_cfg and isinstance(_display_cfg, dict):
|
||||
if "busy_input_mode" in _display_cfg:
|
||||
@@ -11818,6 +11836,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
# (mirrors the same field's treatment in
|
||||
# build_session_context_prompt via _format_untrusted_prompt_value).
|
||||
_safe_user_name = neutralize_untrusted_inline_text(source.user_name)
|
||||
# On Slack, expose the current author's verifiable user ID next to
|
||||
# the display name (#17916): "mention me again" requests need a
|
||||
# trusted `<@U...>` target for the CURRENT speaker — display names
|
||||
# are ambiguous and historical mentions may point at someone else.
|
||||
# The user_id comes from the Slack event envelope (not
|
||||
# user-editable text), so it does not need neutralization.
|
||||
if source.platform == Platform.SLACK and source.user_id:
|
||||
_safe_user_name = (
|
||||
f"{_safe_user_name} | Slack user <@{source.user_id}>"
|
||||
)
|
||||
message_text = f"[{_safe_user_name}] {message_text}"
|
||||
|
||||
# Prepend channel context from history backfill (if any). This
|
||||
@@ -17644,6 +17672,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
("compression", "enabled"),
|
||||
("compression", "threshold"),
|
||||
("compression", "model_thresholds"),
|
||||
("compression", "threshold_tokens"),
|
||||
("compression", "codex_gpt55_autoraise"),
|
||||
("compression", "codex_app_server_auto"),
|
||||
("compression", "target_ratio"),
|
||||
|
||||
+51
-1
@@ -17,7 +17,7 @@ import threading
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -526,6 +526,13 @@ def build_session_context_prompt(
|
||||
"current message's Slack block/attachment payload when available, but "
|
||||
"you still cannot call Slack APIs yourself."
|
||||
)
|
||||
if context.shared_multi_user_session:
|
||||
lines.append(
|
||||
"In shared Slack threads, use the current turn's sender prefix "
|
||||
"as the only verified current-author mention target. Do not "
|
||||
"guess or reuse `<@U...>` mentions from names, memory, or prior "
|
||||
"conversation history."
|
||||
)
|
||||
elif context.source.platform == Platform.DISCORD:
|
||||
# Inject the Discord IDs block only when the agent actually has
|
||||
# Discord tools loaded this session — i.e. the user opted into
|
||||
@@ -691,6 +698,11 @@ class SessionEntry:
|
||||
display_name: Optional[str] = None
|
||||
platform: Optional[Platform] = None
|
||||
chat_type: str = "dm"
|
||||
|
||||
# Lightweight persisted key/value state scoped to this session entry
|
||||
# (e.g. Slack thread-context watermarks). Survives gateway restarts via
|
||||
# the routing index; must stay small and JSON-serializable.
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# Token tracking
|
||||
input_tokens: int = 0
|
||||
@@ -760,6 +772,7 @@ class SessionEntry:
|
||||
"display_name": self.display_name,
|
||||
"platform": self.platform.value if self.platform else None,
|
||||
"chat_type": self.chat_type,
|
||||
"metadata": self.metadata,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"cache_read_tokens": self.cache_read_tokens,
|
||||
@@ -839,6 +852,7 @@ class SessionEntry:
|
||||
display_name=data.get("display_name"),
|
||||
platform=platform,
|
||||
chat_type=data.get("chat_type", "dm"),
|
||||
metadata=dict(data.get("metadata") or {}),
|
||||
input_tokens=data.get("input_tokens", 0),
|
||||
output_tokens=data.get("output_tokens", 0),
|
||||
cache_read_tokens=data.get("cache_read_tokens", 0),
|
||||
@@ -2149,6 +2163,42 @@ class SessionStore:
|
||||
display_name=entry.display_name,
|
||||
)
|
||||
|
||||
def get_session_metadata(
|
||||
self,
|
||||
session_key: str,
|
||||
key: str,
|
||||
default: Any = None,
|
||||
) -> Any:
|
||||
"""Return a metadata value stored on a live session entry."""
|
||||
with self._lock:
|
||||
self._ensure_loaded_locked()
|
||||
entry = self._entries.get(session_key)
|
||||
if entry is None:
|
||||
return default
|
||||
return entry.metadata.get(key, default)
|
||||
|
||||
def set_session_metadata(
|
||||
self,
|
||||
session_key: str,
|
||||
key: str,
|
||||
value: Any,
|
||||
) -> bool:
|
||||
"""Persist a metadata value on a live session entry.
|
||||
|
||||
Values must be small and JSON-serializable — they are written into
|
||||
the routing index (state.db gateway_routing table + the legacy
|
||||
sessions.json mirror) so they survive gateway restarts.
|
||||
"""
|
||||
with self._lock:
|
||||
self._ensure_loaded_locked()
|
||||
entry = self._entries.get(session_key)
|
||||
if entry is None:
|
||||
return False
|
||||
entry.metadata[key] = value
|
||||
entry.updated_at = _now()
|
||||
self._save()
|
||||
return True
|
||||
|
||||
def set_model_override(
|
||||
self, session_key: str, override: Optional[Dict[str, Any]]
|
||||
) -> None:
|
||||
|
||||
@@ -1469,6 +1469,10 @@ DEFAULT_CONFIG = {
|
||||
# floored at 0.75 (raise-only) so compaction
|
||||
# doesn't fire with half the window still free;
|
||||
# set this above 0.75 to override the floor.
|
||||
"threshold_tokens": None, # absolute token cap — when set, compression
|
||||
# triggers at the lower of the ratio-based
|
||||
# threshold and this token count. Clamped to
|
||||
# the model's context length at apply-time.
|
||||
"target_ratio": 0.20, # fraction of threshold to preserve as recent tail
|
||||
"protect_last_n": 20, # minimum recent messages to keep uncompressed
|
||||
"max_attempts": 3, # compression retry rounds before a turn gives up
|
||||
@@ -3249,6 +3253,20 @@ DEFAULT_CONFIG = {
|
||||
# enforcement is a copy/gating change, not new migration code.
|
||||
# "off": suppress the notice entirely.
|
||||
"fts_optimize_notice": "advise",
|
||||
# CJK-bigram search index (messages_fts_cjk, cjk_unicode61 loadable
|
||||
# tokenizer). When the extension is built (native/fts5_cjk/build.sh →
|
||||
# ~/.hermes/lib/libfts5_cjk.so), 1-2 char CJK terms (일본, 项目, ...)
|
||||
# get index-speed exact matching instead of LIKE full-table scans.
|
||||
# True (default): use the index when the extension is present; the
|
||||
# setting is inert when it isn't. False: never load the extension or
|
||||
# serve the cjk index. Bridged to HERMES_CJK_FTS (internal carrier).
|
||||
"cjk_fts": True,
|
||||
# Slow session-search log threshold in milliseconds: searches at or
|
||||
# above it log one INFO line with the routing path taken (fts_cjk /
|
||||
# fts5 / trigram / like_scan) so latency regressions stay
|
||||
# attributable per query shape. 0 logs every search. Bridged to
|
||||
# HERMES_SEARCH_SLOW_MS (internal carrier).
|
||||
"search_slow_ms": 1000,
|
||||
},
|
||||
|
||||
# Contextual first-touch onboarding hints (see agent/onboarding.py).
|
||||
@@ -8548,6 +8566,14 @@ def show_config():
|
||||
print(f" Enabled: {'yes' if enabled else 'no'}")
|
||||
if enabled:
|
||||
print(f" Threshold: {compression.get('threshold', 0.50) * 100:.0f}%")
|
||||
_tt = compression.get('threshold_tokens')
|
||||
if _tt is not None:
|
||||
try:
|
||||
_tt = int(_tt)
|
||||
if _tt > 0:
|
||||
print(f" Token cap: {_tt:,} tokens (takes lower of ratio vs absolute)")
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
print(f" Target ratio: {compression.get('target_ratio', 0.20) * 100:.0f}% of threshold preserved")
|
||||
print(f" Protect last: {compression.get('protect_last_n', 20)} messages")
|
||||
print(f" Protect first: {compression.get('protect_first_n', 3)} non-system head messages")
|
||||
|
||||
@@ -6406,6 +6406,10 @@ def _print_fts_optimize_available_notice() -> None:
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1"
|
||||
).fetchone()
|
||||
or db._conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key IN "
|
||||
"('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1"
|
||||
).fetchone()
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
@@ -2407,31 +2407,78 @@ def _run_composite_fallback(plugin_keys, plugin_labels, plugin_selected,
|
||||
print()
|
||||
|
||||
|
||||
# Public aliases for the sidecar helpers defined above — the dashboard
|
||||
# (web_server.py) imports these names; the CLI paths use the private ones.
|
||||
write_catalog_sidecar = _write_catalog_sidecar
|
||||
read_catalog_sidecar = _read_catalog_sidecar
|
||||
|
||||
|
||||
def dashboard_install_plugin(
|
||||
identifier: str,
|
||||
*,
|
||||
force: bool,
|
||||
enable: bool,
|
||||
catalog_name: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Non-interactive install for the web dashboard. Returns a JSON-serializable dict."""
|
||||
"""Non-interactive install for the web dashboard. Returns a JSON-serializable dict.
|
||||
|
||||
When *catalog_name* is given the identifier is resolved from the plugin
|
||||
catalog and the pinned commit SHA is checked out (``ref=``). Removed
|
||||
(blocklisted) plugins are refused with the recorded reason — the
|
||||
dashboard deliberately has no bypass flag (CLI-only decision).
|
||||
"""
|
||||
warnings: list[str] = []
|
||||
try:
|
||||
git_url, _subdir = _resolve_git_url(identifier)
|
||||
if git_url.startswith(("http://", "file://")):
|
||||
warnings.append(
|
||||
"Insecure URL scheme; prefer https:// or git@ for production installs.",
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
entry = None
|
||||
ref: Optional[str] = None
|
||||
|
||||
if catalog_name:
|
||||
from hermes_cli.plugin_catalog import find_removed, get_catalog_entry
|
||||
|
||||
removed = find_removed(catalog_name)
|
||||
if removed is not None:
|
||||
detail = removed.reason or "no reason recorded"
|
||||
if removed.date:
|
||||
detail += f" (removed {removed.date})"
|
||||
return {
|
||||
"ok": False,
|
||||
"error": (
|
||||
f"Plugin '{removed.name}' was removed from the Hermes "
|
||||
f"plugin catalog and is blocked from installation: {detail}"
|
||||
),
|
||||
}
|
||||
entry = get_catalog_entry(catalog_name)
|
||||
if entry is None:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": f"'{catalog_name}' is not in the Hermes plugin catalog.",
|
||||
}
|
||||
identifier = f"{entry.repo}#{entry.subdir}" if entry.subdir else entry.repo
|
||||
ref = entry.sha
|
||||
else:
|
||||
try:
|
||||
git_url, _subdir = _resolve_git_url(identifier)
|
||||
if git_url.startswith(("http://", "file://")):
|
||||
warnings.append(
|
||||
"Insecure URL scheme; prefer https:// or git@ for production installs.",
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
target, installed_manifest, installed_name = _install_plugin_core(
|
||||
identifier,
|
||||
force=force,
|
||||
ref=ref,
|
||||
)
|
||||
except PluginOperationError as exc:
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
if entry is not None:
|
||||
try:
|
||||
write_catalog_sidecar(target, entry)
|
||||
except OSError as exc:
|
||||
warnings.append(f"Could not record catalog provenance: {exc}")
|
||||
|
||||
missing_env = _missing_requires_env_names(installed_manifest)
|
||||
if enable:
|
||||
en = _get_enabled_set()
|
||||
|
||||
+135
-10
@@ -18849,12 +18849,27 @@ class _AgentPluginInstallBody(BaseModel):
|
||||
identifier: str
|
||||
force: bool = False
|
||||
enable: bool = True
|
||||
catalog_name: Optional[str] = None
|
||||
|
||||
|
||||
def _strip_dashboard_manifest(p: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {k: v for k, v in p.items() if not k.startswith("_")}
|
||||
|
||||
|
||||
def _plugin_runtime_status(aliases: set, enabled_set: set, disabled_set: set) -> str:
|
||||
"""Map a plugin's name aliases onto enabled/disabled/inactive.
|
||||
|
||||
Both the path-derived key (nested category plugins) and the bare
|
||||
manifest name count for enabled/disabled state, matching the runtime
|
||||
loader's back-compat lookup.
|
||||
"""
|
||||
if aliases & disabled_set:
|
||||
return "disabled"
|
||||
if aliases & enabled_set:
|
||||
return "enabled"
|
||||
return "inactive"
|
||||
|
||||
|
||||
def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
"""Agent discovery + dashboard manifests + optional provider picker metadata."""
|
||||
from hermes_cli.plugins_cmd import (
|
||||
@@ -18866,6 +18881,7 @@ def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
_get_enabled_set,
|
||||
_read_manifest as _read_plugin_manifest_at,
|
||||
)
|
||||
from hermes_cli.plugin_catalog import find_removed
|
||||
|
||||
dashboard_list = _get_dashboard_plugins()
|
||||
dash_by_name = {str(p["name"]): p for p in dashboard_list}
|
||||
@@ -18881,18 +18897,10 @@ def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
rows: List[Dict[str, Any]] = []
|
||||
|
||||
for name, version, description, source, dir_str, key in _discover_all_plugins():
|
||||
# Both the path-derived key (nested category plugins) and the bare
|
||||
# manifest name count for enabled/disabled state, matching the runtime
|
||||
# loader's back-compat lookup.
|
||||
aliases = {name}
|
||||
if key:
|
||||
aliases.add(key)
|
||||
if aliases & disabled_set:
|
||||
runtime_status = "disabled"
|
||||
elif aliases & enabled_set:
|
||||
runtime_status = "enabled"
|
||||
else:
|
||||
runtime_status = "inactive"
|
||||
runtime_status = _plugin_runtime_status(aliases, enabled_set, disabled_set)
|
||||
|
||||
dir_path = Path(dir_str)
|
||||
dm = dash_by_name.get(name)
|
||||
@@ -18926,6 +18934,14 @@ def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
removed_reason = None
|
||||
try:
|
||||
removed = find_removed(name)
|
||||
if removed is not None:
|
||||
removed_reason = removed.reason or "removed from the plugin catalog"
|
||||
except Exception:
|
||||
removed_reason = None
|
||||
|
||||
rows.append({
|
||||
"name": name,
|
||||
"version": version or "",
|
||||
@@ -18940,6 +18956,7 @@ def _merged_plugins_hub() -> Dict[str, Any]:
|
||||
"auth_required": auth_required,
|
||||
"auth_command": auth_command,
|
||||
"user_hidden": name in hidden_plugins,
|
||||
"removed_reason": removed_reason,
|
||||
})
|
||||
|
||||
agent_names = {r["name"] for r in rows}
|
||||
@@ -18981,15 +18998,123 @@ async def get_plugins_hub(request: Request):
|
||||
raise HTTPException(status_code=500, detail="Failed to build plugins hub.") from exc
|
||||
|
||||
|
||||
def _plugins_catalog_payload() -> Dict[str, Any]:
|
||||
"""Catalog entries merged with installed-state for the dashboard.
|
||||
|
||||
Each entry carries the static catalog metadata plus:
|
||||
|
||||
* ``installed`` — a plugin with the same name is discoverable locally.
|
||||
* ``installed_sha`` — pinned SHA recorded in the plugin's
|
||||
``.hermes-catalog.json`` sidecar at install time (``None`` when the
|
||||
sidecar is absent, e.g. a pre-catalog raw-git install).
|
||||
* ``update_available`` — sidecar SHA differs from the catalog pin.
|
||||
* ``runtime_status`` — enabled/disabled/inactive for installed entries,
|
||||
``None`` otherwise.
|
||||
"""
|
||||
from hermes_cli.plugin_catalog import (
|
||||
entry_capability_summary,
|
||||
load_catalog,
|
||||
load_removed_list,
|
||||
)
|
||||
from hermes_cli.plugins_cmd import (
|
||||
_discover_all_plugins,
|
||||
_get_disabled_set,
|
||||
_get_enabled_set,
|
||||
read_catalog_sidecar,
|
||||
)
|
||||
|
||||
disabled_set = _get_disabled_set()
|
||||
enabled_set = _get_enabled_set()
|
||||
|
||||
installed: Dict[str, Dict[str, Any]] = {}
|
||||
for name, _version, _description, _source, dir_str, key in _discover_all_plugins():
|
||||
aliases = {name}
|
||||
if key:
|
||||
aliases.add(key)
|
||||
info = {
|
||||
"dir": dir_str,
|
||||
"runtime_status": _plugin_runtime_status(aliases, enabled_set, disabled_set),
|
||||
}
|
||||
for alias in aliases:
|
||||
installed[alias] = info
|
||||
|
||||
entries: List[Dict[str, Any]] = []
|
||||
for entry in load_catalog():
|
||||
caps = entry.capabilities
|
||||
local = installed.get(entry.name)
|
||||
installed_sha = None
|
||||
if local is not None:
|
||||
sidecar = read_catalog_sidecar(Path(local["dir"]))
|
||||
if sidecar:
|
||||
raw_sha = sidecar.get("sha")
|
||||
installed_sha = str(raw_sha) if raw_sha else None
|
||||
entries.append({
|
||||
"name": entry.name,
|
||||
"description": entry.description,
|
||||
"repo": entry.repo,
|
||||
"sha": entry.sha,
|
||||
"sha_short": entry.sha[:7],
|
||||
"tier": entry.tier,
|
||||
"maintainer": entry.maintainer,
|
||||
"requires_hermes": entry.requires_hermes,
|
||||
"platforms": entry.platforms,
|
||||
"capabilities": {
|
||||
"provides_tools": caps.provides_tools,
|
||||
"provides_hooks": caps.provides_hooks,
|
||||
"provides_middleware": caps.provides_middleware,
|
||||
"requires_env": caps.requires_env,
|
||||
},
|
||||
"docs_url": entry.docs_url,
|
||||
"capability_summary": entry_capability_summary(entry),
|
||||
"installed": local is not None,
|
||||
"installed_sha": installed_sha,
|
||||
"update_available": bool(installed_sha) and installed_sha != entry.sha,
|
||||
"runtime_status": local["runtime_status"] if local is not None else None,
|
||||
})
|
||||
|
||||
removed = [
|
||||
{"name": r.name, "repo": r.repo, "reason": r.reason, "date": r.date}
|
||||
for r in load_removed_list()
|
||||
]
|
||||
|
||||
return {
|
||||
"entries": entries,
|
||||
"removed": removed,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/dashboard/plugins/catalog")
|
||||
async def get_plugins_catalog(request: Request):
|
||||
"""Curated plugin catalog merged with installed-state (session protected)."""
|
||||
_require_token(request)
|
||||
try:
|
||||
return _plugins_catalog_payload()
|
||||
except Exception as exc:
|
||||
_log.warning("plugins/catalog failed: %s", exc)
|
||||
raise HTTPException(
|
||||
status_code=500, detail="Failed to build plugins catalog."
|
||||
) from exc
|
||||
|
||||
|
||||
@app.post("/api/dashboard/agent-plugins/install")
|
||||
async def post_agent_plugin_install(request: Request, body: _AgentPluginInstallBody):
|
||||
_require_token(request)
|
||||
from hermes_cli.plugins_cmd import dashboard_install_plugin
|
||||
|
||||
catalog_name = (body.catalog_name or "").strip()
|
||||
identifier = body.identifier.strip()
|
||||
if not identifier and not catalog_name:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Provide an identifier or a catalog_name.",
|
||||
)
|
||||
|
||||
result = dashboard_install_plugin(
|
||||
body.identifier.strip(),
|
||||
identifier,
|
||||
force=body.force,
|
||||
enable=body.enable,
|
||||
catalog_name=catalog_name or None,
|
||||
)
|
||||
if not result.get("ok"):
|
||||
raise HTTPException(
|
||||
|
||||
+665
-11
@@ -17,6 +17,7 @@ Key design decisions:
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sqlite3
|
||||
@@ -582,6 +583,13 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
"""
|
||||
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
||||
try:
|
||||
# Best-effort tokenizer load: a DB carrying the messages_fts_cjk
|
||||
# index needs the cjk_unicode61 extension before any statement can
|
||||
# touch that table — including the trigger-driven write probe below.
|
||||
# Without it, this probe sees the DB exactly as a tokenizer-less
|
||||
# SessionDB open would (which drops the cjk triggers to keep writes
|
||||
# working), so tokenizer absence must never classify as corruption.
|
||||
load_fts5_cjk_extension(conn)
|
||||
conn.execute("PRAGMA journal_mode").fetchone()
|
||||
rows = conn.execute("PRAGMA integrity_check").fetchall()
|
||||
problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"]
|
||||
@@ -603,7 +611,7 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
# Catch the full sqlite3 exception hierarchy (not just
|
||||
# OperationalError) so the malformed-shadow-table class is reported
|
||||
# rather than letting it crash the caller.
|
||||
for fts_table in ("messages_fts", "messages_fts_trigram"):
|
||||
for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"):
|
||||
try:
|
||||
# No-op queries against the actual FTS5 APIs the search
|
||||
# tools use. The trigram table is included because it backs
|
||||
@@ -674,6 +682,12 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
msg = str(exc).lower()
|
||||
if "no such table" in msg or "no such column" in msg:
|
||||
return None
|
||||
if "no such tokenizer: cjk_unicode61" in msg:
|
||||
# This probe process couldn't load the cjk extension while
|
||||
# the DB carries the cjk index — capability gap, not
|
||||
# corruption. A tokenizer-capable SessionDB serves it fine;
|
||||
# a tokenizer-less one self-heals by dropping the triggers.
|
||||
return None
|
||||
return str(exc)
|
||||
return None
|
||||
except sqlite3.DatabaseError as exc:
|
||||
@@ -737,13 +751,19 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
||||
try:
|
||||
for table_name in ("messages_fts", "messages_fts_trigram"):
|
||||
# The cjk index can only be rebuilt with its tokenizer loaded;
|
||||
# best-effort (a tokenizer-less host skips it at the probe below).
|
||||
load_fts5_cjk_extension(conn)
|
||||
for table_name in (
|
||||
"messages_fts", "messages_fts_trigram", "messages_fts_cjk"
|
||||
):
|
||||
try:
|
||||
conn.execute(
|
||||
f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')"
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
# Table absent (FTS disabled / trigram off) — skip it.
|
||||
# Table absent (FTS disabled / trigram off / cjk not
|
||||
# present or tokenizer unavailable) — skip it.
|
||||
continue
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -1149,6 +1169,151 @@ BEGIN
|
||||
END;
|
||||
"""
|
||||
|
||||
# ── CJK-bigram FTS index (replaces the trigram index when available) ────
|
||||
#
|
||||
# The trigram tokenizer needs >=3 chars per query term, so 1-2 char CJK
|
||||
# terms (ubiquitous in Korean/Chinese: 일본, 구글, 项目, ...) fall through
|
||||
# to a LIKE full-table scan — measured 3-6s CPU per query on multi-GB
|
||||
# installs and the dominant base cost of session_search on CJK workloads.
|
||||
#
|
||||
# ``cjk_unicode61`` (native/fts5_cjk/, a ~250-line loadable FTS5 tokenizer
|
||||
# with no dependencies) wraps unicode61: maximal CJK runs are re-emitted as
|
||||
# overlapping character bigrams (Lucene CJKAnalyzer semantics), everything
|
||||
# else passes through unchanged. FTS5 phrase semantics turn a query term's
|
||||
# consecutive bigrams into exact substring matching down to 2 chars at
|
||||
# index speed. Contributed by Soju06 (PR #65544).
|
||||
#
|
||||
# Same v23 storage discipline as the trigram table it replaces:
|
||||
# external-content over a tool-row-excluding view (zero inline text
|
||||
# copies; tool rows stay searchable via ``messages_fts``), triggers gated
|
||||
# on a DEDICATED marker pair (``fts_cjk_rebuild_high_water`` /
|
||||
# ``fts_cjk_rebuild_progress``) so a cjk-only backfill — e.g. the
|
||||
# trigram→cjk upgrade on an already-optimized DB — never gates the
|
||||
# complete ``messages_fts`` index's triggers.
|
||||
#
|
||||
# The table exists ONLY when the loadable tokenizer is available
|
||||
# (``~/.hermes/lib/libfts5_cjk.so``, built by ``native/fts5_cjk/build.sh``).
|
||||
# A process that cannot load it self-heals by dropping the cjk triggers
|
||||
# (message writes keep working; the index goes stale and is rebuilt by the
|
||||
# next ``hermes sessions optimize-storage`` on a capable host).
|
||||
#
|
||||
# Split DDL: the table/view part is safe to ensure any time; the triggers
|
||||
# are created ONLY while the index is complete-or-marker-gated. A stale
|
||||
# index (trigger gap of unknown extent) must keep its triggers DROPPED —
|
||||
# an external-content 'delete' op for a rowid the index never held is the
|
||||
# canonical FTS5 index-corruption hazard the v23 marker gating exists to
|
||||
# prevent.
|
||||
FTS_CJK_TABLE_SQL = """
|
||||
CREATE VIEW IF NOT EXISTS messages_fts_cjk_src AS
|
||||
SELECT id, role, content, tool_name, tool_calls
|
||||
FROM messages
|
||||
WHERE role <> 'tool';
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_cjk USING fts5(
|
||||
content,
|
||||
tool_name,
|
||||
tool_calls,
|
||||
content='messages_fts_cjk_src',
|
||||
content_rowid='id',
|
||||
tokenize='cjk_unicode61'
|
||||
);
|
||||
"""
|
||||
|
||||
FTS_CJK_TRIGGER_SQL = """
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_insert AFTER INSERT ON messages
|
||||
WHEN new.role <> 'tool'
|
||||
AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
|
||||
VALUES (new.id, new.content, new.tool_name, new.tool_calls);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_delete AFTER DELETE ON messages
|
||||
WHEN old.role <> 'tool'
|
||||
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
|
||||
VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_update AFTER UPDATE ON messages
|
||||
WHEN (old.content IS NOT new.content
|
||||
OR old.tool_name IS NOT new.tool_name
|
||||
OR old.tool_calls IS NOT new.tool_calls
|
||||
OR old.role IS NOT new.role)
|
||||
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
|
||||
SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls
|
||||
WHERE old.role <> 'tool';
|
||||
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
|
||||
SELECT new.id, new.content, new.tool_name, new.tool_calls
|
||||
WHERE new.role <> 'tool';
|
||||
END;
|
||||
"""
|
||||
|
||||
_FTS_CJK_TRIGGERS = (
|
||||
"messages_fts_cjk_insert",
|
||||
"messages_fts_cjk_delete",
|
||||
"messages_fts_cjk_update",
|
||||
)
|
||||
|
||||
# state_meta breadcrumb set when a tokenizer-less process had to drop the
|
||||
# cjk triggers to keep message writes alive: rows written from that moment
|
||||
# on are missing from the cjk index, so it must not serve reads until
|
||||
# `hermes sessions optimize-storage` rebuilds it on a capable host.
|
||||
FTS_CJK_STALE_KEY = "fts_cjk_stale"
|
||||
|
||||
|
||||
def fts5_cjk_so_path() -> Path:
|
||||
"""Location of the cjk_unicode61 loadable extension."""
|
||||
env = os.getenv("HERMES_FTS5_CJK_SO")
|
||||
if env:
|
||||
return Path(env).expanduser()
|
||||
return get_hermes_home() / "lib" / "libfts5_cjk.so"
|
||||
|
||||
|
||||
def _cjk_fts_config_enabled() -> bool:
|
||||
"""config.yaml ``sessions.cjk_fts`` (default on), via its env bridge."""
|
||||
return os.getenv("HERMES_CJK_FTS", "1").strip().lower() not in (
|
||||
"0", "false", "off", "no",
|
||||
)
|
||||
|
||||
|
||||
def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:
|
||||
"""Best-effort load of the cjk_unicode61 tokenizer into ``conn``.
|
||||
|
||||
Returns False (never raises) when the .so is absent, the feature is
|
||||
disabled via ``sessions.cjk_fts``, or this Python build has extension
|
||||
loading compiled out — every caller treats False as "behave exactly as
|
||||
before the cjk index existed".
|
||||
"""
|
||||
if not _cjk_fts_config_enabled():
|
||||
return False
|
||||
path = fts5_cjk_so_path()
|
||||
if not path.exists():
|
||||
return False
|
||||
try:
|
||||
conn.enable_load_extension(True)
|
||||
try:
|
||||
conn.load_extension(str(path))
|
||||
finally:
|
||||
conn.enable_load_extension(False)
|
||||
return True
|
||||
except Exception:
|
||||
logger.warning("fts5_cjk extension load failed (%s)", path, exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
# ── Legacy (v22 / inline-content) FTS DDL ──────────────────────────────
|
||||
# Used ONLY to keep an existing pre-v23 install's search working and its
|
||||
@@ -1269,6 +1434,12 @@ class SessionDB:
|
||||
self._fts_runtime_rebuild_attempted = False
|
||||
self._fts_enabled = False
|
||||
self._trigram_available = False
|
||||
# CJK-bigram index (cjk_unicode61 loadable tokenizer). _fts_cjk_loaded:
|
||||
# extension present on the writer connection; _fts_cjk_available: the
|
||||
# messages_fts_cjk table is queryable AND not marked stale. Set during
|
||||
# _init_schema / _probe_fts_cjk.
|
||||
self._fts_cjk_loaded = False
|
||||
self._fts_cjk_available = False
|
||||
self._fts_unavailable_warned = False
|
||||
self._conn = None
|
||||
try:
|
||||
@@ -1309,6 +1480,7 @@ class SessionDB:
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
apply_wal_with_fallback(self._conn, db_label="state.db")
|
||||
self._conn.execute("PRAGMA foreign_keys=ON")
|
||||
self._fts_cjk_loaded = load_fts5_cjk_extension(self._conn)
|
||||
self._init_schema()
|
||||
|
||||
try:
|
||||
@@ -1372,12 +1544,25 @@ class SessionDB:
|
||||
# Scope to trigram specifically to avoid masking unrelated tokenizer errors.
|
||||
if "no such tokenizer: trigram" in err:
|
||||
return True
|
||||
# The cjk_unicode61 tokenizer is a loadable extension — a process
|
||||
# that couldn't load it sees the same capability-error shape.
|
||||
if "no such tokenizer: cjk_unicode61" in err:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _is_trigram_unavailable_error(exc: sqlite3.OperationalError) -> bool:
|
||||
"""True when only the trigram tokenizer is missing (FTS5 itself works)."""
|
||||
return "no such tokenizer: trigram" in str(exc).lower()
|
||||
"""True when only an optional tokenizer is missing (FTS5 itself works).
|
||||
|
||||
Covers the built-in trigram tokenizer (needs SQLite >= 3.34) and the
|
||||
loadable cjk_unicode61 tokenizer — both mean "this one index can't be
|
||||
served here", never "disable FTS".
|
||||
"""
|
||||
err = str(exc).lower()
|
||||
return (
|
||||
"no such tokenizer: trigram" in err
|
||||
or "no such tokenizer: cjk_unicode61" in err
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _db_has_legacy_inline_fts(cursor: sqlite3.Cursor) -> bool:
|
||||
@@ -1444,6 +1629,124 @@ class SessionDB:
|
||||
self._warn_fts5_unavailable(exc)
|
||||
return False
|
||||
|
||||
def _ensure_fts_cjk_schema(self, cursor) -> None:
|
||||
"""Create / repair / self-heal the CJK-bigram index surface.
|
||||
|
||||
``cursor`` may be a Cursor or a Connection (both expose execute /
|
||||
executescript). Called only for v23-shape DBs with the base FTS
|
||||
surface healthy. Sets ``self._fts_cjk_available``. Never raises;
|
||||
every failure mode degrades to "no cjk index" (trigram/LIKE routing
|
||||
keeps working).
|
||||
|
||||
Cases:
|
||||
tokenizer loaded, table absent → create. Empty DB: index is
|
||||
complete by construction (triggers cover everything). Populated
|
||||
DB: set the cjk backfill markers so the id-gated triggers stay
|
||||
correct and `optimize-storage` can backfill; the index is NOT
|
||||
served until the backfill completes.
|
||||
tokenizer loaded, table present → ensure triggers (recreates any
|
||||
dropped by a tokenizer-less process), honour the stale
|
||||
breadcrumb (serve only when absent and no backfill pending).
|
||||
tokenizer NOT loaded, table present with live triggers → drop the
|
||||
cjk triggers so message INSERTs don't fail at trigger time,
|
||||
and leave the stale breadcrumb (#self-heal). The table itself
|
||||
stays for a later capable open to rebuild.
|
||||
"""
|
||||
cjk_present = bool(cursor.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name = 'messages_fts_cjk'"
|
||||
).fetchone())
|
||||
|
||||
if not self._fts_cjk_loaded:
|
||||
if cjk_present:
|
||||
live = [
|
||||
r[0] for r in cursor.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'trigger' "
|
||||
f"AND name IN ({','.join('?' for _ in _FTS_CJK_TRIGGERS)})",
|
||||
_FTS_CJK_TRIGGERS,
|
||||
).fetchall()
|
||||
]
|
||||
if live:
|
||||
# Self-heal: this process cannot tokenize, so every
|
||||
# message INSERT would die inside the cjk trigger.
|
||||
# Breadcrumb FIRST (crash between the two statements is
|
||||
# merely conservative), then drop.
|
||||
logger.warning(
|
||||
"messages_fts_cjk triggers present but the "
|
||||
"cjk_unicode61 tokenizer is unavailable (%s) — "
|
||||
"dropping the cjk triggers so message writes keep "
|
||||
"working. CJK search falls back to trigram/LIKE; "
|
||||
"run `hermes sessions optimize-storage` on a host "
|
||||
"with the extension to rebuild.",
|
||||
fts5_cjk_so_path(),
|
||||
)
|
||||
cursor.execute(
|
||||
"INSERT INTO state_meta (key, value) VALUES (?, '1') "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = '1'",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
)
|
||||
for trig in live:
|
||||
cursor.execute(f"DROP TRIGGER IF EXISTS {trig}")
|
||||
self._fts_cjk_available = False
|
||||
return
|
||||
|
||||
try:
|
||||
cursor.executescript(FTS_CJK_TABLE_SQL)
|
||||
if not cjk_present:
|
||||
# Freshly created. An empty DB's index is complete by
|
||||
# construction (triggers will cover every future row); a
|
||||
# populated DB (e.g. a v23 install predating the cjk index)
|
||||
# gets the dedicated marker pair so the id-gated triggers
|
||||
# keep NEW rows indexed while old rows await the
|
||||
# `optimize-storage` backfill. Either way any old stale
|
||||
# breadcrumb refers to a table that no longer exists.
|
||||
cursor.execute(
|
||||
"DELETE FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
)
|
||||
n_msgs = cursor.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
if n_msgs > 0:
|
||||
hw = cursor.execute(
|
||||
"SELECT COALESCE(MAX(id), 0) FROM messages"
|
||||
).fetchone()[0]
|
||||
for k, v in (
|
||||
("fts_cjk_rebuild_high_water", str(hw)),
|
||||
("fts_cjk_rebuild_progress", "0"),
|
||||
):
|
||||
cursor.execute(
|
||||
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(k, v),
|
||||
)
|
||||
stale = cursor.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
).fetchone()
|
||||
if stale:
|
||||
# A tokenizer-less process dropped the triggers at some
|
||||
# unknown point — the index has a gap of unknown extent.
|
||||
# Do NOT reinstall triggers (an external-content 'delete'
|
||||
# for an unindexed rowid corrupts the index); the next
|
||||
# `optimize-storage` run rebuilds from scratch.
|
||||
self._fts_cjk_available = False
|
||||
return
|
||||
cursor.executescript(FTS_CJK_TRIGGER_SQL)
|
||||
backfill_pending = cursor.execute(
|
||||
"SELECT 1 FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_high_water' LIMIT 1"
|
||||
).fetchone()
|
||||
self._fts_cjk_available = not backfill_pending
|
||||
except sqlite3.OperationalError:
|
||||
# Includes "no such tokenizer: cjk_unicode61" if the extension
|
||||
# loaded but registration failed — degrade to trigram/LIKE.
|
||||
logger.warning(
|
||||
"messages_fts_cjk ensure failed; CJK search stays on "
|
||||
"trigram/LIKE", exc_info=True,
|
||||
)
|
||||
self._fts_cjk_available = False
|
||||
|
||||
@staticmethod
|
||||
def _drop_fts_triggers(cursor: sqlite3.Cursor) -> None:
|
||||
for trigger in _FTS_TRIGGERS:
|
||||
@@ -1968,6 +2271,136 @@ class SessionDB:
|
||||
return False
|
||||
return bool(more)
|
||||
|
||||
# ── CJK-bigram index backfill (dedicated marker pair) ──
|
||||
#
|
||||
# Same chunk engine as the main deferred rebuild, but on the
|
||||
# ``fts_cjk_rebuild_*`` markers so a cjk-only backfill (the common case:
|
||||
# an already-optimized v23 DB gaining the cjk index) never gates the
|
||||
# complete ``messages_fts`` / trigram triggers.
|
||||
|
||||
def fts_cjk_rebuild_status(self) -> Optional[Dict[str, Any]]:
|
||||
"""CJK-index backfill progress, or None when none is pending."""
|
||||
high_water = self.get_meta("fts_cjk_rebuild_high_water")
|
||||
if high_water is None:
|
||||
return None
|
||||
progress = int(self.get_meta("fts_cjk_rebuild_progress") or 0)
|
||||
total = int(high_water)
|
||||
if total <= 0:
|
||||
return None
|
||||
pct = min(100, int(100 * progress / total))
|
||||
return {"pending": True, "total": total, "indexed": progress, "percent": pct}
|
||||
|
||||
def fts_cjk_rebuild_step(self) -> bool:
|
||||
"""Backfill one chunk of the CJK index. True while work remains."""
|
||||
if not self._fts_enabled or not self._fts_cjk_loaded:
|
||||
return False
|
||||
high_water_raw = self.get_meta("fts_cjk_rebuild_high_water")
|
||||
if high_water_raw is None:
|
||||
return False
|
||||
high_water = int(high_water_raw)
|
||||
chunk = self._FTS_REBUILD_CHUNK_ROWS
|
||||
|
||||
def _do(conn):
|
||||
row = conn.execute(
|
||||
"SELECT value FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_progress'"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False # finished (or cleared) by another process
|
||||
progress = int(row[0])
|
||||
if progress >= high_water:
|
||||
return False
|
||||
upper = min(progress + chunk, high_water)
|
||||
conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) "
|
||||
"SELECT id, content, tool_name, tool_calls FROM messages "
|
||||
"WHERE id > ? AND id <= ? AND role <> 'tool'",
|
||||
(progress, upper),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE state_meta SET value = ? "
|
||||
"WHERE key = 'fts_cjk_rebuild_progress'",
|
||||
(str(upper),),
|
||||
)
|
||||
return upper < high_water
|
||||
|
||||
try:
|
||||
more = self._execute_write(_do)
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.debug("CJK FTS rebuild chunk failed (will retry): %s", exc)
|
||||
return True
|
||||
if more is False:
|
||||
status = self.fts_cjk_rebuild_status()
|
||||
if status is not None and status["indexed"] >= status["total"]:
|
||||
self._fts_cjk_rebuild_finish()
|
||||
return False
|
||||
return bool(more)
|
||||
|
||||
def _fts_cjk_rebuild_finish(self) -> None:
|
||||
"""Boundary sweep + clear the cjk markers; index becomes servable."""
|
||||
def _do(conn):
|
||||
hw_row = conn.execute(
|
||||
"SELECT value FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_high_water'"
|
||||
).fetchone()
|
||||
if hw_row is not None:
|
||||
hw = int(hw_row[0])
|
||||
lo, hi = hw - 1000, hw + 1000
|
||||
conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) "
|
||||
"SELECT m.id, m.content, m.tool_name, m.tool_calls "
|
||||
"FROM messages m "
|
||||
"WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' "
|
||||
"AND NOT EXISTS (SELECT 1 FROM messages_fts_cjk_docsize d WHERE d.id = m.id)",
|
||||
(lo, hi),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM state_meta WHERE key IN "
|
||||
"('fts_cjk_rebuild_high_water', 'fts_cjk_rebuild_progress')"
|
||||
)
|
||||
self._execute_write(_do)
|
||||
self._fts_cjk_available = True
|
||||
logger.info("CJK FTS index backfill complete — serving CJK search.")
|
||||
|
||||
def _fts_cjk_reset_if_stale(self) -> None:
|
||||
"""Rebuild path for a stale cjk index (triggers were dropped).
|
||||
|
||||
The gap's extent is unknown, so the only safe recovery is a from-
|
||||
scratch rebuild: drop the table + triggers, clear the breadcrumb,
|
||||
recreate via ``_ensure_fts_cjk_schema`` (which sets fresh backfill
|
||||
markers on a populated DB). Called from ``optimize_fts_storage`` on
|
||||
a tokenizer-capable host; no-op when not stale.
|
||||
"""
|
||||
if not self._fts_cjk_loaded:
|
||||
return
|
||||
|
||||
def _do(conn):
|
||||
stale = conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
).fetchone()
|
||||
if not stale:
|
||||
return False
|
||||
for trig in _FTS_CJK_TRIGGERS:
|
||||
conn.execute(f"DROP TRIGGER IF EXISTS {trig}")
|
||||
conn.execute("DROP TABLE IF EXISTS messages_fts_cjk")
|
||||
conn.execute("DROP VIEW IF EXISTS messages_fts_cjk_src")
|
||||
conn.execute(
|
||||
"DELETE FROM state_meta WHERE key IN "
|
||||
f"('{FTS_CJK_STALE_KEY}', 'fts_cjk_rebuild_high_water', "
|
||||
"'fts_cjk_rebuild_progress')"
|
||||
)
|
||||
return True
|
||||
was_stale = self._execute_write(_do)
|
||||
if was_stale:
|
||||
# Recreate outside the write transaction — _ensure_fts_cjk_schema
|
||||
# uses executescript(), which implicitly commits any pending
|
||||
# transaction and must not run inside _execute_write's BEGIN
|
||||
# IMMEDIATE. Sets fresh backfill markers on a populated DB.
|
||||
with self._lock:
|
||||
self._ensure_fts_cjk_schema(self._conn)
|
||||
self._conn.commit()
|
||||
|
||||
# ── Opt-in v23 FTS storage optimization (`hermes sessions optimize-storage`) ──
|
||||
#
|
||||
# This is the ONLY path that migrates an existing legacy (v22 inline) DB
|
||||
@@ -1983,8 +2416,10 @@ class SessionDB:
|
||||
is a legacy inline-FTS install that can be optimized to the v23
|
||||
external-content schema, or a previous optimize run was interrupted
|
||||
(legacy vtables already demoted, but backfill markers and/or trash
|
||||
tables remain) and re-running would resume it. False for fresh and
|
||||
fully-optimized installs (and when FTS5 is unavailable)."""
|
||||
tables remain) and re-running would resume it, or the CJK-bigram
|
||||
index needs a backfill/rebuild on this tokenizer-capable host.
|
||||
False for fresh and fully-optimized installs (and when FTS5 is
|
||||
unavailable)."""
|
||||
if not self._fts_enabled or self.read_only:
|
||||
return False
|
||||
with self._lock:
|
||||
@@ -2000,6 +2435,14 @@ class SessionDB:
|
||||
"WHERE key = 'fts_rebuild_high_water' LIMIT 1"
|
||||
).fetchone():
|
||||
return True
|
||||
# CJK-bigram index work — only offerable when THIS process can
|
||||
# tokenize: a pending backfill (markers set at creation on a
|
||||
# populated DB) or a stale index awaiting a from-scratch rebuild.
|
||||
if self._fts_cjk_loaded and self._conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key IN "
|
||||
f"('fts_cjk_rebuild_high_water', '{FTS_CJK_STALE_KEY}') LIMIT 1"
|
||||
).fetchone():
|
||||
return True
|
||||
return self._has_fts_trash(self._conn)
|
||||
|
||||
def _has_fts_trash(self, conn) -> bool:
|
||||
@@ -2089,10 +2532,24 @@ class SessionDB:
|
||||
if legacy and not pending:
|
||||
self._demote_legacy_fts_to_trash()
|
||||
|
||||
# A stale CJK index (triggers dropped by a tokenizer-less process)
|
||||
# can only be recovered from scratch — reset it now so the cjk
|
||||
# backfill phase below rebuilds it. No-op without the tokenizer.
|
||||
self._fts_cjk_reset_if_stale()
|
||||
# An optimized v23 DB gaining the cjk index for the first time (no
|
||||
# legacy work left, tokenizer newly installed): ensure the table +
|
||||
# markers exist so the backfill phase has work to claim.
|
||||
if self._fts_cjk_loaded:
|
||||
with self._lock:
|
||||
self._ensure_fts_cjk_schema(self._conn)
|
||||
self._conn.commit()
|
||||
|
||||
def _emit(phase: str) -> None:
|
||||
if progress_cb is None:
|
||||
return
|
||||
st = self.fts_rebuild_status()
|
||||
if st is None:
|
||||
st = self.fts_cjk_rebuild_status()
|
||||
progress_cb({
|
||||
"phase": phase,
|
||||
"percent": st["percent"] if st else 100,
|
||||
@@ -2125,6 +2582,15 @@ class SessionDB:
|
||||
_pause(time.monotonic() - _t0)
|
||||
_emit("backfill")
|
||||
|
||||
# Phase 1b: backfill the CJK-bigram index (its own marker pair; a
|
||||
# no-op when the tokenizer isn't loadable or nothing is pending).
|
||||
while True:
|
||||
_t0 = time.monotonic()
|
||||
if not self.fts_cjk_rebuild_step():
|
||||
break
|
||||
_emit("backfill")
|
||||
_pause(time.monotonic() - _t0)
|
||||
|
||||
# Phase 2: tear down the demoted legacy shadow tables in chunks.
|
||||
_emit("teardown")
|
||||
while True:
|
||||
@@ -2687,6 +3153,9 @@ class SessionDB:
|
||||
cursor,
|
||||
include_trigram=trigram_enabled,
|
||||
)
|
||||
# CJK-bigram index (cjk_unicode61). Strictly additive to
|
||||
# the surfaces above and gated on the loadable tokenizer:
|
||||
self._ensure_fts_cjk_schema(cursor)
|
||||
|
||||
self._conn.commit()
|
||||
|
||||
@@ -6290,6 +6759,24 @@ class SessionDB:
|
||||
"""Count CJK characters in text."""
|
||||
return sum(1 for ch in text if cls._is_cjk_codepoint(ord(ch)))
|
||||
|
||||
@classmethod
|
||||
def _has_lone_cjk_run(cls, query: str) -> bool:
|
||||
"""True when any maximal CJK run in the query is a single char.
|
||||
|
||||
The cjk-bigram index stores bigrams for runs >=2 chars and unigrams
|
||||
only for isolated chars, so a 1-char CJK term can't match inside
|
||||
longer runs there — those queries keep the LIKE substring route.
|
||||
"""
|
||||
run = 0
|
||||
for ch in query:
|
||||
if cls._is_cjk_codepoint(ord(ch)):
|
||||
run += 1
|
||||
else:
|
||||
if run == 1:
|
||||
return True
|
||||
run = 0
|
||||
return run == 1
|
||||
|
||||
def search_messages(
|
||||
self,
|
||||
query: str,
|
||||
@@ -6300,6 +6787,76 @@ class SessionDB:
|
||||
offset: int = 0,
|
||||
sort: str = None,
|
||||
include_inactive: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Instrumented wrapper around :meth:`_search_messages_impl`.
|
||||
|
||||
Logs one line per slow search with the routing path taken, so
|
||||
production latency stays attributable per query shape (the 2026-07
|
||||
session_search investigation needed trace archaeology to discover
|
||||
the LIKE full scans; this makes the next regression a grep).
|
||||
Threshold: HERMES_SEARCH_SLOW_MS (default 1000; 0 logs every call).
|
||||
"""
|
||||
started = time.time()
|
||||
rows = None
|
||||
try:
|
||||
rows = self._search_messages_impl(
|
||||
query,
|
||||
source_filter=source_filter,
|
||||
exclude_sources=exclude_sources,
|
||||
role_filter=role_filter,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
sort=sort,
|
||||
include_inactive=include_inactive,
|
||||
)
|
||||
return rows
|
||||
finally:
|
||||
try:
|
||||
threshold = float(os.getenv("HERMES_SEARCH_SLOW_MS", "1000"))
|
||||
except (TypeError, ValueError):
|
||||
threshold = 1000.0
|
||||
elapsed_ms = (time.time() - started) * 1000.0
|
||||
if elapsed_ms >= threshold:
|
||||
logger.info(
|
||||
"slow session search: path=%s elapsed=%.0fms rows=%s query=%r",
|
||||
self._describe_search_path(query),
|
||||
elapsed_ms,
|
||||
len(rows) if rows is not None else "err",
|
||||
query[:200],
|
||||
)
|
||||
|
||||
def _describe_search_path(self, query: str) -> str:
|
||||
"""Best-effort name of the routing path a query takes (log-only)."""
|
||||
try:
|
||||
sanitized = self._sanitize_fts5_query(query or "")
|
||||
if not sanitized:
|
||||
return "empty"
|
||||
if not self._contains_cjk(sanitized):
|
||||
return "fts5"
|
||||
raw = sanitized.strip('"').strip()
|
||||
if self._fts_cjk_available and not self._has_lone_cjk_run(raw):
|
||||
return "fts_cjk"
|
||||
tokens = [
|
||||
t for t in raw.split()
|
||||
if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t)
|
||||
]
|
||||
short = any(self._count_cjk(t) < 3 for t in tokens)
|
||||
if self._count_cjk(raw) >= 3 and not short and self._trigram_available:
|
||||
return "trigram"
|
||||
return "like_scan"
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
def _search_messages_impl(
|
||||
self,
|
||||
query: str,
|
||||
source_filter: List[str] = None,
|
||||
exclude_sources: List[str] = None,
|
||||
role_filter: List[str] = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
sort: str = None,
|
||||
include_inactive: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Full-text search across session messages using FTS5.
|
||||
@@ -6437,8 +6994,104 @@ class SessionDB:
|
||||
# query explicitly filtering on role='tool' must therefore use
|
||||
# the LIKE fallback, which scans the base table directly.
|
||||
_wants_tool_rows = bool(role_filter) and "tool" in role_filter
|
||||
|
||||
# ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ──────
|
||||
# When the bigram index is available it serves EVERY CJK query
|
||||
# shape the legacy code split between trigram (>=3 chars/token)
|
||||
# and LIKE full scans (1-2 char tokens) — the whole point of the
|
||||
# index (PR #65544). Exceptions stay on the legacy routes:
|
||||
# - role_filter=['tool'] queries (tool rows aren't in the cjk
|
||||
# index, same exclusion as trigram),
|
||||
# - queries containing a LONE 1-char CJK run: the index stores
|
||||
# bigrams for runs >=2, so a single-char term can only match
|
||||
# isolated chars — LIKE substring semantics are broader.
|
||||
if (
|
||||
cjk_count >= 3
|
||||
self._fts_cjk_available
|
||||
and not _wants_tool_rows
|
||||
and not self._has_lone_cjk_run(raw_query)
|
||||
):
|
||||
tokens = raw_query.split()
|
||||
parts = []
|
||||
for tok in tokens:
|
||||
if tok.upper() in {"AND", "OR", "NOT"}:
|
||||
parts.append(tok)
|
||||
else:
|
||||
parts.append('"' + tok.replace('"', '""') + '"')
|
||||
cjk_query = " ".join(parts)
|
||||
cjk_where = ["messages_fts_cjk MATCH ?"]
|
||||
cjk_params: list = [cjk_query]
|
||||
if not include_inactive:
|
||||
cjk_where.append("(m.active = 1 OR m.compacted = 1)")
|
||||
if source_filter is not None:
|
||||
cjk_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})")
|
||||
cjk_params.extend(source_filter)
|
||||
if exclude_sources is not None:
|
||||
cjk_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})")
|
||||
cjk_params.extend(exclude_sources)
|
||||
if role_filter:
|
||||
cjk_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})")
|
||||
cjk_params.extend(role_filter)
|
||||
cjk_sql = f"""
|
||||
SELECT
|
||||
m.id,
|
||||
m.session_id,
|
||||
m.role,
|
||||
snippet(messages_fts_cjk, -1, '>>>', '<<<', '...', 40) AS snippet,
|
||||
m.content,
|
||||
m.timestamp,
|
||||
m.tool_name,
|
||||
s.source,
|
||||
s.model,
|
||||
s.started_at AS session_started
|
||||
FROM messages_fts_cjk
|
||||
JOIN messages m ON m.id = messages_fts_cjk.rowid
|
||||
JOIN sessions s ON s.id = m.session_id
|
||||
WHERE {' AND '.join(cjk_where)}
|
||||
{order_by_sql}
|
||||
LIMIT ? OFFSET ?
|
||||
"""
|
||||
cjk_params.extend([limit, offset])
|
||||
try:
|
||||
with self._lock:
|
||||
cjk_cursor = self._conn.execute(cjk_sql, cjk_params)
|
||||
matches = [dict(row) for row in cjk_cursor.fetchall()]
|
||||
_trigram_succeeded = True
|
||||
except sqlite3.OperationalError:
|
||||
# Tokenizer missing on this connection / query syntax —
|
||||
# the trigram + LIKE routes below still answer.
|
||||
logger.debug(
|
||||
"messages_fts_cjk query failed; falling back to "
|
||||
"trigram/LIKE", exc_info=True,
|
||||
)
|
||||
except sqlite3.DatabaseError as exc:
|
||||
# Same corruption class as the other FTS reads: rebuild
|
||||
# in place once and retry; on refusal/failure fall back.
|
||||
if self._try_runtime_fts_rebuild(exc):
|
||||
try:
|
||||
with self._lock:
|
||||
cjk_cursor = self._conn.execute(
|
||||
cjk_sql, cjk_params
|
||||
)
|
||||
matches = [
|
||||
dict(row) for row in cjk_cursor.fetchall()
|
||||
]
|
||||
_trigram_succeeded = True
|
||||
except sqlite3.DatabaseError:
|
||||
logger.warning(
|
||||
"CJK-bigram FTS search still failing after "
|
||||
"in-place rebuild; falling back to "
|
||||
"trigram/LIKE."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"CJK-bigram FTS search hit a corruption error "
|
||||
"(%s) and no in-place rebuild was possible; "
|
||||
"falling back to trigram/LIKE.", exc,
|
||||
)
|
||||
|
||||
if (
|
||||
not _trigram_succeeded
|
||||
and cjk_count >= 3
|
||||
and not _any_short_cjk
|
||||
and self._trigram_available
|
||||
and not _wants_tool_rows
|
||||
@@ -8534,9 +9187,10 @@ class SessionDB:
|
||||
# ── Space reclamation ──
|
||||
|
||||
# FTS5 virtual tables whose b-tree segments we merge on optimize. The
|
||||
# trigram table is created lazily / may be disabled, so we probe before
|
||||
# touching it (see optimize_fts).
|
||||
_FTS_TABLES = ("messages_fts", "messages_fts_trigram")
|
||||
# trigram table is created lazily / may be disabled, and the cjk-bigram
|
||||
# table only exists (and is only queryable) when the loadable tokenizer
|
||||
# is present — so we probe each before touching it (see optimize_fts).
|
||||
_FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
|
||||
|
||||
def _fts_table_exists(self, name: str) -> bool:
|
||||
"""True if an FTS5 virtual table is queryable in this DB."""
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# fts5_cjk — cjk_unicode61 FTS5 tokenizer
|
||||
|
||||
unicode61 + CJK character bigrams (Lucene CJKAnalyzer semantics). Fixes
|
||||
1-2 char Korean/Chinese/Japanese terms falling through to LIKE full-table
|
||||
scans in session search.
|
||||
|
||||
Build & install to `~/.hermes/lib/`:
|
||||
|
||||
./build.sh
|
||||
|
||||
Uses the system `sqlite3ext.h` when available, else the vendored copy in
|
||||
`vendor/` — no libsqlite3-dev required.
|
||||
|
||||
Once the extension is installed, the next `SessionDB` open creates the
|
||||
`messages_fts_cjk` index (external-content, tool rows excluded — same v23
|
||||
storage discipline as the other indexes). On a populated database, run
|
||||
|
||||
hermes sessions optimize-storage
|
||||
|
||||
to backfill it; new messages are indexed live either way. Set
|
||||
`sessions.cjk_fts: false` in `~/.hermes/config.yaml` to disable. Override
|
||||
the .so location with `HERMES_FTS5_CJK_SO`.
|
||||
|
||||
Contributed by Soju06 (PR #65544).
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# Build libfts5_cjk.so and install to ~/.hermes/lib/ (or $1).
|
||||
#
|
||||
# Uses the system sqlite3ext.h when present, else the vendored copy in
|
||||
# vendor/ (public-domain SQLite amalgamation headers) so the build works
|
||||
# without libsqlite3-dev installed.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
CFLAGS_EXTRA=""
|
||||
if ! echo '#include <sqlite3ext.h>' | gcc -E -xc - >/dev/null 2>&1; then
|
||||
CFLAGS_EXTRA="-Ivendor"
|
||||
fi
|
||||
|
||||
gcc -shared -fPIC -O2 -Wall -Wextra $CFLAGS_EXTRA fts5_cjk.c -o libfts5_cjk.so
|
||||
dest="${1:-$HOME/.hermes/lib}"
|
||||
mkdir -p "$dest"
|
||||
install -m 0644 libfts5_cjk.so "$dest/libfts5_cjk.so"
|
||||
echo "installed: $dest/libfts5_cjk.so"
|
||||
@@ -0,0 +1,252 @@
|
||||
/*
|
||||
** fts5_cjk.c — "cjk_unicode61" FTS5 tokenizer: unicode61 + CJK bigrams.
|
||||
**
|
||||
** Why: SQLite's unicode61 tokenizer treats a CJK run as ONE token
|
||||
** ("웅기가말했다" indexes as a single 6-char token), so a 2-char Korean
|
||||
** query can never match inside it. The stock trigram tokenizer fixes
|
||||
** substring search but needs >=3 chars per query term — 2-char Korean
|
||||
** words (일본, 구글, 우리, ...) fall through to a full-table LIKE scan,
|
||||
** measured at 3-6s per query on a 6.8GB messages table and the #1 driver
|
||||
** of hermes session_search latency.
|
||||
**
|
||||
** What: wrap unicode61. Every token it emits is re-examined; maximal CJK
|
||||
** runs inside the token are re-emitted as overlapping character BIGRAMS
|
||||
** (Lucene CJKAnalyzer semantics), non-CJK segments pass through unchanged.
|
||||
** A lone CJK char (run length 1) is emitted as a unigram. Because FTS5
|
||||
** turns consecutive tokens emitted from one query term into a phrase,
|
||||
** a query word like 캘린더 → [캘린][린더] gets exact substring semantics
|
||||
** with index-speed lookups, down to 2-char terms.
|
||||
**
|
||||
** Build: gcc -shared -fPIC -O2 fts5_cjk.c -o libfts5_cjk.so
|
||||
** Load: conn.load_extension(path) # default entrypoint sqlite3_ftscjk_init
|
||||
** Use: CREATE VIRTUAL TABLE t USING fts5(c, tokenize='cjk_unicode61');
|
||||
** Extra args pass through to unicode61:
|
||||
** tokenize='cjk_unicode61 remove_diacritics 2'
|
||||
*/
|
||||
#include <sqlite3ext.h>
|
||||
SQLITE_EXTENSION_INIT1
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/* ── CJK classification ──────────────────────────────────────────────── */
|
||||
|
||||
static int cjk_is_cjk(unsigned int cp) {
|
||||
return (cp >= 0xAC00 && cp <= 0xD7A3) /* Hangul syllables */
|
||||
|| (cp >= 0x1100 && cp <= 0x11FF) /* Hangul Jamo */
|
||||
|| (cp >= 0x3130 && cp <= 0x318F) /* Hangul compat Jamo */
|
||||
|| (cp >= 0xA960 && cp <= 0xA97F) /* Hangul Jamo ext-A */
|
||||
|| (cp >= 0xD7B0 && cp <= 0xD7FF) /* Hangul Jamo ext-B */
|
||||
|| (cp >= 0x4E00 && cp <= 0x9FFF) /* CJK unified ideographs */
|
||||
|| (cp >= 0x3400 && cp <= 0x4DBF) /* CJK ext A */
|
||||
|| (cp >= 0xF900 && cp <= 0xFAFF) /* CJK compat ideographs */
|
||||
|| (cp >= 0x20000 && cp <= 0x2FA1F) /* CJK ext B..F, compat sup*/
|
||||
|| (cp >= 0x3040 && cp <= 0x309F) /* Hiragana */
|
||||
|| (cp >= 0x30A0 && cp <= 0x30FF) /* Katakana */
|
||||
|| (cp >= 0x31F0 && cp <= 0x31FF); /* Katakana phonetic ext */
|
||||
}
|
||||
|
||||
/* Decode one UTF-8 codepoint at p (n bytes available). Returns byte len
|
||||
** consumed (>=1); stores codepoint in *pCp. Invalid bytes decode as
|
||||
** themselves so segmentation still terminates. */
|
||||
static int cjk_utf8_decode(const unsigned char *p, int n, unsigned int *pCp) {
|
||||
unsigned int c = p[0];
|
||||
if (c < 0x80) { *pCp = c; return 1; }
|
||||
if ((c & 0xE0) == 0xC0 && n >= 2) {
|
||||
*pCp = ((c & 0x1F) << 6) | (p[1] & 0x3F);
|
||||
return 2;
|
||||
}
|
||||
if ((c & 0xF0) == 0xE0 && n >= 3) {
|
||||
*pCp = ((c & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F);
|
||||
return 3;
|
||||
}
|
||||
if ((c & 0xF8) == 0xF0 && n >= 4) {
|
||||
*pCp = ((c & 0x07) << 18) | ((p[1] & 0x3F) << 12) |
|
||||
((p[2] & 0x3F) << 6) | (p[3] & 0x3F);
|
||||
return 4;
|
||||
}
|
||||
*pCp = c;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ── tokenizer plumbing ──────────────────────────────────────────────── */
|
||||
|
||||
typedef struct CjkTokenizer CjkTokenizer;
|
||||
struct CjkTokenizer {
|
||||
fts5_tokenizer inner; /* unicode61 methods */
|
||||
Fts5Tokenizer *pInner; /* unicode61 instance */
|
||||
};
|
||||
|
||||
typedef struct CjkCallbackCtx CjkCallbackCtx;
|
||||
struct CjkCallbackCtx {
|
||||
void *pOuterCtx;
|
||||
int (*xOuterToken)(void*, int, const char*, int, int, int);
|
||||
};
|
||||
|
||||
/* Re-emit one unicode61 token, splitting CJK runs into bigrams.
|
||||
**
|
||||
** Offsets: unicode61 reports [iStart,iEnd) into the ORIGINAL text. For
|
||||
** CJK bytes unicode61's folding is the identity, and ASCII case folding
|
||||
** preserves byte length, so mapping sub-token offsets by byte position
|
||||
** within the token is exact for CJK and correct-length for ASCII. For
|
||||
** rare length-changing folds (accented latin) the highlight offsets can
|
||||
** drift by a few bytes inside that token; matching is unaffected. Every
|
||||
** emitted offset is clamped to [iStart,iEnd).
|
||||
*/
|
||||
static int cjk_emit(CjkCallbackCtx *p, int tflags,
|
||||
const char *pToken, int nToken, int iStart, int iEnd) {
|
||||
const unsigned char *z = (const unsigned char*)pToken;
|
||||
int i = 0;
|
||||
int rc = SQLITE_OK;
|
||||
|
||||
/* Fast path: no CJK anywhere → pass through untouched. */
|
||||
int hasCjk = 0;
|
||||
while (i < nToken) {
|
||||
unsigned int cp;
|
||||
i += cjk_utf8_decode(z + i, nToken - i, &cp);
|
||||
if (cjk_is_cjk(cp)) { hasCjk = 1; break; }
|
||||
}
|
||||
if (!hasCjk) {
|
||||
return p->xOuterToken(p->pOuterCtx, tflags, pToken, nToken, iStart, iEnd);
|
||||
}
|
||||
|
||||
#define CJK_CLAMP_END(v) ((iStart + (v)) > iEnd ? iEnd : (iStart + (v)))
|
||||
i = 0;
|
||||
while (i < nToken && rc == SQLITE_OK) {
|
||||
unsigned int cp;
|
||||
int segStart = i;
|
||||
int len = cjk_utf8_decode(z + i, nToken - i, &cp);
|
||||
if (!cjk_is_cjk(cp)) {
|
||||
/* non-CJK segment: extend to the next CJK char (or end) */
|
||||
i += len;
|
||||
while (i < nToken) {
|
||||
int l2 = cjk_utf8_decode(z + i, nToken - i, &cp);
|
||||
if (cjk_is_cjk(cp)) break;
|
||||
i += l2;
|
||||
}
|
||||
rc = p->xOuterToken(p->pOuterCtx, tflags,
|
||||
pToken + segStart, i - segStart,
|
||||
CJK_CLAMP_END(segStart), CJK_CLAMP_END(i));
|
||||
} else {
|
||||
/* CJK run: collect char byte-boundaries, emit bigrams. */
|
||||
int bounds[3]; /* rolling window: start, mid, end */
|
||||
bounds[0] = segStart;
|
||||
bounds[1] = segStart + len;
|
||||
i += len;
|
||||
int nChars = 1;
|
||||
while (i < nToken) {
|
||||
int l2 = cjk_utf8_decode(z + i, nToken - i, &cp);
|
||||
if (!cjk_is_cjk(cp)) break;
|
||||
i += l2;
|
||||
nChars++;
|
||||
if (nChars >= 2) {
|
||||
bounds[2] = i;
|
||||
if (nChars == 2) {
|
||||
/* first bigram spans bounds[0]..bounds[2] */
|
||||
}
|
||||
rc = p->xOuterToken(p->pOuterCtx, tflags,
|
||||
pToken + bounds[0], bounds[2] - bounds[0],
|
||||
CJK_CLAMP_END(bounds[0]), CJK_CLAMP_END(bounds[2]));
|
||||
if (rc != SQLITE_OK) break;
|
||||
bounds[0] = bounds[1];
|
||||
bounds[1] = bounds[2];
|
||||
}
|
||||
}
|
||||
if (rc == SQLITE_OK && nChars == 1) {
|
||||
/* lone CJK char: emit as unigram */
|
||||
rc = p->xOuterToken(p->pOuterCtx, tflags,
|
||||
pToken + segStart, bounds[1] - segStart,
|
||||
CJK_CLAMP_END(segStart), CJK_CLAMP_END(bounds[1]));
|
||||
}
|
||||
}
|
||||
}
|
||||
#undef CJK_CLAMP_END
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int cjkInnerCallback(void *pCtx, int tflags,
|
||||
const char *pToken, int nToken,
|
||||
int iStart, int iEnd) {
|
||||
return cjk_emit((CjkCallbackCtx*)pCtx, tflags, pToken, nToken, iStart, iEnd);
|
||||
}
|
||||
|
||||
static int cjkCreate(void *pApiCtx, const char **azArg, int nArg,
|
||||
Fts5Tokenizer **ppOut) {
|
||||
fts5_api *pApi = (fts5_api*)pApiCtx;
|
||||
CjkTokenizer *p;
|
||||
void *pInnerCtx = 0;
|
||||
int rc;
|
||||
|
||||
p = (CjkTokenizer*)sqlite3_malloc(sizeof(CjkTokenizer));
|
||||
if (!p) return SQLITE_NOMEM;
|
||||
memset(p, 0, sizeof(*p));
|
||||
|
||||
rc = pApi->xFindTokenizer(pApi, "unicode61", &pInnerCtx, &p->inner);
|
||||
if (rc == SQLITE_OK) {
|
||||
rc = p->inner.xCreate(pInnerCtx, azArg, nArg, &p->pInner);
|
||||
}
|
||||
if (rc != SQLITE_OK) {
|
||||
sqlite3_free(p);
|
||||
return rc;
|
||||
}
|
||||
*ppOut = (Fts5Tokenizer*)p;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static void cjkDelete(Fts5Tokenizer *pTok) {
|
||||
CjkTokenizer *p = (CjkTokenizer*)pTok;
|
||||
if (p) {
|
||||
if (p->pInner) p->inner.xDelete(p->pInner);
|
||||
sqlite3_free(p);
|
||||
}
|
||||
}
|
||||
|
||||
static int cjkTokenize(Fts5Tokenizer *pTok, void *pCtx, int flags,
|
||||
const char *pText, int nText,
|
||||
int (*xToken)(void*, int, const char*, int, int, int)) {
|
||||
CjkTokenizer *p = (CjkTokenizer*)pTok;
|
||||
CjkCallbackCtx cb;
|
||||
cb.pOuterCtx = pCtx;
|
||||
cb.xOuterToken = xToken;
|
||||
return p->inner.xTokenize(p->pInner, &cb, flags, pText, nText,
|
||||
cjkInnerCallback);
|
||||
}
|
||||
|
||||
/* ── registration ────────────────────────────────────────────────────── */
|
||||
|
||||
static fts5_api *cjkFts5Api(sqlite3 *db) {
|
||||
fts5_api *pRet = 0;
|
||||
sqlite3_stmt *pStmt = 0;
|
||||
if (sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0) == SQLITE_OK) {
|
||||
sqlite3_bind_pointer(pStmt, 1, (void*)&pRet, "fts5_api_ptr", 0);
|
||||
sqlite3_step(pStmt);
|
||||
}
|
||||
sqlite3_finalize(pStmt);
|
||||
return pRet;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
__declspec(dllexport)
|
||||
#endif
|
||||
int sqlite3_ftscjk_init(sqlite3 *db, char **pzErrMsg,
|
||||
const sqlite3_api_routines *pApi) {
|
||||
fts5_api *pFts;
|
||||
static fts5_tokenizer tok = { cjkCreate, cjkDelete, cjkTokenize };
|
||||
SQLITE_EXTENSION_INIT2(pApi);
|
||||
(void)pzErrMsg;
|
||||
pFts = cjkFts5Api(db);
|
||||
if (!pFts) {
|
||||
if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("fts5_cjk: FTS5 unavailable");
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
return pFts->xCreateTokenizer(pFts, "cjk_unicode61", (void*)pFts, &tok, 0);
|
||||
}
|
||||
|
||||
/* Alias for callers that spell out the underscored basename. */
|
||||
#ifdef _WIN32
|
||||
__declspec(dllexport)
|
||||
#endif
|
||||
int sqlite3_fts5_cjk_init(sqlite3 *db, char **pzErrMsg,
|
||||
const sqlite3_api_routines *pApi) {
|
||||
return sqlite3_ftscjk_init(db, pzErrMsg, pApi);
|
||||
}
|
||||
Vendored
+13775
File diff suppressed because it is too large
Load Diff
Vendored
+723
@@ -0,0 +1,723 @@
|
||||
/*
|
||||
** 2006 June 7
|
||||
**
|
||||
** The author disclaims copyright to this source code. In place of
|
||||
** a legal notice, here is a blessing:
|
||||
**
|
||||
** May you do good and not evil.
|
||||
** May you find forgiveness for yourself and forgive others.
|
||||
** May you share freely, never taking more than you give.
|
||||
**
|
||||
*************************************************************************
|
||||
** This header file defines the SQLite interface for use by
|
||||
** shared libraries that want to be imported as extensions into
|
||||
** an SQLite instance. Shared libraries that intend to be loaded
|
||||
** as extensions by SQLite should #include this file instead of
|
||||
** sqlite3.h.
|
||||
*/
|
||||
#ifndef SQLITE3EXT_H
|
||||
#define SQLITE3EXT_H
|
||||
#include "sqlite3.h"
|
||||
|
||||
/*
|
||||
** The following structure holds pointers to all of the SQLite API
|
||||
** routines.
|
||||
**
|
||||
** WARNING: In order to maintain backwards compatibility, add new
|
||||
** interfaces to the end of this structure only. If you insert new
|
||||
** interfaces in the middle of this structure, then older different
|
||||
** versions of SQLite will not be able to load each other's shared
|
||||
** libraries!
|
||||
*/
|
||||
struct sqlite3_api_routines {
|
||||
void * (*aggregate_context)(sqlite3_context*,int nBytes);
|
||||
int (*aggregate_count)(sqlite3_context*);
|
||||
int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*));
|
||||
int (*bind_double)(sqlite3_stmt*,int,double);
|
||||
int (*bind_int)(sqlite3_stmt*,int,int);
|
||||
int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64);
|
||||
int (*bind_null)(sqlite3_stmt*,int);
|
||||
int (*bind_parameter_count)(sqlite3_stmt*);
|
||||
int (*bind_parameter_index)(sqlite3_stmt*,const char*zName);
|
||||
const char * (*bind_parameter_name)(sqlite3_stmt*,int);
|
||||
int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*));
|
||||
int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*));
|
||||
int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*);
|
||||
int (*busy_handler)(sqlite3*,int(*)(void*,int),void*);
|
||||
int (*busy_timeout)(sqlite3*,int ms);
|
||||
int (*changes)(sqlite3*);
|
||||
int (*close)(sqlite3*);
|
||||
int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,
|
||||
int eTextRep,const char*));
|
||||
int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,
|
||||
int eTextRep,const void*));
|
||||
const void * (*column_blob)(sqlite3_stmt*,int iCol);
|
||||
int (*column_bytes)(sqlite3_stmt*,int iCol);
|
||||
int (*column_bytes16)(sqlite3_stmt*,int iCol);
|
||||
int (*column_count)(sqlite3_stmt*pStmt);
|
||||
const char * (*column_database_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_database_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_decltype)(sqlite3_stmt*,int i);
|
||||
const void * (*column_decltype16)(sqlite3_stmt*,int);
|
||||
double (*column_double)(sqlite3_stmt*,int iCol);
|
||||
int (*column_int)(sqlite3_stmt*,int iCol);
|
||||
sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol);
|
||||
const char * (*column_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_origin_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_origin_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_table_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_table_name16)(sqlite3_stmt*,int);
|
||||
const unsigned char * (*column_text)(sqlite3_stmt*,int iCol);
|
||||
const void * (*column_text16)(sqlite3_stmt*,int iCol);
|
||||
int (*column_type)(sqlite3_stmt*,int iCol);
|
||||
sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol);
|
||||
void * (*commit_hook)(sqlite3*,int(*)(void*),void*);
|
||||
int (*complete)(const char*sql);
|
||||
int (*complete16)(const void*sql);
|
||||
int (*create_collation)(sqlite3*,const char*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*));
|
||||
int (*create_collation16)(sqlite3*,const void*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*));
|
||||
int (*create_function)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*));
|
||||
int (*create_function16)(sqlite3*,const void*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*));
|
||||
int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*);
|
||||
int (*data_count)(sqlite3_stmt*pStmt);
|
||||
sqlite3 * (*db_handle)(sqlite3_stmt*);
|
||||
int (*declare_vtab)(sqlite3*,const char*);
|
||||
int (*enable_shared_cache)(int);
|
||||
int (*errcode)(sqlite3*db);
|
||||
const char * (*errmsg)(sqlite3*);
|
||||
const void * (*errmsg16)(sqlite3*);
|
||||
int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**);
|
||||
int (*expired)(sqlite3_stmt*);
|
||||
int (*finalize)(sqlite3_stmt*pStmt);
|
||||
void (*free)(void*);
|
||||
void (*free_table)(char**result);
|
||||
int (*get_autocommit)(sqlite3*);
|
||||
void * (*get_auxdata)(sqlite3_context*,int);
|
||||
int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**);
|
||||
int (*global_recover)(void);
|
||||
void (*interruptx)(sqlite3*);
|
||||
sqlite_int64 (*last_insert_rowid)(sqlite3*);
|
||||
const char * (*libversion)(void);
|
||||
int (*libversion_number)(void);
|
||||
void *(*malloc)(int);
|
||||
char * (*mprintf)(const char*,...);
|
||||
int (*open)(const char*,sqlite3**);
|
||||
int (*open16)(const void*,sqlite3**);
|
||||
int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
|
||||
int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
|
||||
void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*);
|
||||
void (*progress_handler)(sqlite3*,int,int(*)(void*),void*);
|
||||
void *(*realloc)(void*,int);
|
||||
int (*reset)(sqlite3_stmt*pStmt);
|
||||
void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_double)(sqlite3_context*,double);
|
||||
void (*result_error)(sqlite3_context*,const char*,int);
|
||||
void (*result_error16)(sqlite3_context*,const void*,int);
|
||||
void (*result_int)(sqlite3_context*,int);
|
||||
void (*result_int64)(sqlite3_context*,sqlite_int64);
|
||||
void (*result_null)(sqlite3_context*);
|
||||
void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*));
|
||||
void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_value)(sqlite3_context*,sqlite3_value*);
|
||||
void * (*rollback_hook)(sqlite3*,void(*)(void*),void*);
|
||||
int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
|
||||
const char*,const char*),void*);
|
||||
void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
|
||||
char * (*xsnprintf)(int,char*,const char*,...);
|
||||
int (*step)(sqlite3_stmt*);
|
||||
int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
|
||||
char const**,char const**,int*,int*,int*);
|
||||
void (*thread_cleanup)(void);
|
||||
int (*total_changes)(sqlite3*);
|
||||
void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*);
|
||||
int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*);
|
||||
void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,
|
||||
sqlite_int64),void*);
|
||||
void * (*user_data)(sqlite3_context*);
|
||||
const void * (*value_blob)(sqlite3_value*);
|
||||
int (*value_bytes)(sqlite3_value*);
|
||||
int (*value_bytes16)(sqlite3_value*);
|
||||
double (*value_double)(sqlite3_value*);
|
||||
int (*value_int)(sqlite3_value*);
|
||||
sqlite_int64 (*value_int64)(sqlite3_value*);
|
||||
int (*value_numeric_type)(sqlite3_value*);
|
||||
const unsigned char * (*value_text)(sqlite3_value*);
|
||||
const void * (*value_text16)(sqlite3_value*);
|
||||
const void * (*value_text16be)(sqlite3_value*);
|
||||
const void * (*value_text16le)(sqlite3_value*);
|
||||
int (*value_type)(sqlite3_value*);
|
||||
char *(*vmprintf)(const char*,va_list);
|
||||
/* Added ??? */
|
||||
int (*overload_function)(sqlite3*, const char *zFuncName, int nArg);
|
||||
/* Added by 3.3.13 */
|
||||
int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
|
||||
int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
|
||||
int (*clear_bindings)(sqlite3_stmt*);
|
||||
/* Added by 3.4.1 */
|
||||
int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,
|
||||
void (*xDestroy)(void *));
|
||||
/* Added by 3.5.0 */
|
||||
int (*bind_zeroblob)(sqlite3_stmt*,int,int);
|
||||
int (*blob_bytes)(sqlite3_blob*);
|
||||
int (*blob_close)(sqlite3_blob*);
|
||||
int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,
|
||||
int,sqlite3_blob**);
|
||||
int (*blob_read)(sqlite3_blob*,void*,int,int);
|
||||
int (*blob_write)(sqlite3_blob*,const void*,int,int);
|
||||
int (*create_collation_v2)(sqlite3*,const char*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*),
|
||||
void(*)(void*));
|
||||
int (*file_control)(sqlite3*,const char*,int,void*);
|
||||
sqlite3_int64 (*memory_highwater)(int);
|
||||
sqlite3_int64 (*memory_used)(void);
|
||||
sqlite3_mutex *(*mutex_alloc)(int);
|
||||
void (*mutex_enter)(sqlite3_mutex*);
|
||||
void (*mutex_free)(sqlite3_mutex*);
|
||||
void (*mutex_leave)(sqlite3_mutex*);
|
||||
int (*mutex_try)(sqlite3_mutex*);
|
||||
int (*open_v2)(const char*,sqlite3**,int,const char*);
|
||||
int (*release_memory)(int);
|
||||
void (*result_error_nomem)(sqlite3_context*);
|
||||
void (*result_error_toobig)(sqlite3_context*);
|
||||
int (*sleep)(int);
|
||||
void (*soft_heap_limit)(int);
|
||||
sqlite3_vfs *(*vfs_find)(const char*);
|
||||
int (*vfs_register)(sqlite3_vfs*,int);
|
||||
int (*vfs_unregister)(sqlite3_vfs*);
|
||||
int (*xthreadsafe)(void);
|
||||
void (*result_zeroblob)(sqlite3_context*,int);
|
||||
void (*result_error_code)(sqlite3_context*,int);
|
||||
int (*test_control)(int, ...);
|
||||
void (*randomness)(int,void*);
|
||||
sqlite3 *(*context_db_handle)(sqlite3_context*);
|
||||
int (*extended_result_codes)(sqlite3*,int);
|
||||
int (*limit)(sqlite3*,int,int);
|
||||
sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*);
|
||||
const char *(*sql)(sqlite3_stmt*);
|
||||
int (*status)(int,int*,int*,int);
|
||||
int (*backup_finish)(sqlite3_backup*);
|
||||
sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*);
|
||||
int (*backup_pagecount)(sqlite3_backup*);
|
||||
int (*backup_remaining)(sqlite3_backup*);
|
||||
int (*backup_step)(sqlite3_backup*,int);
|
||||
const char *(*compileoption_get)(int);
|
||||
int (*compileoption_used)(const char*);
|
||||
int (*create_function_v2)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*),
|
||||
void(*xDestroy)(void*));
|
||||
int (*db_config)(sqlite3*,int,...);
|
||||
sqlite3_mutex *(*db_mutex)(sqlite3*);
|
||||
int (*db_status)(sqlite3*,int,int*,int*,int);
|
||||
int (*extended_errcode)(sqlite3*);
|
||||
void (*log)(int,const char*,...);
|
||||
sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64);
|
||||
const char *(*sourceid)(void);
|
||||
int (*stmt_status)(sqlite3_stmt*,int,int);
|
||||
int (*strnicmp)(const char*,const char*,int);
|
||||
int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
|
||||
int (*wal_autocheckpoint)(sqlite3*,int);
|
||||
int (*wal_checkpoint)(sqlite3*,const char*);
|
||||
void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
|
||||
int (*blob_reopen)(sqlite3_blob*,sqlite3_int64);
|
||||
int (*vtab_config)(sqlite3*,int op,...);
|
||||
int (*vtab_on_conflict)(sqlite3*);
|
||||
/* Version 3.7.16 and later */
|
||||
int (*close_v2)(sqlite3*);
|
||||
const char *(*db_filename)(sqlite3*,const char*);
|
||||
int (*db_readonly)(sqlite3*,const char*);
|
||||
int (*db_release_memory)(sqlite3*);
|
||||
const char *(*errstr)(int);
|
||||
int (*stmt_busy)(sqlite3_stmt*);
|
||||
int (*stmt_readonly)(sqlite3_stmt*);
|
||||
int (*stricmp)(const char*,const char*);
|
||||
int (*uri_boolean)(const char*,const char*,int);
|
||||
sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64);
|
||||
const char *(*uri_parameter)(const char*,const char*);
|
||||
char *(*xvsnprintf)(int,char*,const char*,va_list);
|
||||
int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*);
|
||||
/* Version 3.8.7 and later */
|
||||
int (*auto_extension)(void(*)(void));
|
||||
int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64,
|
||||
void(*)(void*));
|
||||
int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64,
|
||||
void(*)(void*),unsigned char);
|
||||
int (*cancel_auto_extension)(void(*)(void));
|
||||
int (*load_extension)(sqlite3*,const char*,const char*,char**);
|
||||
void *(*malloc64)(sqlite3_uint64);
|
||||
sqlite3_uint64 (*msize)(void*);
|
||||
void *(*realloc64)(void*,sqlite3_uint64);
|
||||
void (*reset_auto_extension)(void);
|
||||
void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64,
|
||||
void(*)(void*));
|
||||
void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64,
|
||||
void(*)(void*), unsigned char);
|
||||
int (*strglob)(const char*,const char*);
|
||||
/* Version 3.8.11 and later */
|
||||
sqlite3_value *(*value_dup)(const sqlite3_value*);
|
||||
void (*value_free)(sqlite3_value*);
|
||||
int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64);
|
||||
int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64);
|
||||
/* Version 3.9.0 and later */
|
||||
unsigned int (*value_subtype)(sqlite3_value*);
|
||||
void (*result_subtype)(sqlite3_context*,unsigned int);
|
||||
/* Version 3.10.0 and later */
|
||||
int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int);
|
||||
int (*strlike)(const char*,const char*,unsigned int);
|
||||
int (*db_cacheflush)(sqlite3*);
|
||||
/* Version 3.12.0 and later */
|
||||
int (*system_errno)(sqlite3*);
|
||||
/* Version 3.14.0 and later */
|
||||
int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*);
|
||||
char *(*expanded_sql)(sqlite3_stmt*);
|
||||
/* Version 3.18.0 and later */
|
||||
void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64);
|
||||
/* Version 3.20.0 and later */
|
||||
int (*prepare_v3)(sqlite3*,const char*,int,unsigned int,
|
||||
sqlite3_stmt**,const char**);
|
||||
int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int,
|
||||
sqlite3_stmt**,const void**);
|
||||
int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*));
|
||||
void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*));
|
||||
void *(*value_pointer)(sqlite3_value*,const char*);
|
||||
int (*vtab_nochange)(sqlite3_context*);
|
||||
int (*value_nochange)(sqlite3_value*);
|
||||
const char *(*vtab_collation)(sqlite3_index_info*,int);
|
||||
/* Version 3.24.0 and later */
|
||||
int (*keyword_count)(void);
|
||||
int (*keyword_name)(int,const char**,int*);
|
||||
int (*keyword_check)(const char*,int);
|
||||
sqlite3_str *(*str_new)(sqlite3*);
|
||||
char *(*str_finish)(sqlite3_str*);
|
||||
void (*str_appendf)(sqlite3_str*, const char *zFormat, ...);
|
||||
void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list);
|
||||
void (*str_append)(sqlite3_str*, const char *zIn, int N);
|
||||
void (*str_appendall)(sqlite3_str*, const char *zIn);
|
||||
void (*str_appendchar)(sqlite3_str*, int N, char C);
|
||||
void (*str_reset)(sqlite3_str*);
|
||||
int (*str_errcode)(sqlite3_str*);
|
||||
int (*str_length)(sqlite3_str*);
|
||||
char *(*str_value)(sqlite3_str*);
|
||||
/* Version 3.25.0 and later */
|
||||
int (*create_window_function)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*),
|
||||
void (*xValue)(sqlite3_context*),
|
||||
void (*xInv)(sqlite3_context*,int,sqlite3_value**),
|
||||
void(*xDestroy)(void*));
|
||||
/* Version 3.26.0 and later */
|
||||
const char *(*normalized_sql)(sqlite3_stmt*);
|
||||
/* Version 3.28.0 and later */
|
||||
int (*stmt_isexplain)(sqlite3_stmt*);
|
||||
int (*value_frombind)(sqlite3_value*);
|
||||
/* Version 3.30.0 and later */
|
||||
int (*drop_modules)(sqlite3*,const char**);
|
||||
/* Version 3.31.0 and later */
|
||||
sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64);
|
||||
const char *(*uri_key)(const char*,int);
|
||||
const char *(*filename_database)(const char*);
|
||||
const char *(*filename_journal)(const char*);
|
||||
const char *(*filename_wal)(const char*);
|
||||
/* Version 3.32.0 and later */
|
||||
const char *(*create_filename)(const char*,const char*,const char*,
|
||||
int,const char**);
|
||||
void (*free_filename)(const char*);
|
||||
sqlite3_file *(*database_file_object)(const char*);
|
||||
/* Version 3.34.0 and later */
|
||||
int (*txn_state)(sqlite3*,const char*);
|
||||
/* Version 3.36.1 and later */
|
||||
sqlite3_int64 (*changes64)(sqlite3*);
|
||||
sqlite3_int64 (*total_changes64)(sqlite3*);
|
||||
/* Version 3.37.0 and later */
|
||||
int (*autovacuum_pages)(sqlite3*,
|
||||
unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int),
|
||||
void*, void(*)(void*));
|
||||
/* Version 3.38.0 and later */
|
||||
int (*error_offset)(sqlite3*);
|
||||
int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**);
|
||||
int (*vtab_distinct)(sqlite3_index_info*);
|
||||
int (*vtab_in)(sqlite3_index_info*,int,int);
|
||||
int (*vtab_in_first)(sqlite3_value*,sqlite3_value**);
|
||||
int (*vtab_in_next)(sqlite3_value*,sqlite3_value**);
|
||||
/* Version 3.39.0 and later */
|
||||
int (*deserialize)(sqlite3*,const char*,unsigned char*,
|
||||
sqlite3_int64,sqlite3_int64,unsigned);
|
||||
unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*,
|
||||
unsigned int);
|
||||
const char *(*db_name)(sqlite3*,int);
|
||||
/* Version 3.40.0 and later */
|
||||
int (*value_encoding)(sqlite3_value*);
|
||||
/* Version 3.41.0 and later */
|
||||
int (*is_interrupted)(sqlite3*);
|
||||
/* Version 3.43.0 and later */
|
||||
int (*stmt_explain)(sqlite3_stmt*,int);
|
||||
/* Version 3.44.0 and later */
|
||||
void *(*get_clientdata)(sqlite3*,const char*);
|
||||
int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*));
|
||||
/* Version 3.50.0 and later */
|
||||
int (*setlk_timeout)(sqlite3*,int,int);
|
||||
};
|
||||
|
||||
/*
|
||||
** This is the function signature used for all extension entry points. It
|
||||
** is also defined in the file "loadext.c".
|
||||
*/
|
||||
typedef int (*sqlite3_loadext_entry)(
|
||||
sqlite3 *db, /* Handle to the database. */
|
||||
char **pzErrMsg, /* Used to set error string on failure. */
|
||||
const sqlite3_api_routines *pThunk /* Extension API function pointers. */
|
||||
);
|
||||
|
||||
/*
|
||||
** The following macros redefine the API routines so that they are
|
||||
** redirected through the global sqlite3_api structure.
|
||||
**
|
||||
** This header file is also used by the loadext.c source file
|
||||
** (part of the main SQLite library - not an extension) so that
|
||||
** it can get access to the sqlite3_api_routines structure
|
||||
** definition. But the main library does not want to redefine
|
||||
** the API. So the redefinition macros are only valid if the
|
||||
** SQLITE_CORE macros is undefined.
|
||||
*/
|
||||
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
|
||||
#define sqlite3_aggregate_context sqlite3_api->aggregate_context
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_aggregate_count sqlite3_api->aggregate_count
|
||||
#endif
|
||||
#define sqlite3_bind_blob sqlite3_api->bind_blob
|
||||
#define sqlite3_bind_double sqlite3_api->bind_double
|
||||
#define sqlite3_bind_int sqlite3_api->bind_int
|
||||
#define sqlite3_bind_int64 sqlite3_api->bind_int64
|
||||
#define sqlite3_bind_null sqlite3_api->bind_null
|
||||
#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count
|
||||
#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index
|
||||
#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name
|
||||
#define sqlite3_bind_text sqlite3_api->bind_text
|
||||
#define sqlite3_bind_text16 sqlite3_api->bind_text16
|
||||
#define sqlite3_bind_value sqlite3_api->bind_value
|
||||
#define sqlite3_busy_handler sqlite3_api->busy_handler
|
||||
#define sqlite3_busy_timeout sqlite3_api->busy_timeout
|
||||
#define sqlite3_changes sqlite3_api->changes
|
||||
#define sqlite3_close sqlite3_api->close
|
||||
#define sqlite3_collation_needed sqlite3_api->collation_needed
|
||||
#define sqlite3_collation_needed16 sqlite3_api->collation_needed16
|
||||
#define sqlite3_column_blob sqlite3_api->column_blob
|
||||
#define sqlite3_column_bytes sqlite3_api->column_bytes
|
||||
#define sqlite3_column_bytes16 sqlite3_api->column_bytes16
|
||||
#define sqlite3_column_count sqlite3_api->column_count
|
||||
#define sqlite3_column_database_name sqlite3_api->column_database_name
|
||||
#define sqlite3_column_database_name16 sqlite3_api->column_database_name16
|
||||
#define sqlite3_column_decltype sqlite3_api->column_decltype
|
||||
#define sqlite3_column_decltype16 sqlite3_api->column_decltype16
|
||||
#define sqlite3_column_double sqlite3_api->column_double
|
||||
#define sqlite3_column_int sqlite3_api->column_int
|
||||
#define sqlite3_column_int64 sqlite3_api->column_int64
|
||||
#define sqlite3_column_name sqlite3_api->column_name
|
||||
#define sqlite3_column_name16 sqlite3_api->column_name16
|
||||
#define sqlite3_column_origin_name sqlite3_api->column_origin_name
|
||||
#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16
|
||||
#define sqlite3_column_table_name sqlite3_api->column_table_name
|
||||
#define sqlite3_column_table_name16 sqlite3_api->column_table_name16
|
||||
#define sqlite3_column_text sqlite3_api->column_text
|
||||
#define sqlite3_column_text16 sqlite3_api->column_text16
|
||||
#define sqlite3_column_type sqlite3_api->column_type
|
||||
#define sqlite3_column_value sqlite3_api->column_value
|
||||
#define sqlite3_commit_hook sqlite3_api->commit_hook
|
||||
#define sqlite3_complete sqlite3_api->complete
|
||||
#define sqlite3_complete16 sqlite3_api->complete16
|
||||
#define sqlite3_create_collation sqlite3_api->create_collation
|
||||
#define sqlite3_create_collation16 sqlite3_api->create_collation16
|
||||
#define sqlite3_create_function sqlite3_api->create_function
|
||||
#define sqlite3_create_function16 sqlite3_api->create_function16
|
||||
#define sqlite3_create_module sqlite3_api->create_module
|
||||
#define sqlite3_create_module_v2 sqlite3_api->create_module_v2
|
||||
#define sqlite3_data_count sqlite3_api->data_count
|
||||
#define sqlite3_db_handle sqlite3_api->db_handle
|
||||
#define sqlite3_declare_vtab sqlite3_api->declare_vtab
|
||||
#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache
|
||||
#define sqlite3_errcode sqlite3_api->errcode
|
||||
#define sqlite3_errmsg sqlite3_api->errmsg
|
||||
#define sqlite3_errmsg16 sqlite3_api->errmsg16
|
||||
#define sqlite3_exec sqlite3_api->exec
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_expired sqlite3_api->expired
|
||||
#endif
|
||||
#define sqlite3_finalize sqlite3_api->finalize
|
||||
#define sqlite3_free sqlite3_api->free
|
||||
#define sqlite3_free_table sqlite3_api->free_table
|
||||
#define sqlite3_get_autocommit sqlite3_api->get_autocommit
|
||||
#define sqlite3_get_auxdata sqlite3_api->get_auxdata
|
||||
#define sqlite3_get_table sqlite3_api->get_table
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_global_recover sqlite3_api->global_recover
|
||||
#endif
|
||||
#define sqlite3_interrupt sqlite3_api->interruptx
|
||||
#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid
|
||||
#define sqlite3_libversion sqlite3_api->libversion
|
||||
#define sqlite3_libversion_number sqlite3_api->libversion_number
|
||||
#define sqlite3_malloc sqlite3_api->malloc
|
||||
#define sqlite3_mprintf sqlite3_api->mprintf
|
||||
#define sqlite3_open sqlite3_api->open
|
||||
#define sqlite3_open16 sqlite3_api->open16
|
||||
#define sqlite3_prepare sqlite3_api->prepare
|
||||
#define sqlite3_prepare16 sqlite3_api->prepare16
|
||||
#define sqlite3_prepare_v2 sqlite3_api->prepare_v2
|
||||
#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2
|
||||
#define sqlite3_profile sqlite3_api->profile
|
||||
#define sqlite3_progress_handler sqlite3_api->progress_handler
|
||||
#define sqlite3_realloc sqlite3_api->realloc
|
||||
#define sqlite3_reset sqlite3_api->reset
|
||||
#define sqlite3_result_blob sqlite3_api->result_blob
|
||||
#define sqlite3_result_double sqlite3_api->result_double
|
||||
#define sqlite3_result_error sqlite3_api->result_error
|
||||
#define sqlite3_result_error16 sqlite3_api->result_error16
|
||||
#define sqlite3_result_int sqlite3_api->result_int
|
||||
#define sqlite3_result_int64 sqlite3_api->result_int64
|
||||
#define sqlite3_result_null sqlite3_api->result_null
|
||||
#define sqlite3_result_text sqlite3_api->result_text
|
||||
#define sqlite3_result_text16 sqlite3_api->result_text16
|
||||
#define sqlite3_result_text16be sqlite3_api->result_text16be
|
||||
#define sqlite3_result_text16le sqlite3_api->result_text16le
|
||||
#define sqlite3_result_value sqlite3_api->result_value
|
||||
#define sqlite3_rollback_hook sqlite3_api->rollback_hook
|
||||
#define sqlite3_set_authorizer sqlite3_api->set_authorizer
|
||||
#define sqlite3_set_auxdata sqlite3_api->set_auxdata
|
||||
#define sqlite3_snprintf sqlite3_api->xsnprintf
|
||||
#define sqlite3_step sqlite3_api->step
|
||||
#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata
|
||||
#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup
|
||||
#define sqlite3_total_changes sqlite3_api->total_changes
|
||||
#define sqlite3_trace sqlite3_api->trace
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings
|
||||
#endif
|
||||
#define sqlite3_update_hook sqlite3_api->update_hook
|
||||
#define sqlite3_user_data sqlite3_api->user_data
|
||||
#define sqlite3_value_blob sqlite3_api->value_blob
|
||||
#define sqlite3_value_bytes sqlite3_api->value_bytes
|
||||
#define sqlite3_value_bytes16 sqlite3_api->value_bytes16
|
||||
#define sqlite3_value_double sqlite3_api->value_double
|
||||
#define sqlite3_value_int sqlite3_api->value_int
|
||||
#define sqlite3_value_int64 sqlite3_api->value_int64
|
||||
#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type
|
||||
#define sqlite3_value_text sqlite3_api->value_text
|
||||
#define sqlite3_value_text16 sqlite3_api->value_text16
|
||||
#define sqlite3_value_text16be sqlite3_api->value_text16be
|
||||
#define sqlite3_value_text16le sqlite3_api->value_text16le
|
||||
#define sqlite3_value_type sqlite3_api->value_type
|
||||
#define sqlite3_vmprintf sqlite3_api->vmprintf
|
||||
#define sqlite3_vsnprintf sqlite3_api->xvsnprintf
|
||||
#define sqlite3_overload_function sqlite3_api->overload_function
|
||||
#define sqlite3_prepare_v2 sqlite3_api->prepare_v2
|
||||
#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2
|
||||
#define sqlite3_clear_bindings sqlite3_api->clear_bindings
|
||||
#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob
|
||||
#define sqlite3_blob_bytes sqlite3_api->blob_bytes
|
||||
#define sqlite3_blob_close sqlite3_api->blob_close
|
||||
#define sqlite3_blob_open sqlite3_api->blob_open
|
||||
#define sqlite3_blob_read sqlite3_api->blob_read
|
||||
#define sqlite3_blob_write sqlite3_api->blob_write
|
||||
#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2
|
||||
#define sqlite3_file_control sqlite3_api->file_control
|
||||
#define sqlite3_memory_highwater sqlite3_api->memory_highwater
|
||||
#define sqlite3_memory_used sqlite3_api->memory_used
|
||||
#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc
|
||||
#define sqlite3_mutex_enter sqlite3_api->mutex_enter
|
||||
#define sqlite3_mutex_free sqlite3_api->mutex_free
|
||||
#define sqlite3_mutex_leave sqlite3_api->mutex_leave
|
||||
#define sqlite3_mutex_try sqlite3_api->mutex_try
|
||||
#define sqlite3_open_v2 sqlite3_api->open_v2
|
||||
#define sqlite3_release_memory sqlite3_api->release_memory
|
||||
#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem
|
||||
#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig
|
||||
#define sqlite3_sleep sqlite3_api->sleep
|
||||
#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit
|
||||
#define sqlite3_vfs_find sqlite3_api->vfs_find
|
||||
#define sqlite3_vfs_register sqlite3_api->vfs_register
|
||||
#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister
|
||||
#define sqlite3_threadsafe sqlite3_api->xthreadsafe
|
||||
#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob
|
||||
#define sqlite3_result_error_code sqlite3_api->result_error_code
|
||||
#define sqlite3_test_control sqlite3_api->test_control
|
||||
#define sqlite3_randomness sqlite3_api->randomness
|
||||
#define sqlite3_context_db_handle sqlite3_api->context_db_handle
|
||||
#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes
|
||||
#define sqlite3_limit sqlite3_api->limit
|
||||
#define sqlite3_next_stmt sqlite3_api->next_stmt
|
||||
#define sqlite3_sql sqlite3_api->sql
|
||||
#define sqlite3_status sqlite3_api->status
|
||||
#define sqlite3_backup_finish sqlite3_api->backup_finish
|
||||
#define sqlite3_backup_init sqlite3_api->backup_init
|
||||
#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount
|
||||
#define sqlite3_backup_remaining sqlite3_api->backup_remaining
|
||||
#define sqlite3_backup_step sqlite3_api->backup_step
|
||||
#define sqlite3_compileoption_get sqlite3_api->compileoption_get
|
||||
#define sqlite3_compileoption_used sqlite3_api->compileoption_used
|
||||
#define sqlite3_create_function_v2 sqlite3_api->create_function_v2
|
||||
#define sqlite3_db_config sqlite3_api->db_config
|
||||
#define sqlite3_db_mutex sqlite3_api->db_mutex
|
||||
#define sqlite3_db_status sqlite3_api->db_status
|
||||
#define sqlite3_extended_errcode sqlite3_api->extended_errcode
|
||||
#define sqlite3_log sqlite3_api->log
|
||||
#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64
|
||||
#define sqlite3_sourceid sqlite3_api->sourceid
|
||||
#define sqlite3_stmt_status sqlite3_api->stmt_status
|
||||
#define sqlite3_strnicmp sqlite3_api->strnicmp
|
||||
#define sqlite3_unlock_notify sqlite3_api->unlock_notify
|
||||
#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
|
||||
#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
|
||||
#define sqlite3_wal_hook sqlite3_api->wal_hook
|
||||
#define sqlite3_blob_reopen sqlite3_api->blob_reopen
|
||||
#define sqlite3_vtab_config sqlite3_api->vtab_config
|
||||
#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict
|
||||
/* Version 3.7.16 and later */
|
||||
#define sqlite3_close_v2 sqlite3_api->close_v2
|
||||
#define sqlite3_db_filename sqlite3_api->db_filename
|
||||
#define sqlite3_db_readonly sqlite3_api->db_readonly
|
||||
#define sqlite3_db_release_memory sqlite3_api->db_release_memory
|
||||
#define sqlite3_errstr sqlite3_api->errstr
|
||||
#define sqlite3_stmt_busy sqlite3_api->stmt_busy
|
||||
#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly
|
||||
#define sqlite3_stricmp sqlite3_api->stricmp
|
||||
#define sqlite3_uri_boolean sqlite3_api->uri_boolean
|
||||
#define sqlite3_uri_int64 sqlite3_api->uri_int64
|
||||
#define sqlite3_uri_parameter sqlite3_api->uri_parameter
|
||||
#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf
|
||||
#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2
|
||||
/* Version 3.8.7 and later */
|
||||
#define sqlite3_auto_extension sqlite3_api->auto_extension
|
||||
#define sqlite3_bind_blob64 sqlite3_api->bind_blob64
|
||||
#define sqlite3_bind_text64 sqlite3_api->bind_text64
|
||||
#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension
|
||||
#define sqlite3_load_extension sqlite3_api->load_extension
|
||||
#define sqlite3_malloc64 sqlite3_api->malloc64
|
||||
#define sqlite3_msize sqlite3_api->msize
|
||||
#define sqlite3_realloc64 sqlite3_api->realloc64
|
||||
#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension
|
||||
#define sqlite3_result_blob64 sqlite3_api->result_blob64
|
||||
#define sqlite3_result_text64 sqlite3_api->result_text64
|
||||
#define sqlite3_strglob sqlite3_api->strglob
|
||||
/* Version 3.8.11 and later */
|
||||
#define sqlite3_value_dup sqlite3_api->value_dup
|
||||
#define sqlite3_value_free sqlite3_api->value_free
|
||||
#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64
|
||||
#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64
|
||||
/* Version 3.9.0 and later */
|
||||
#define sqlite3_value_subtype sqlite3_api->value_subtype
|
||||
#define sqlite3_result_subtype sqlite3_api->result_subtype
|
||||
/* Version 3.10.0 and later */
|
||||
#define sqlite3_status64 sqlite3_api->status64
|
||||
#define sqlite3_strlike sqlite3_api->strlike
|
||||
#define sqlite3_db_cacheflush sqlite3_api->db_cacheflush
|
||||
/* Version 3.12.0 and later */
|
||||
#define sqlite3_system_errno sqlite3_api->system_errno
|
||||
/* Version 3.14.0 and later */
|
||||
#define sqlite3_trace_v2 sqlite3_api->trace_v2
|
||||
#define sqlite3_expanded_sql sqlite3_api->expanded_sql
|
||||
/* Version 3.18.0 and later */
|
||||
#define sqlite3_set_last_insert_rowid sqlite3_api->set_last_insert_rowid
|
||||
/* Version 3.20.0 and later */
|
||||
#define sqlite3_prepare_v3 sqlite3_api->prepare_v3
|
||||
#define sqlite3_prepare16_v3 sqlite3_api->prepare16_v3
|
||||
#define sqlite3_bind_pointer sqlite3_api->bind_pointer
|
||||
#define sqlite3_result_pointer sqlite3_api->result_pointer
|
||||
#define sqlite3_value_pointer sqlite3_api->value_pointer
|
||||
/* Version 3.22.0 and later */
|
||||
#define sqlite3_vtab_nochange sqlite3_api->vtab_nochange
|
||||
#define sqlite3_value_nochange sqlite3_api->value_nochange
|
||||
#define sqlite3_vtab_collation sqlite3_api->vtab_collation
|
||||
/* Version 3.24.0 and later */
|
||||
#define sqlite3_keyword_count sqlite3_api->keyword_count
|
||||
#define sqlite3_keyword_name sqlite3_api->keyword_name
|
||||
#define sqlite3_keyword_check sqlite3_api->keyword_check
|
||||
#define sqlite3_str_new sqlite3_api->str_new
|
||||
#define sqlite3_str_finish sqlite3_api->str_finish
|
||||
#define sqlite3_str_appendf sqlite3_api->str_appendf
|
||||
#define sqlite3_str_vappendf sqlite3_api->str_vappendf
|
||||
#define sqlite3_str_append sqlite3_api->str_append
|
||||
#define sqlite3_str_appendall sqlite3_api->str_appendall
|
||||
#define sqlite3_str_appendchar sqlite3_api->str_appendchar
|
||||
#define sqlite3_str_reset sqlite3_api->str_reset
|
||||
#define sqlite3_str_errcode sqlite3_api->str_errcode
|
||||
#define sqlite3_str_length sqlite3_api->str_length
|
||||
#define sqlite3_str_value sqlite3_api->str_value
|
||||
/* Version 3.25.0 and later */
|
||||
#define sqlite3_create_window_function sqlite3_api->create_window_function
|
||||
/* Version 3.26.0 and later */
|
||||
#define sqlite3_normalized_sql sqlite3_api->normalized_sql
|
||||
/* Version 3.28.0 and later */
|
||||
#define sqlite3_stmt_isexplain sqlite3_api->stmt_isexplain
|
||||
#define sqlite3_value_frombind sqlite3_api->value_frombind
|
||||
/* Version 3.30.0 and later */
|
||||
#define sqlite3_drop_modules sqlite3_api->drop_modules
|
||||
/* Version 3.31.0 and later */
|
||||
#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64
|
||||
#define sqlite3_uri_key sqlite3_api->uri_key
|
||||
#define sqlite3_filename_database sqlite3_api->filename_database
|
||||
#define sqlite3_filename_journal sqlite3_api->filename_journal
|
||||
#define sqlite3_filename_wal sqlite3_api->filename_wal
|
||||
/* Version 3.32.0 and later */
|
||||
#define sqlite3_create_filename sqlite3_api->create_filename
|
||||
#define sqlite3_free_filename sqlite3_api->free_filename
|
||||
#define sqlite3_database_file_object sqlite3_api->database_file_object
|
||||
/* Version 3.34.0 and later */
|
||||
#define sqlite3_txn_state sqlite3_api->txn_state
|
||||
/* Version 3.36.1 and later */
|
||||
#define sqlite3_changes64 sqlite3_api->changes64
|
||||
#define sqlite3_total_changes64 sqlite3_api->total_changes64
|
||||
/* Version 3.37.0 and later */
|
||||
#define sqlite3_autovacuum_pages sqlite3_api->autovacuum_pages
|
||||
/* Version 3.38.0 and later */
|
||||
#define sqlite3_error_offset sqlite3_api->error_offset
|
||||
#define sqlite3_vtab_rhs_value sqlite3_api->vtab_rhs_value
|
||||
#define sqlite3_vtab_distinct sqlite3_api->vtab_distinct
|
||||
#define sqlite3_vtab_in sqlite3_api->vtab_in
|
||||
#define sqlite3_vtab_in_first sqlite3_api->vtab_in_first
|
||||
#define sqlite3_vtab_in_next sqlite3_api->vtab_in_next
|
||||
/* Version 3.39.0 and later */
|
||||
#ifndef SQLITE_OMIT_DESERIALIZE
|
||||
#define sqlite3_deserialize sqlite3_api->deserialize
|
||||
#define sqlite3_serialize sqlite3_api->serialize
|
||||
#endif
|
||||
#define sqlite3_db_name sqlite3_api->db_name
|
||||
/* Version 3.40.0 and later */
|
||||
#define sqlite3_value_encoding sqlite3_api->value_encoding
|
||||
/* Version 3.41.0 and later */
|
||||
#define sqlite3_is_interrupted sqlite3_api->is_interrupted
|
||||
/* Version 3.43.0 and later */
|
||||
#define sqlite3_stmt_explain sqlite3_api->stmt_explain
|
||||
/* Version 3.44.0 and later */
|
||||
#define sqlite3_get_clientdata sqlite3_api->get_clientdata
|
||||
#define sqlite3_set_clientdata sqlite3_api->set_clientdata
|
||||
/* Version 3.50.0 and later */
|
||||
#define sqlite3_setlk_timeout sqlite3_api->setlk_timeout
|
||||
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
|
||||
|
||||
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
|
||||
/* This case when the file really is being compiled as a loadable
|
||||
** extension */
|
||||
# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0;
|
||||
# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v;
|
||||
# define SQLITE_EXTENSION_INIT3 \
|
||||
extern const sqlite3_api_routines *sqlite3_api;
|
||||
#else
|
||||
/* This case when the file is being statically linked into the
|
||||
** application */
|
||||
# define SQLITE_EXTENSION_INIT1 /*no-op*/
|
||||
# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */
|
||||
# define SQLITE_EXTENSION_INIT3 /*no-op*/
|
||||
#endif
|
||||
|
||||
#endif /* SQLITE3EXT_H */
|
||||
@@ -1,8 +1,8 @@
|
||||
name: example-plugin
|
||||
name: plugin-llm-example
|
||||
repo: https://github.com/NousResearch/hermes-example-plugins
|
||||
sha: 38fe0fb53eff98d477f807432e965429e665ca33
|
||||
subdir: ""
|
||||
description: Reference example plugins for the Hermes plugin system.
|
||||
subdir: "plugin-llm-example"
|
||||
description: Reference plugin showing host-owned structured LLM access via ctx.llm.
|
||||
maintainer: NousResearch
|
||||
tier: official
|
||||
docs_url: ""
|
||||
@@ -1037,6 +1037,7 @@ class DiscordAdapter(BasePlatformAdapter):
|
||||
"""Connect to Discord and start receiving events."""
|
||||
if not DISCORD_AVAILABLE:
|
||||
logger.error("[%s] discord.py not installed. Run: pip install discord.py", self.name)
|
||||
self._set_fatal_error("missing_dependency", "discord.py not installed", retryable=False)
|
||||
return False
|
||||
|
||||
# Load opus codec for voice channel support
|
||||
@@ -1073,6 +1074,7 @@ class DiscordAdapter(BasePlatformAdapter):
|
||||
|
||||
if not self.config.token:
|
||||
logger.error("[%s] No bot token configured", self.name)
|
||||
self._set_fatal_error("missing_credentials", "No bot token configured", retryable=False)
|
||||
return False
|
||||
|
||||
try:
|
||||
|
||||
+952
-149
File diff suppressed because it is too large
Load Diff
@@ -3434,10 +3434,12 @@ class TelegramAdapter(BasePlatformAdapter):
|
||||
"[%s] python-telegram-bot not installed. Run: pip install python-telegram-bot",
|
||||
self.name,
|
||||
)
|
||||
self._set_fatal_error("missing_dependency", "python-telegram-bot not installed", retryable=False)
|
||||
return False
|
||||
|
||||
if not self.config.token:
|
||||
logger.error("[%s] No bot token configured", self.name)
|
||||
self._set_fatal_error("missing_credentials", "No bot token configured", retryable=False)
|
||||
return False
|
||||
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Standalone structural validator for plugin-catalog entry files.
|
||||
|
||||
Validates ``plugin-catalog/*.yaml`` catalog entries and
|
||||
``plugin-catalog/removed.yaml`` against the catalog contract schema, using
|
||||
only stdlib + PyYAML so the admission CI (and third-party repos) can run it
|
||||
WITHOUT installing hermes-agent.
|
||||
|
||||
NOTE: this script intentionally duplicates the schema rules instead of
|
||||
importing ``hermes_cli`` — the whole point is the no-install requirement for
|
||||
cheap cross-repo CI use. The runtime twin of this schema lives in
|
||||
``hermes_cli/plugin_catalog.py``; if the contract changes there, update the
|
||||
rules here in lockstep.
|
||||
|
||||
Usage:
|
||||
python3 scripts/validate_plugin_catalog.py plugin-catalog/
|
||||
python3 scripts/validate_plugin_catalog.py entry.yaml removed.yaml
|
||||
python3 scripts/validate_plugin_catalog.py --json plugin-catalog/
|
||||
|
||||
Exit codes: 0 = all files valid (warnings allowed), 1 = at least one error.
|
||||
Human output is one ``<file>: ERROR: ...`` / ``<file>: warning: ...`` line
|
||||
per finding; ``--json`` emits a machine-readable report on stdout instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover - dependency guidance only
|
||||
print(
|
||||
"ERROR: PyYAML is required (pip install pyyaml)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
NAME_RE = re.compile(r"^[a-z0-9_-]{1,64}$")
|
||||
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
TIERS = ("official", "community")
|
||||
PLATFORMS = ("linux", "macos", "windows")
|
||||
CAPABILITY_KEYS = (
|
||||
"provides_tools",
|
||||
"provides_hooks",
|
||||
"provides_middleware",
|
||||
"requires_env",
|
||||
)
|
||||
# Top-level keys the contract knows about. Unknown keys WARN (forward
|
||||
# compatibility: newer catalogs must stay valid under older validators).
|
||||
KNOWN_KEYS = {
|
||||
"name",
|
||||
"repo",
|
||||
"sha",
|
||||
"subdir",
|
||||
"description",
|
||||
"maintainer",
|
||||
"tier",
|
||||
"requires_hermes",
|
||||
"docs_url",
|
||||
"platforms",
|
||||
"capabilities",
|
||||
}
|
||||
REQUIRED_KEYS = ("name", "repo", "sha", "description", "maintainer")
|
||||
|
||||
# One comparator clause of a requires_hermes spec, e.g. ">=0.19" or "!=1.2.3".
|
||||
_COMPARATOR_RE = re.compile(r"^(>=|<=|==|!=|>|<)\s*\d+(\.\d+)*$")
|
||||
|
||||
|
||||
def _is_nonempty_str(value: object) -> bool:
|
||||
return isinstance(value, str) and value.strip() != ""
|
||||
|
||||
|
||||
def _check_requires_hermes(spec: object, errors: list[str]) -> None:
|
||||
if not isinstance(spec, str):
|
||||
errors.append(f"requires_hermes must be a string, got {type(spec).__name__}")
|
||||
return
|
||||
if spec.strip() == "":
|
||||
return # empty = no constraint
|
||||
for clause in spec.split(","):
|
||||
if not _COMPARATOR_RE.match(clause.strip()):
|
||||
errors.append(
|
||||
f"requires_hermes clause {clause.strip()!r} is not a valid "
|
||||
"comparator spec (expected e.g. '>=0.19')"
|
||||
)
|
||||
|
||||
|
||||
def validate_entry(data: object) -> tuple[list[str], list[str]]:
|
||||
"""Validate one catalog entry document. Returns (errors, warnings)."""
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return ["top-level document must be a YAML mapping"], warnings
|
||||
|
||||
for key in REQUIRED_KEYS:
|
||||
if key not in data:
|
||||
errors.append(f"missing required key: {key}")
|
||||
|
||||
for key in sorted(set(data) - KNOWN_KEYS):
|
||||
warnings.append(f"unknown top-level key {key!r} (ignored by this validator)")
|
||||
|
||||
name = data.get("name")
|
||||
if "name" in data and (not isinstance(name, str) or not NAME_RE.match(name)):
|
||||
errors.append(f"name {name!r} must match [a-z0-9_-]{{1,64}}")
|
||||
|
||||
repo = data.get("repo")
|
||||
if "repo" in data and (
|
||||
not isinstance(repo, str) or not repo.startswith("https://")
|
||||
):
|
||||
errors.append(f"repo {repo!r} must be an https:// URL")
|
||||
|
||||
sha = data.get("sha")
|
||||
if "sha" in data and (not isinstance(sha, str) or not SHA_RE.match(sha)):
|
||||
errors.append(f"sha {sha!r} must be exactly 40 lowercase hex characters")
|
||||
|
||||
for key in ("description", "maintainer"):
|
||||
if key in data and not _is_nonempty_str(data[key]):
|
||||
errors.append(f"{key} must be a non-empty string")
|
||||
|
||||
tier = data.get("tier", "community")
|
||||
if tier not in TIERS:
|
||||
errors.append(f"tier {tier!r} must be one of {list(TIERS)}")
|
||||
|
||||
if "requires_hermes" in data:
|
||||
_check_requires_hermes(data["requires_hermes"], errors)
|
||||
|
||||
platforms = data.get("platforms", [])
|
||||
if platforms is None:
|
||||
platforms = []
|
||||
if not isinstance(platforms, list):
|
||||
errors.append("platforms must be a list")
|
||||
else:
|
||||
bad = [p for p in platforms if p not in PLATFORMS]
|
||||
if bad:
|
||||
errors.append(f"platforms {bad!r} not in allowed set {list(PLATFORMS)}")
|
||||
|
||||
caps = data.get("capabilities", {})
|
||||
if caps is None:
|
||||
caps = {}
|
||||
if not isinstance(caps, dict):
|
||||
errors.append("capabilities must be a mapping")
|
||||
else:
|
||||
for key in sorted(set(caps) - set(CAPABILITY_KEYS)):
|
||||
warnings.append(f"unknown capabilities key {key!r}")
|
||||
for key in CAPABILITY_KEYS:
|
||||
if key not in caps:
|
||||
continue
|
||||
value = caps[key]
|
||||
if not isinstance(value, list) or not all(
|
||||
isinstance(item, str) for item in value
|
||||
):
|
||||
errors.append(f"capabilities.{key} must be a list of strings")
|
||||
|
||||
return errors, warnings
|
||||
|
||||
|
||||
def validate_removed(data: object) -> tuple[list[str], list[str]]:
|
||||
"""Validate the removed.yaml document. Returns (errors, warnings)."""
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return ["top-level document must be a YAML mapping"], warnings
|
||||
|
||||
removed = data.get("removed")
|
||||
if removed is None:
|
||||
errors.append("missing required key: removed")
|
||||
return errors, warnings
|
||||
if not isinstance(removed, list):
|
||||
errors.append("removed must be a list")
|
||||
return errors, warnings
|
||||
|
||||
for i, item in enumerate(removed):
|
||||
if not isinstance(item, dict):
|
||||
errors.append(f"removed[{i}] must be a mapping")
|
||||
continue
|
||||
if not _is_nonempty_str(item.get("name")):
|
||||
errors.append(f"removed[{i}] missing non-empty 'name'")
|
||||
for key in ("repo", "reason", "date"):
|
||||
if key in item and not isinstance(item[key], str):
|
||||
errors.append(f"removed[{i}].{key} must be a string")
|
||||
|
||||
return errors, warnings
|
||||
|
||||
|
||||
def validate_file(path: Path) -> tuple[list[str], list[str]]:
|
||||
"""Validate one YAML file (dispatching on filename). Returns (errors, warnings)."""
|
||||
try:
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
except OSError as exc:
|
||||
return [f"cannot read file: {exc}"], []
|
||||
except yaml.YAMLError as exc:
|
||||
return [f"invalid YAML: {exc}"], []
|
||||
|
||||
if path.name == "removed.yaml":
|
||||
return validate_removed(data)
|
||||
return validate_entry(data)
|
||||
|
||||
|
||||
def collect_paths(args: list[str]) -> list[Path]:
|
||||
paths: list[Path] = []
|
||||
for arg in args:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
paths.extend(sorted(p.glob("*.yaml")))
|
||||
paths.extend(sorted(p.glob("*.yml")))
|
||||
else:
|
||||
paths.append(p)
|
||||
return paths
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Standalone structural validator for plugin-catalog entry files."
|
||||
)
|
||||
parser.add_argument(
|
||||
"paths",
|
||||
nargs="+",
|
||||
help="catalog entry files, removed.yaml, or a directory of them",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="emit a machine-readable JSON report on stdout",
|
||||
)
|
||||
opts = parser.parse_args(argv)
|
||||
|
||||
files = collect_paths(opts.paths)
|
||||
if not files:
|
||||
print("ERROR: no YAML files found", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
report = []
|
||||
any_errors = False
|
||||
for path in files:
|
||||
errors, warnings = validate_file(path)
|
||||
any_errors = any_errors or bool(errors)
|
||||
report.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"ok": not errors,
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
}
|
||||
)
|
||||
|
||||
if opts.json:
|
||||
print(json.dumps({"ok": not any_errors, "files": report}, indent=2))
|
||||
else:
|
||||
for entry in report:
|
||||
for err in entry["errors"]:
|
||||
print(f"{entry['path']}: ERROR: {err}")
|
||||
for warn in entry["warnings"]:
|
||||
print(f"{entry['path']}: warning: {warn}")
|
||||
checked = len(report)
|
||||
bad = sum(1 for e in report if not e["ok"])
|
||||
if any_errors:
|
||||
print(f"FAIL: {bad}/{checked} file(s) invalid")
|
||||
else:
|
||||
print(f"OK: {checked} file(s) valid")
|
||||
|
||||
return 1 if any_errors else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Strict redaction at every compaction text boundary (issue #43666 item 2).
|
||||
|
||||
Compaction summaries persist across sessions and re-enter every subsequent
|
||||
summarizer prompt, so ``_redact_compaction_text()`` applies strict mode
|
||||
(``force=True, redact_url_credentials=True``) at each boundary:
|
||||
|
||||
- serializer input (``_serialize_for_summary``: message content + tool args)
|
||||
- deterministic fallback summary (``_build_static_fallback_summary``)
|
||||
- summarizer LLM output (``_generate_summary`` return / ``_previous_summary``)
|
||||
- focus text (manual ``/compress <focus>`` and ``_derive_auto_focus_topic``)
|
||||
- previous-summary re-entry into the iterative-update prompt
|
||||
|
||||
Every test disables the global redaction flag (simulating
|
||||
``security.redact_secrets: false``) to prove ``force=True`` still redacts at
|
||||
the persistence boundary, and uses an OAuth-callback-style URL to prove
|
||||
``redact_url_credentials=True`` strips opaque URL tokens that default-mode
|
||||
redaction deliberately passes through.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.context_compressor import (
|
||||
ContextCompressor,
|
||||
SUMMARY_PREFIX,
|
||||
_redact_compaction_text,
|
||||
)
|
||||
|
||||
SECRET = "sk-proj-" + ("a" * 40)
|
||||
OAUTH_URL = (
|
||||
"https://localhost/callback?code=opaque-code-123"
|
||||
"&access_token=opaque-token-456&state=keep"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _redaction_globally_disabled(monkeypatch):
|
||||
"""Simulate security.redact_secrets: false — force=True must still win."""
|
||||
monkeypatch.setattr("agent.redact._REDACT_ENABLED", False)
|
||||
|
||||
|
||||
def _compressor() -> ContextCompressor:
|
||||
with patch(
|
||||
"agent.context_compressor.get_model_context_length",
|
||||
return_value=100000,
|
||||
):
|
||||
return ContextCompressor(model="test/model", quiet_mode=True)
|
||||
|
||||
|
||||
def _response(content: str):
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = content
|
||||
return mock_response
|
||||
|
||||
|
||||
def _assert_clean(text: str):
|
||||
assert SECRET not in text
|
||||
assert "sk-proj-" not in text
|
||||
assert "code=opaque-code-123" not in text
|
||||
assert "access_token=opaque-token-456" not in text
|
||||
assert "code=***" in text
|
||||
assert "access_token=***" in text
|
||||
assert "state=keep" in text
|
||||
|
||||
|
||||
def test_helper_is_strict_even_when_redaction_disabled():
|
||||
result = _redact_compaction_text(f"key {SECRET} url {OAUTH_URL}")
|
||||
_assert_clean(result)
|
||||
# None-safety: helper is used on optional fields.
|
||||
assert _redact_compaction_text(None) == ""
|
||||
|
||||
|
||||
def test_serializer_input_redacts_content_and_tool_args():
|
||||
c = _compressor()
|
||||
messages = [
|
||||
{"role": "user", "content": f"token {SECRET} url {OAUTH_URL}"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"function": {
|
||||
"name": "terminal",
|
||||
"arguments": (
|
||||
f'{{"command": "curl {OAUTH_URL}",'
|
||||
f' "note": "{SECRET}"}}'
|
||||
),
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call-1", "content": f"got {SECRET}"},
|
||||
]
|
||||
|
||||
serialized = c._serialize_for_summary(messages)
|
||||
|
||||
_assert_clean(serialized)
|
||||
|
||||
|
||||
def test_fallback_summary_redacts_secrets():
|
||||
c = _compressor()
|
||||
turns = [
|
||||
{"role": "user", "content": f"deploy with {SECRET} via {OAUTH_URL}"},
|
||||
{"role": "assistant", "content": f"ran curl {OAUTH_URL}"},
|
||||
]
|
||||
|
||||
summary = c._build_static_fallback_summary(turns, reason="test outage")
|
||||
|
||||
_assert_clean(summary)
|
||||
|
||||
|
||||
def test_summary_output_redacts_llm_echoed_secrets():
|
||||
c = _compressor()
|
||||
leaked = f"Summary leaked OPENAI_API_KEY {SECRET} and {OAUTH_URL}"
|
||||
|
||||
with patch(
|
||||
"agent.context_compressor.call_llm", return_value=_response(leaked)
|
||||
):
|
||||
summary = c._generate_summary([{"role": "user", "content": "hi"}])
|
||||
|
||||
assert summary is not None
|
||||
_assert_clean(summary)
|
||||
# The stored iterative-update seed must be clean too.
|
||||
_assert_clean(c._previous_summary)
|
||||
|
||||
|
||||
def test_manual_focus_topic_redacted_before_summary_prompt():
|
||||
c = _compressor()
|
||||
turns = [
|
||||
{"role": "user", "content": "Summarize safely"},
|
||||
{"role": "assistant", "content": "OK"},
|
||||
]
|
||||
|
||||
with patch(
|
||||
"agent.context_compressor.call_llm",
|
||||
return_value=_response("## Goal\nSafe summary."),
|
||||
) as mock_call:
|
||||
result = c._generate_summary(
|
||||
turns, focus_topic=f"manual focus {SECRET} {OAUTH_URL}"
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
|
||||
_assert_clean(prompt)
|
||||
|
||||
|
||||
def test_auto_focus_topic_redacted():
|
||||
c = _compressor()
|
||||
|
||||
focus = c._derive_auto_focus_topic(
|
||||
[
|
||||
{"role": "assistant", "content": "older assistant turn"},
|
||||
{"role": "user", "content": f"focus has {SECRET} and {OAUTH_URL}"},
|
||||
]
|
||||
)
|
||||
|
||||
assert focus is not None
|
||||
_assert_clean(focus)
|
||||
|
||||
|
||||
def test_previous_summary_redacted_before_iterative_prompt_reentry():
|
||||
"""Legacy persisted summaries may predate compaction redaction."""
|
||||
c = _compressor()
|
||||
c._previous_summary = f"Old summary leaked {SECRET} and {OAUTH_URL}"
|
||||
|
||||
with patch(
|
||||
"agent.context_compressor.call_llm",
|
||||
return_value=_response("updated summary"),
|
||||
) as mock_call:
|
||||
result = c._generate_summary(
|
||||
[
|
||||
{"role": "user", "content": "new turn"},
|
||||
{"role": "assistant", "content": "new work"},
|
||||
]
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
|
||||
assert "PREVIOUS SUMMARY:" in prompt
|
||||
_assert_clean(prompt)
|
||||
# After generation, _previous_summary holds the new (clean) LLM output —
|
||||
# the leaked secret must not have survived anywhere in it.
|
||||
assert SECRET not in c._previous_summary
|
||||
assert "access_token=opaque-token-456" not in c._previous_summary
|
||||
|
||||
|
||||
def test_resumed_handoff_summary_redacted_before_iterative_prompt():
|
||||
"""Persisted handoff messages may contain pre-fix secrets after resume."""
|
||||
with patch(
|
||||
"agent.context_compressor.get_model_context_length",
|
||||
return_value=100000,
|
||||
):
|
||||
c = ContextCompressor(
|
||||
model="test/model",
|
||||
threshold_percent=0.85,
|
||||
protect_first_n=1,
|
||||
protect_last_n=1,
|
||||
quiet_mode=True,
|
||||
)
|
||||
old_summary = f"RESUMED-SUMMARY leaked {SECRET} and {OAUTH_URL}"
|
||||
messages = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": f"{SUMMARY_PREFIX}\n{old_summary}"},
|
||||
{"role": "assistant", "content": "handoff acknowledged after resume"},
|
||||
{"role": "user", "content": "new user turn after resume"},
|
||||
{"role": "assistant", "content": "new assistant work after resume"},
|
||||
{"role": "user", "content": "more new work after resume"},
|
||||
{"role": "assistant", "content": "latest tail response"},
|
||||
{"role": "user", "content": "final active request stays in tail"},
|
||||
]
|
||||
|
||||
with patch(
|
||||
"agent.context_compressor.call_llm",
|
||||
return_value=_response("updated summary"),
|
||||
) as mock_call:
|
||||
c.compress(messages)
|
||||
|
||||
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
|
||||
assert "PREVIOUS SUMMARY:" in prompt
|
||||
_assert_clean(prompt)
|
||||
@@ -0,0 +1,151 @@
|
||||
import json
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from agent.conversation_compression import compress_context
|
||||
from agent.context_compressor import ContextCompressor
|
||||
|
||||
|
||||
class _TodoStore:
|
||||
def format_for_injection(self):
|
||||
return ""
|
||||
|
||||
|
||||
class _Agent:
|
||||
def __init__(self, compressor):
|
||||
self.context_compressor = compressor
|
||||
self.session_id = "session-telemetry-test"
|
||||
self.platform = "cli"
|
||||
self.model = "test/main-model"
|
||||
self.provider = "test-provider"
|
||||
self.tools = []
|
||||
self._compression_feasibility_checked = True
|
||||
self.compression_in_place = False
|
||||
self._memory_manager = None
|
||||
self._session_db = None
|
||||
self._todo_store = _TodoStore()
|
||||
self._cached_system_prompt = None
|
||||
|
||||
def _emit_status(self, _message):
|
||||
pass
|
||||
|
||||
def _emit_warning(self, _message):
|
||||
pass
|
||||
|
||||
def _invalidate_system_prompt(self):
|
||||
self._cached_system_prompt = None
|
||||
|
||||
def _build_system_prompt(self, system_message):
|
||||
return system_message
|
||||
|
||||
def commit_memory_session(self, _messages):
|
||||
pass
|
||||
|
||||
|
||||
def _messages(secret_text="TOPSECRET_TRANSCRIPT_TEXT"):
|
||||
msgs = [{"role": "system", "content": "system prompt"}]
|
||||
for idx in range(10):
|
||||
msgs.append({"role": "user", "content": f"user message {idx} {secret_text}"})
|
||||
msgs.append({"role": "assistant", "content": f"assistant reply {idx} {secret_text}"})
|
||||
return msgs
|
||||
|
||||
|
||||
def _extract_telemetry(caplog):
|
||||
records = [
|
||||
record.getMessage()
|
||||
for record in caplog.records
|
||||
if "context compression attempt telemetry:" in record.getMessage()
|
||||
]
|
||||
assert len(records) == 1
|
||||
return json.loads(records[0].split("context compression attempt telemetry: ", 1)[1])
|
||||
|
||||
|
||||
def test_compression_attempt_telemetry_is_metadata_only(caplog):
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
compressor = ContextCompressor(
|
||||
model="test/main-model",
|
||||
provider="test-provider",
|
||||
threshold_percent=0.50,
|
||||
quiet_mode=True,
|
||||
config_context_length=100_000,
|
||||
)
|
||||
compressor.tail_token_budget = 10
|
||||
agent = _Agent(compressor)
|
||||
|
||||
with patch.object(compressor, "_generate_summary", return_value="SANITIZED SUMMARY"):
|
||||
with caplog.at_level(logging.INFO, logger="agent.conversation_compression"):
|
||||
compressed, system_prompt = compress_context(
|
||||
agent,
|
||||
_messages(),
|
||||
"system prompt",
|
||||
approx_tokens=75_000,
|
||||
force=True,
|
||||
)
|
||||
|
||||
assert system_prompt == "system prompt"
|
||||
assert compressed is not None
|
||||
payload = _extract_telemetry(caplog)
|
||||
|
||||
assert payload["event"] == "compression_attempt"
|
||||
assert payload["attempt_id"]
|
||||
assert payload["session_id"] == "session-telemetry-test"
|
||||
assert payload["trigger_source"] == "manual"
|
||||
assert payload["main_model"] == "test/main-model"
|
||||
assert payload["main_context_limit"] == 100_000
|
||||
assert payload["current_estimated_tokens"] == 75_000
|
||||
assert payload["effective_threshold"] == compressor.threshold_tokens
|
||||
assert payload["protected_head_tokens"] is not None
|
||||
assert payload["protected_tail_tokens"] is not None
|
||||
assert payload["middle_window_tokens"] is not None
|
||||
assert payload["chunking"] is False
|
||||
assert payload["chunk_count"] in {0, 1}
|
||||
assert payload["commit_status"] == "committed"
|
||||
assert payload["split_status"] == "not_applicable"
|
||||
assert payload["fallback_used"] is False
|
||||
assert isinstance(payload["total_duration_ms"], int)
|
||||
|
||||
raw_log = json.dumps(payload)
|
||||
assert "TOPSECRET_TRANSCRIPT_TEXT" not in raw_log
|
||||
assert "SANITIZED SUMMARY" not in raw_log
|
||||
assert "user message" not in raw_log
|
||||
assert "assistant reply" not in raw_log
|
||||
|
||||
|
||||
def test_aux_call_telemetry_records_durations_without_content(caplog):
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
compressor = ContextCompressor(
|
||||
model="test/main-model",
|
||||
provider="test-provider",
|
||||
threshold_percent=0.50,
|
||||
quiet_mode=True,
|
||||
config_context_length=100_000,
|
||||
)
|
||||
compressor.tail_token_budget = 10
|
||||
agent = _Agent(compressor)
|
||||
response = SimpleNamespace(
|
||||
choices=[SimpleNamespace(message=SimpleNamespace(content="SANITIZED SUMMARY"))]
|
||||
)
|
||||
|
||||
with patch("agent.context_compressor.call_llm", return_value=response):
|
||||
with caplog.at_level(logging.INFO, logger="agent.conversation_compression"):
|
||||
compress_context(
|
||||
agent,
|
||||
_messages(),
|
||||
"system prompt",
|
||||
approx_tokens=75_000,
|
||||
)
|
||||
|
||||
payload = _extract_telemetry(caplog)
|
||||
assert payload["aux_prompt_tokens"] is not None
|
||||
# Current main intentionally omits max_tokens from the aux summary call
|
||||
# (the summary budget is prompt-level guidance only), so no output
|
||||
# reservation is recorded.
|
||||
assert payload["aux_output_reservation"] is None
|
||||
assert isinstance(payload["aux_call_duration_ms"], int)
|
||||
assert payload["aux_provider"]
|
||||
assert payload["aux_model"]
|
||||
|
||||
raw_log = json.dumps(payload)
|
||||
assert "TOPSECRET_TRANSCRIPT_TEXT" not in raw_log
|
||||
assert "SANITIZED SUMMARY" not in raw_log
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Regression tests for blank user echoes displacing actionable compaction state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.context_compressor import (
|
||||
COMPRESSED_SUMMARY_METADATA_KEY,
|
||||
SUMMARY_PREFIX,
|
||||
ContextCompressor,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def compressor() -> ContextCompressor:
|
||||
with patch(
|
||||
"agent.context_compressor.get_model_context_length",
|
||||
return_value=100_000,
|
||||
):
|
||||
instance = ContextCompressor(
|
||||
model="test/model",
|
||||
threshold_percent=0.85,
|
||||
protect_first_n=2,
|
||||
protect_last_n=2,
|
||||
quiet_mode=True,
|
||||
)
|
||||
instance.tail_token_budget = 10
|
||||
return instance
|
||||
|
||||
|
||||
def _append_tool_run(messages: list[dict], prefix: str, count: int = 6) -> None:
|
||||
for index in range(count):
|
||||
call_id = f"{prefix}-{index}"
|
||||
messages.extend(
|
||||
[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": call_id,
|
||||
"function": {"name": "read_file", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": call_id,
|
||||
"content": "x" * 400,
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _compress(compressor: ContextCompressor, messages: list[dict]) -> list[dict]:
|
||||
with patch.object(
|
||||
compressor,
|
||||
"_generate_summary",
|
||||
return_value=f"{SUMMARY_PREFIX}\nsummary of older work",
|
||||
):
|
||||
return compressor.compress(messages, current_tokens=90_000)
|
||||
|
||||
|
||||
def _assert_no_adjacent_user_roles(messages: list[dict]) -> None:
|
||||
for previous, current in zip(messages, messages[1:]):
|
||||
assert (previous.get("role"), current.get("role")) != ("user", "user")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"blank",
|
||||
[
|
||||
"",
|
||||
" \n\t",
|
||||
None,
|
||||
[],
|
||||
[{"type": "text", "text": " "}],
|
||||
[{"type": "input_text", "text": " "}],
|
||||
],
|
||||
)
|
||||
def test_blank_echo_does_not_displace_async_completion(compressor, blank):
|
||||
completion = "[ASYNC DELEGATION BATCH COMPLETE — deleg_current]\nnew result"
|
||||
messages = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "old request"},
|
||||
{"role": "assistant", "content": "old reply"},
|
||||
{"role": "user", "content": completion},
|
||||
{"role": "user", "content": blank},
|
||||
{"role": "assistant", "content": "working from the completion"},
|
||||
]
|
||||
|
||||
assert compressor._find_last_user_message_idx(messages, head_end=1) == 3
|
||||
|
||||
|
||||
def test_leading_blank_without_actionable_user_is_not_removed(compressor):
|
||||
messages = [
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "assistant", "content": "visible reply"},
|
||||
]
|
||||
|
||||
assert compressor._find_last_user_message_idx(messages, 0) == -1
|
||||
assert compressor._blank_echo_indices_after(messages, -1) == set()
|
||||
|
||||
|
||||
def test_image_only_user_turn_survives_compaction(compressor):
|
||||
image_content = [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,AA=="},
|
||||
}
|
||||
]
|
||||
messages: list[dict] = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "old request"},
|
||||
{"role": "assistant", "content": "old reply"},
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": f"older question {index}"}
|
||||
if index % 2 == 0
|
||||
else {"role": "assistant", "content": f"older reply {index}"}
|
||||
for index in range(6)
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": image_content},
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "assistant", "content": "analyzing the image"},
|
||||
]
|
||||
_append_tool_run(messages, "image")
|
||||
|
||||
result = _compress(compressor, messages)
|
||||
|
||||
assert any(message.get("content") == image_content for message in result)
|
||||
assert all(not compressor._is_blank_user_turn(message) for message in result)
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
[{"type": "audio", "source": {"data": "AA=="}}],
|
||||
[{"type": "input_audio", "input_audio": {"data": "AA=="}}],
|
||||
[{"type": "future_input", "payload": {"value": 7}}],
|
||||
],
|
||||
ids=["audio", "input-audio", "unknown-structured"],
|
||||
)
|
||||
def test_structured_non_text_user_turn_survives_compaction(compressor, payload):
|
||||
messages: list[dict] = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "old request"},
|
||||
{"role": "assistant", "content": "old reply"},
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": f"older question {index}"}
|
||||
if index % 2 == 0
|
||||
else {"role": "assistant", "content": f"older reply {index}"}
|
||||
for index in range(6)
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": payload},
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "assistant", "content": "processing structured input"},
|
||||
]
|
||||
_append_tool_run(messages, "structured")
|
||||
|
||||
result = _compress(compressor, messages)
|
||||
|
||||
assert any(message.get("content") == payload for message in result)
|
||||
assert all(not compressor._is_blank_user_turn(message) for message in result)
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
|
||||
|
||||
def test_completion_survives_compaction_verbatim_after_blank_echo(compressor):
|
||||
completion = (
|
||||
"[ASYNC DELEGATION BATCH COMPLETE — deleg_current]\n"
|
||||
"The newest result that must remain actionable."
|
||||
)
|
||||
messages: list[dict] = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "initial request"},
|
||||
{"role": "assistant", "content": "initial reply"},
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": f"older question {index}"}
|
||||
if index % 2 == 0
|
||||
else {"role": "assistant", "content": f"older reply {index}"}
|
||||
for index in range(6)
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": completion},
|
||||
{"role": "user", "content": " \n"},
|
||||
{"role": "assistant", "content": "working from the completion"},
|
||||
]
|
||||
_append_tool_run(messages, "tail")
|
||||
|
||||
result = _compress(compressor, messages)
|
||||
|
||||
completion_rows = [message for message in result if message.get("content") == completion]
|
||||
assert len(completion_rows) == 1
|
||||
assert not completion_rows[0].get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
summary_rows = [
|
||||
message for message in result if message.get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
]
|
||||
assert len(summary_rows) == 1
|
||||
assert summary_rows[0].get("role") == "user"
|
||||
assert all(not compressor._is_blank_user_turn(message) for message in result)
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
|
||||
second_result = _compress(compressor, result)
|
||||
second_completion_rows = [
|
||||
message for message in second_result if message.get("content") == completion
|
||||
]
|
||||
assert len(second_completion_rows) == 1
|
||||
assert not second_completion_rows[0].get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
|
||||
|
||||
def test_completion_at_compress_start_survives_when_blank_echo_is_compress_end(
|
||||
compressor,
|
||||
):
|
||||
completion = "latest actionable completion at the compression boundary"
|
||||
messages = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "initial request"},
|
||||
{"role": "assistant", "content": "initial reply"},
|
||||
{"role": "user", "content": completion},
|
||||
{"role": "user", "content": ""},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "boundary-call",
|
||||
"function": {"name": "read_file", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "boundary-call", "content": "result"},
|
||||
{"role": "assistant", "content": "working from the completion"},
|
||||
]
|
||||
|
||||
with (
|
||||
patch.object(compressor, "_protect_head_size", return_value=3),
|
||||
patch.object(compressor, "_find_tail_cut_by_tokens", return_value=3),
|
||||
patch.object(compressor, "_generate_summary") as generate_summary,
|
||||
):
|
||||
result = compressor.compress(messages, current_tokens=90_000)
|
||||
|
||||
completion_rows = [message for message in result if message.get("content") == completion]
|
||||
assert len(completion_rows) == 1
|
||||
assert not completion_rows[0].get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
assert not any(
|
||||
message.get(COMPRESSED_SUMMARY_METADATA_KEY) for message in result
|
||||
)
|
||||
assert len(result) == len(messages) - 1
|
||||
assert compressor.compression_count == 0
|
||||
assert compressor._last_compression_savings_pct == 0.0
|
||||
generate_summary.assert_not_called()
|
||||
assert any(message.get("tool_call_id") == "boundary-call" for message in result)
|
||||
assert result[-1].get("content") == "working from the completion"
|
||||
assert [message.get("role") for message in result] == [
|
||||
"system",
|
||||
"user",
|
||||
"assistant",
|
||||
"user",
|
||||
"assistant",
|
||||
"tool",
|
||||
"assistant",
|
||||
]
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
|
||||
|
||||
def test_tool_call_head_compacts_without_rewriting_event(compressor):
|
||||
completion = "latest actionable completion"
|
||||
messages: list[dict] = [
|
||||
{"role": "user", "content": "initial request"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "head-call",
|
||||
"function": {"name": "read_file", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "head-call", "content": "head result"},
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": f"older question {index}"}
|
||||
if index % 2 == 0
|
||||
else {"role": "assistant", "content": f"older reply {index}"}
|
||||
for index in range(6)
|
||||
]
|
||||
messages += [
|
||||
{"role": "user", "content": completion},
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "assistant", "content": "working"},
|
||||
]
|
||||
_append_tool_run(messages, "tail")
|
||||
|
||||
result = _compress(compressor, messages)
|
||||
|
||||
assert compressor._last_compress_aborted is False
|
||||
assert any(message.get("content") == completion for message in result)
|
||||
head = next(
|
||||
message
|
||||
for message in result
|
||||
if any(call.get("id") == "head-call" for call in message.get("tool_calls", []))
|
||||
)
|
||||
assert not head.get(COMPRESSED_SUMMARY_METADATA_KEY)
|
||||
assert any(message.get("tool_call_id") == "head-call" for message in result)
|
||||
_assert_no_adjacent_user_roles(result)
|
||||
@@ -245,7 +245,9 @@ class TestCompress:
|
||||
assert "Summary generation was unavailable" in combined
|
||||
assert "removed to free context space but could not be summarized" not in combined
|
||||
assert c._last_summary_fallback_used is True
|
||||
assert c._last_summary_dropped_count == 3
|
||||
# The assistant immediately before the latest actionable user turn is
|
||||
# retained as a role bridge, so only the two genuinely older rows drop.
|
||||
assert c._last_summary_dropped_count == 2
|
||||
|
||||
def test_fallback_summary_does_not_triplicate_latest_user_ask(self):
|
||||
"""Regression for #49307: the deterministic fallback summary used to
|
||||
@@ -3165,6 +3167,207 @@ class TestUpdateModelResetsCalibration:
|
||||
assert comp._summary_failure_cooldown_until == cooldown_until
|
||||
|
||||
|
||||
class TestThresholdTokensCap:
|
||||
"""Tests for the absolute token cap (compression.threshold_tokens).
|
||||
|
||||
The cap takes the lower of the ratio-based threshold and the absolute
|
||||
count. It must survive model switches (update_model re-applies it)
|
||||
and be clamped to the model's context length.
|
||||
"""
|
||||
|
||||
def test_cap_lower_than_ratio_uses_cap(self):
|
||||
"""When the cap is lower than the ratio-based threshold, the cap wins."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=50_000,
|
||||
)
|
||||
# Ratio-based: 200000 * 0.50 = 100000. Cap: 50000. Effective: 50000.
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
def test_cap_higher_than_ratio_uses_ratio(self):
|
||||
"""When the cap is higher than the ratio-based threshold, the ratio wins."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=2_000_000,
|
||||
)
|
||||
# Ratio-based: 1000000 * 0.50 = 500000. Cap: 2000000, clamped to 1000000.
|
||||
# Effective: min(500000, 1000000) = 500000.
|
||||
assert comp.threshold_tokens == 500_000
|
||||
|
||||
def test_no_cap_uses_ratio_only(self):
|
||||
"""Without a cap, the ratio-based threshold is used."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
)
|
||||
assert comp.threshold_tokens == 500_000
|
||||
assert comp.threshold_tokens_cap is None
|
||||
|
||||
def test_cap_survives_model_switch(self):
|
||||
"""The cap must be re-applied after update_model() switches to a
|
||||
different context length. This is the core sweeper feedback: the
|
||||
old PR's post-construction patch was undone by update_model()
|
||||
restoring _configured_threshold_percent."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=40_000,
|
||||
)
|
||||
assert comp.threshold_tokens == 40_000 # cap wins on 200K model
|
||||
|
||||
# Switch to a 100K model — ratio-based would be 50000, but cap is 40000
|
||||
comp.update_model("model-b", context_length=100_000)
|
||||
assert comp.threshold_tokens == 40_000 # cap still wins
|
||||
|
||||
def test_cap_survives_model_switch_to_smaller_window(self):
|
||||
"""When switching to a model whose ratio-based threshold is below
|
||||
the cap, the ratio-based threshold wins (cap is a ceiling, not a floor)."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=50_000,
|
||||
)
|
||||
assert comp.threshold_tokens == 50_000 # cap wins on 200K (ratio=100K)
|
||||
|
||||
# Switch to a 64K model — ratio-based floor is 64000 (MINIMUM_CONTEXT_LENGTH)
|
||||
# which is > 50000 cap, so... actually 64000 > 50000 means cap still wins
|
||||
# Let's test with a 80K model: ratio=40000, cap=50000 → ratio wins
|
||||
comp.update_model("model-b", context_length=80_000)
|
||||
assert comp.threshold_tokens <= 50_000 # cap is a ceiling
|
||||
# 80000 * 0.50 = 40000, floored to 64000, cap 50000 → min(64000, 50000) = 50000
|
||||
# The floor raises it to 64000, then cap clamps to 50000
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
def test_cap_clamped_to_context_length(self):
|
||||
"""A cap larger than the context length is clamped, so the
|
||||
ratio-based threshold wins for small-context models."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=64_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=500_000,
|
||||
)
|
||||
# 64000 * 0.50 = 32000, floored to 64000 (MINIMUM_CONTEXT_LENGTH),
|
||||
# degenerate: floored >= window → 85% of 64000 = 54400.
|
||||
# Cap 500000 clamped to 64000. min(54400, 64000) = 54400.
|
||||
assert comp.threshold_tokens == 54400 # ratio-based wins
|
||||
|
||||
def test_cap_with_max_tokens_reservation(self):
|
||||
"""The cap applies after max_tokens reservation is factored in."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
max_tokens=32_768,
|
||||
threshold_tokens_cap=50_000,
|
||||
)
|
||||
# effective_window = 200000 - 32768 = 167232
|
||||
# ratio: 167232 * 0.50 = 83616, floored to max(83616, 64000) = 83616
|
||||
# cap: min(50000, 200000) = 50000. min(83616, 50000) = 50000.
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
def test_cap_survives_model_switch_with_max_tokens(self):
|
||||
"""The cap survives model switch even when max_tokens changes."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
max_tokens=32_768,
|
||||
threshold_tokens_cap=50_000,
|
||||
)
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
# Switch to a smaller model with different max_tokens
|
||||
comp.update_model("model-b", context_length=100_000, max_tokens=16_384)
|
||||
# effective_window = 100000 - 16384 = 83616
|
||||
# ratio: 83616 * 0.50 = 41808, floored to max(41808, 64000) = 64000
|
||||
# degenerate: floored (64000) >= effective_window (83616)? No, 64000 < 83616.
|
||||
# So threshold = 64000. cap: min(50000, 100000) = 50000. min(64000, 50000) = 50000.
|
||||
assert comp.threshold_tokens == 50_000
|
||||
|
||||
def test_invalid_cap_treated_as_none(self):
|
||||
"""Non-numeric, zero, or negative cap values are treated as None."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
comp0 = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=0,
|
||||
)
|
||||
assert comp0.threshold_tokens_cap is None
|
||||
assert comp0.threshold_tokens == 500_000
|
||||
|
||||
comp_neg = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=-100,
|
||||
)
|
||||
assert comp_neg.threshold_tokens_cap is None
|
||||
|
||||
comp_str = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap="not-a-number",
|
||||
)
|
||||
assert comp_str.threshold_tokens_cap is None
|
||||
|
||||
def test_should_compress_fires_at_cap_below_ratio_threshold(self):
|
||||
"""Behavioral: with a cap below the ratio-based threshold,
|
||||
should_compress() fires once usage crosses the cap — even though
|
||||
the percentage threshold has not been reached (first-fires-wins)."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=200_000,
|
||||
)
|
||||
# Ratio-based would be 500K; cap pulls the trigger down to 200K.
|
||||
assert comp.should_compress(150_000) is False # below cap
|
||||
assert comp.should_compress(200_000) is True # at cap (below 500K pct)
|
||||
assert comp.should_compress(250_000) is True # above cap
|
||||
|
||||
def test_default_config_disabled_and_no_behavior_change(self):
|
||||
"""DEFAULT_CONFIG ships threshold_tokens=None (disabled) and both
|
||||
None and 0 leave the ratio-based trigger byte-identical."""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
assert DEFAULT_CONFIG["compression"]["threshold_tokens"] is None
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000):
|
||||
baseline = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
)
|
||||
comp_none = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=None,
|
||||
)
|
||||
comp_zero = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=0,
|
||||
)
|
||||
assert comp_none.threshold_tokens == baseline.threshold_tokens
|
||||
assert comp_zero.threshold_tokens == baseline.threshold_tokens
|
||||
# And after a model switch, still identical to baseline.
|
||||
baseline.update_model("model-b", context_length=200_000)
|
||||
comp_none.update_model("model-b", context_length=200_000)
|
||||
comp_zero.update_model("model-b", context_length=200_000)
|
||||
assert comp_none.threshold_tokens == baseline.threshold_tokens
|
||||
assert comp_zero.threshold_tokens == baseline.threshold_tokens
|
||||
|
||||
def test_pct_floor_unaffected_by_cap(self):
|
||||
"""The small-context pct floor (raise-only to 0.75 under 512K) is
|
||||
computed independently of the cap: the cap clamps the resulting
|
||||
token threshold but never changes threshold_percent, and a
|
||||
cap-free small-context model keeps the floored pct."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=200_000):
|
||||
comp = ContextCompressor(
|
||||
"model-a", threshold_percent=0.50, quiet_mode=True,
|
||||
threshold_tokens_cap=100_000,
|
||||
)
|
||||
# Floor raised pct to 0.75 (200K < 512K) regardless of the cap.
|
||||
assert comp.threshold_percent == 0.75
|
||||
# Cap clamps the token trigger below the floored pct value (150K).
|
||||
assert comp.threshold_tokens == 100_000
|
||||
# Switching to a large-context model drops the pct back to the
|
||||
# configured 0.50 — cap presence doesn't perturb the re-derivation.
|
||||
comp.update_model("model-b", context_length=1_000_000)
|
||||
assert comp.threshold_percent == 0.50
|
||||
assert comp.threshold_tokens == 100_000 # cap still wins over 500K
|
||||
|
||||
|
||||
class TestTruncateToolCallArgsJson:
|
||||
"""Regression tests for #11762.
|
||||
|
||||
|
||||
@@ -2,7 +2,14 @@
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from agent.context_compressor import ContextCompressor, SUMMARY_PREFIX
|
||||
from agent.context_compressor import (
|
||||
COMPRESSED_SUMMARY_METADATA_KEY,
|
||||
ContextCompressor,
|
||||
SUMMARY_PREFIX,
|
||||
_MERGED_PRIOR_CONTEXT_HEADER,
|
||||
_MERGED_SUMMARY_DELIMITER,
|
||||
_SUMMARY_END_MARKER,
|
||||
)
|
||||
|
||||
|
||||
def _compressor() -> ContextCompressor:
|
||||
@@ -36,6 +43,21 @@ def _messages_with_handoff(summary_body: str):
|
||||
]
|
||||
|
||||
|
||||
def _messages_with_merged_handoff(summary_body: str, prior_tail: str):
|
||||
merged = {
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"{_MERGED_PRIOR_CONTEXT_HEADER}\n{prior_tail}\n\n"
|
||||
f"{_MERGED_SUMMARY_DELIMITER}\n\n"
|
||||
f"{SUMMARY_PREFIX}\n{summary_body}\n\n{_SUMMARY_END_MARKER}"
|
||||
),
|
||||
COMPRESSED_SUMMARY_METADATA_KEY: True,
|
||||
}
|
||||
messages = _messages_with_handoff(summary_body)
|
||||
messages[1] = merged
|
||||
return messages
|
||||
|
||||
|
||||
def test_existing_previous_summary_is_not_serialized_again_as_new_turn():
|
||||
"""Same-process iterative compression should not feed the old handoff twice."""
|
||||
compressor = _compressor()
|
||||
@@ -112,3 +134,127 @@ def test_handoff_in_protected_head_is_replaced_not_duplicated():
|
||||
assert "UPDATED summary body" in str(summary_messages[0]["content"])
|
||||
assert old_summary not in str(summary_messages[0]["content"])
|
||||
assert old_summary not in "\n".join(str(msg.get("content") or "") for msg in compressed)
|
||||
|
||||
|
||||
def test_recompression_drops_prior_protected_handoff_from_output():
|
||||
"""Repeated compression must not preserve stale handoff bubbles forever."""
|
||||
compressor = _compressor()
|
||||
old_summary = "DUPLICATE-HANDOFF-BODY unique old facts"
|
||||
|
||||
with patch.object(
|
||||
compressor,
|
||||
"_generate_summary",
|
||||
return_value=ContextCompressor._with_summary_prefix(
|
||||
"updated summary with old facts folded in"
|
||||
),
|
||||
):
|
||||
result = compressor.compress(_messages_with_handoff(old_summary))
|
||||
|
||||
joined = "\n".join(str(message.get("content", "")) for message in result)
|
||||
assert old_summary not in joined
|
||||
assert joined.count(SUMMARY_PREFIX) == 1
|
||||
assert "updated summary with old facts folded in" in joined
|
||||
|
||||
|
||||
def test_legacy_string_merged_handoff_preserves_real_tail_text():
|
||||
"""Pre-delimiter string handoffs still unwrap content after the end marker."""
|
||||
message = {
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"{SUMMARY_PREFIX}\nold summary\n\n"
|
||||
f"{_SUMMARY_END_MARKER}\n\nreal tail message"
|
||||
),
|
||||
COMPRESSED_SUMMARY_METADATA_KEY: True,
|
||||
}
|
||||
|
||||
result = ContextCompressor._strip_context_summary_handoff_message(message)
|
||||
|
||||
assert result == {"role": "user", "content": "real tail message"}
|
||||
|
||||
|
||||
def test_recompression_of_current_merged_handoff_preserves_prior_tail_once():
|
||||
"""Current merged handoffs lose only stale summary data on recompression."""
|
||||
compressor = _compressor()
|
||||
old_summary = "CURRENT-MERGED-OLD-SUMMARY unique continuity facts"
|
||||
prior_tail = "PRESERVED-PRIOR-TAIL real user content"
|
||||
|
||||
with patch.object(
|
||||
compressor,
|
||||
"_generate_summary",
|
||||
return_value=ContextCompressor._with_summary_prefix(
|
||||
"fresh replacement summary"
|
||||
),
|
||||
):
|
||||
result = compressor.compress(
|
||||
_messages_with_merged_handoff(old_summary, prior_tail)
|
||||
)
|
||||
|
||||
joined = "\n".join(str(message.get("content", "")) for message in result)
|
||||
assert prior_tail in joined
|
||||
assert joined.count(prior_tail) == 1
|
||||
assert old_summary not in joined
|
||||
assert joined.count(SUMMARY_PREFIX) == 1
|
||||
assert "fresh replacement summary" in joined
|
||||
|
||||
|
||||
def test_current_multimodal_merged_handoff_preserves_original_blocks():
|
||||
"""Unwrapping current list content must retain text and image blocks."""
|
||||
prior_text = {"type": "text", "text": "real multimodal tail"}
|
||||
prior_image = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,AAAA"},
|
||||
}
|
||||
message = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": f"{_MERGED_PRIOR_CONTEXT_HEADER}\n"},
|
||||
prior_text,
|
||||
prior_image,
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
f"\n\n{_MERGED_SUMMARY_DELIMITER}\n\n"
|
||||
f"{SUMMARY_PREFIX}\nstale summary\n\n{_SUMMARY_END_MARKER}"
|
||||
),
|
||||
},
|
||||
],
|
||||
COMPRESSED_SUMMARY_METADATA_KEY: True,
|
||||
}
|
||||
|
||||
result = ContextCompressor._strip_context_summary_handoff_message(message)
|
||||
|
||||
assert result == {
|
||||
"role": "user",
|
||||
"content": [prior_text, prior_image],
|
||||
}
|
||||
|
||||
|
||||
def test_legacy_multimodal_merged_handoff_preserves_original_blocks():
|
||||
"""Persisted pre-delimiter list handoffs must not lose their real tail."""
|
||||
prior_text = {"type": "text", "text": "legacy real tail"}
|
||||
prior_image = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,BBBB"},
|
||||
}
|
||||
message = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
f"{SUMMARY_PREFIX}\nlegacy stale summary\n\n"
|
||||
f"{_SUMMARY_END_MARKER}\n\n"
|
||||
),
|
||||
},
|
||||
prior_text,
|
||||
prior_image,
|
||||
],
|
||||
COMPRESSED_SUMMARY_METADATA_KEY: True,
|
||||
}
|
||||
|
||||
result = ContextCompressor._strip_context_summary_handoff_message(message)
|
||||
|
||||
assert result == {
|
||||
"role": "user",
|
||||
"content": [prior_text, prior_image],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""config.yaml sessions.* bridges for the search-index knobs (config-authoritative).
|
||||
|
||||
Salvaged from PR #65544 (adapted: agent.fts_v2_read → sessions.cjk_fts).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
import gateway.run as gateway_run
|
||||
|
||||
|
||||
def _write_home(tmp_path: Path, sessions_cfg: dict, env_text: str = "") -> Path:
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
yaml.safe_dump({"sessions": sessions_cfg}), encoding="utf-8"
|
||||
)
|
||||
(hermes_home / ".env").write_text(env_text, encoding="utf-8")
|
||||
return hermes_home
|
||||
|
||||
|
||||
def test_cjk_fts_bridged_from_config(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"cjk_fts": False})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "1")
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_CJK_FTS"] == "False"
|
||||
|
||||
|
||||
def test_search_slow_ms_bridged_from_config(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"search_slow_ms": 250})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.delenv("HERMES_SEARCH_SLOW_MS", raising=False)
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_SEARCH_SLOW_MS"] == "250"
|
||||
|
||||
|
||||
def test_env_survives_when_config_omits_search_knobs(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"auto_prune": False})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "0")
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "700")
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_CJK_FTS"] == "0"
|
||||
assert os.environ["HERMES_SEARCH_SLOW_MS"] == "700"
|
||||
|
||||
|
||||
def test_search_knobs_have_documented_defaults():
|
||||
"""The advertised config surface must exist in DEFAULT_CONFIG (no
|
||||
user-facing env switch): cjk index default ON, slow-search log at 1s."""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
|
||||
assert DEFAULT_CONFIG["sessions"]["cjk_fts"] is True
|
||||
assert DEFAULT_CONFIG["sessions"]["search_slow_ms"] == 1000
|
||||
|
||||
|
||||
def test_config_false_disables_cjk_semantics(tmp_path, monkeypatch):
|
||||
"""The bridged 'False' string must parse as OFF in hermes_state."""
|
||||
from hermes_state import _cjk_fts_config_enabled
|
||||
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "False")
|
||||
assert not _cjk_fts_config_enabled()
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "True")
|
||||
assert _cjk_fts_config_enabled()
|
||||
monkeypatch.delenv("HERMES_CJK_FTS", raising=False)
|
||||
assert _cjk_fts_config_enabled() # default on
|
||||
@@ -1076,3 +1076,37 @@ async def test_safe_sync_detects_contexts_drift():
|
||||
fake_http.edit_global_command.assert_not_awaited()
|
||||
fake_http.delete_global_command.assert_awaited_once_with(999, 77)
|
||||
fake_http.upsert_global_command.assert_awaited_once_with(999, desired)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# #31049: unconfigured platform skips reconnection (non-retryable fatal error)
|
||||
# ============================================================================
|
||||
|
||||
class TestDiscordUnconfiguredNonRetryable:
|
||||
"""Verify that missing dependency/token sets a non-retryable fatal error
|
||||
so the gateway does not queue the platform for background reconnection."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_discord_lib_sets_non_retryable_fatal(self, monkeypatch):
|
||||
"""connect() with discord.py unavailable → non-retryable fatal error."""
|
||||
_ensure_discord_mock()
|
||||
adapter = DiscordAdapter(PlatformConfig(enabled=True, token="fake"))
|
||||
# Simulate discord.py not installed
|
||||
monkeypatch.setattr(discord_platform, "DISCORD_AVAILABLE", False)
|
||||
result = await adapter.connect()
|
||||
assert result is False
|
||||
assert adapter.has_fatal_error is True
|
||||
assert adapter.fatal_error_retryable is False
|
||||
assert adapter.fatal_error_code == "missing_dependency"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_bot_token_sets_non_retryable_fatal(self, monkeypatch):
|
||||
"""connect() with empty token → non-retryable fatal error."""
|
||||
_ensure_discord_mock()
|
||||
monkeypatch.setattr(discord_platform, "DISCORD_AVAILABLE", True)
|
||||
adapter = DiscordAdapter(PlatformConfig(enabled=True, token=""))
|
||||
result = await adapter.connect()
|
||||
assert result is False
|
||||
assert adapter.has_fatal_error is True
|
||||
assert adapter.fatal_error_retryable is False
|
||||
assert adapter.fatal_error_code == "missing_credentials"
|
||||
|
||||
@@ -299,6 +299,49 @@ class TestBuildSessionContextPrompt:
|
||||
assert "pin" in prompt.lower()
|
||||
assert "current message's slack block/attachment payload" in prompt.lower()
|
||||
|
||||
def test_shared_slack_prompt_warns_against_guessed_self_mentions(self):
|
||||
"""Shared Slack threads must instruct the agent to bind mention
|
||||
targets to the current turn's sender prefix (#17916)."""
|
||||
config = GatewayConfig(
|
||||
platforms={
|
||||
Platform.SLACK: PlatformConfig(enabled=True, token="fake"),
|
||||
},
|
||||
)
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="C123",
|
||||
chat_name="team-channel",
|
||||
chat_type="group",
|
||||
user_id="U123",
|
||||
user_name="Alice",
|
||||
thread_id="171.000",
|
||||
)
|
||||
ctx = build_session_context(source, config)
|
||||
prompt = build_session_context_prompt(ctx)
|
||||
|
||||
assert "current turn's sender prefix" in prompt
|
||||
assert "Do not guess or reuse `<@U...>` mentions" in prompt
|
||||
|
||||
def test_non_shared_slack_prompt_omits_self_mention_guidance(self):
|
||||
"""1:1 Slack DMs are single-user: the shared-thread mention guidance
|
||||
must not appear."""
|
||||
config = GatewayConfig(
|
||||
platforms={
|
||||
Platform.SLACK: PlatformConfig(enabled=True, token="fake"),
|
||||
},
|
||||
)
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="D123",
|
||||
chat_type="dm",
|
||||
user_id="U123",
|
||||
user_name="Alice",
|
||||
)
|
||||
ctx = build_session_context(source, config)
|
||||
prompt = build_session_context_prompt(ctx)
|
||||
|
||||
assert "current turn's sender prefix" not in prompt
|
||||
|
||||
def test_discord_prompt_with_channel_topic(self):
|
||||
"""Channel topic should appear in the session context prompt."""
|
||||
config = GatewayConfig(
|
||||
@@ -1571,6 +1614,91 @@ class TestLastPromptTokens:
|
||||
store.update_session("k1", last_prompt_tokens=0)
|
||||
assert entry.last_prompt_tokens == 0
|
||||
|
||||
|
||||
class TestSessionMetadata:
|
||||
"""SessionEntry metadata should persist arbitrary lightweight state."""
|
||||
|
||||
def test_session_entry_metadata_roundtrip(self):
|
||||
from gateway.session import SessionEntry
|
||||
from datetime import datetime
|
||||
|
||||
entry = SessionEntry(
|
||||
session_key="test",
|
||||
session_id="s1",
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
metadata={"slack_thread_watermark:C123:123.000": "123.456"},
|
||||
)
|
||||
|
||||
restored = SessionEntry.from_dict(entry.to_dict())
|
||||
assert restored.metadata == {"slack_thread_watermark:C123:123.000": "123.456"}
|
||||
|
||||
def test_store_session_metadata_get_set(self, tmp_path):
|
||||
"""set/get_session_metadata round-trips through the store and
|
||||
persists via _save (restart survival is provided by the routing
|
||||
index — state.db gateway_routing + sessions.json mirror)."""
|
||||
config = GatewayConfig()
|
||||
with patch("gateway.session.SessionStore._ensure_loaded"):
|
||||
store = SessionStore(sessions_dir=tmp_path, config=config)
|
||||
store._loaded = True
|
||||
store._db = None
|
||||
store._save = MagicMock()
|
||||
|
||||
from gateway.session import SessionEntry
|
||||
from datetime import datetime
|
||||
entry = SessionEntry(
|
||||
session_key="k1",
|
||||
session_id="s1",
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
store._entries = {"k1": entry}
|
||||
|
||||
assert store.set_session_metadata(
|
||||
"k1", "slack_thread_watermark:C123:123.000", "123.456"
|
||||
)
|
||||
store._save.assert_called_once()
|
||||
assert (
|
||||
store.get_session_metadata("k1", "slack_thread_watermark:C123:123.000")
|
||||
== "123.456"
|
||||
)
|
||||
# Missing entry / missing key fall back safely.
|
||||
assert store.set_session_metadata("missing", "k", "v") is False
|
||||
assert store.get_session_metadata("missing", "k", "dflt") == "dflt"
|
||||
assert store.get_session_metadata("k1", "other", "dflt") == "dflt"
|
||||
|
||||
def test_session_metadata_survives_reload(self, tmp_path):
|
||||
"""Metadata written through the store must survive a full reload
|
||||
from disk (simulated gateway restart)."""
|
||||
config = GatewayConfig()
|
||||
store = SessionStore(sessions_dir=tmp_path, config=config)
|
||||
store._db = None # force sessions.json path
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="C123",
|
||||
chat_type="group",
|
||||
user_id="U123",
|
||||
thread_id="123.000",
|
||||
)
|
||||
|
||||
entry = store.get_or_create_session(source)
|
||||
assert store.set_session_metadata(
|
||||
entry.session_key,
|
||||
"slack_thread_watermark:C123:123.000",
|
||||
"123.456",
|
||||
)
|
||||
|
||||
reloaded = SessionStore(sessions_dir=tmp_path, config=config)
|
||||
reloaded._db = None
|
||||
assert (
|
||||
reloaded.get_session_metadata(
|
||||
entry.session_key,
|
||||
"slack_thread_watermark:C123:123.000",
|
||||
)
|
||||
== "123.456"
|
||||
)
|
||||
|
||||
|
||||
class TestRewriteTranscriptPreservesReasoning:
|
||||
"""rewrite_transcript must not drop reasoning fields from SQLite."""
|
||||
|
||||
|
||||
@@ -68,3 +68,64 @@ async def test_preprocess_keeps_plain_text_for_default_group_sessions():
|
||||
)
|
||||
|
||||
assert result == "hello"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preprocess_includes_slack_author_mention_for_shared_thread():
|
||||
"""Shared Slack threads expose the current author's verifiable user ID
|
||||
next to the display name so 'mention me again' requests can bind the
|
||||
mention to the CURRENT speaker (#17916)."""
|
||||
runner = _make_runner(
|
||||
GatewayConfig(
|
||||
platforms={
|
||||
Platform.SLACK: PlatformConfig(enabled=True, token="fake"),
|
||||
},
|
||||
)
|
||||
)
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="C123",
|
||||
chat_name="team-channel",
|
||||
chat_type="group",
|
||||
user_id="U123",
|
||||
user_name="Alice",
|
||||
thread_id="171.000",
|
||||
)
|
||||
event = MessageEvent(text="mention me again", source=source)
|
||||
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event,
|
||||
source=source,
|
||||
history=[],
|
||||
)
|
||||
|
||||
assert result == "[Alice | Slack user <@U123>] mention me again"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preprocess_slack_shared_thread_without_user_id_keeps_name_only():
|
||||
"""No user_id on the source → fall back to the plain name prefix."""
|
||||
runner = _make_runner(
|
||||
GatewayConfig(
|
||||
platforms={
|
||||
Platform.SLACK: PlatformConfig(enabled=True, token="fake"),
|
||||
},
|
||||
)
|
||||
)
|
||||
source = SessionSource(
|
||||
platform=Platform.SLACK,
|
||||
chat_id="C123",
|
||||
chat_name="team-channel",
|
||||
chat_type="group",
|
||||
user_name="Alice",
|
||||
thread_id="171.000",
|
||||
)
|
||||
event = MessageEvent(text="hello", source=source)
|
||||
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event,
|
||||
source=source,
|
||||
history=[],
|
||||
)
|
||||
|
||||
assert result == "[Alice] hello"
|
||||
|
||||
@@ -12,6 +12,7 @@ import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch, call
|
||||
|
||||
import pytest
|
||||
@@ -467,6 +468,7 @@ class TestSlackConnectCleanup:
|
||||
)
|
||||
|
||||
second_handler = MagicMock()
|
||||
second_handler.close_async = AsyncMock(return_value=None)
|
||||
# _start_socket_mode_handler awaits the result of start_async via
|
||||
# asyncio.create_task — so the stub must return a real coroutine, not a
|
||||
# bare MagicMock.
|
||||
@@ -489,6 +491,62 @@ class TestSlackConnectCleanup:
|
||||
first_handler.close_async.assert_awaited_once_with()
|
||||
assert adapter._handler is second_handler
|
||||
|
||||
with patch("gateway.status.release_scoped_lock"):
|
||||
await adapter.disconnect()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_closes_workspace_clients_and_clears_runtime_state(self):
|
||||
"""Regression for #51465: shutdown must close Slack WebClients.
|
||||
|
||||
``hermes gateway run --replace`` takes the old process through the
|
||||
normal adapter.disconnect() path. If Slack leaves AsyncWebClient
|
||||
instances open there, aiohttp logs ``Unclosed client session`` while
|
||||
the old gateway exits after SIGTERM.
|
||||
"""
|
||||
config = PlatformConfig(enabled=True, token="xoxb-fake")
|
||||
adapter = SlackAdapter(config)
|
||||
|
||||
socket_task = asyncio.create_task(_pending_for_fake_task())
|
||||
handler = MagicMock()
|
||||
handler.close_async = AsyncMock(return_value=None)
|
||||
|
||||
primary_client = MagicMock()
|
||||
primary_client.close = AsyncMock(return_value=None)
|
||||
team_client = MagicMock()
|
||||
team_client.close = AsyncMock(return_value=None)
|
||||
|
||||
adapter._running = True
|
||||
adapter._handler = handler
|
||||
adapter._socket_mode_task = socket_task
|
||||
adapter._app = MagicMock()
|
||||
adapter._app.client = primary_client
|
||||
adapter._team_clients = {"T_FAKE": team_client}
|
||||
adapter._team_bot_user_ids = {"T_FAKE": "U_BOT"}
|
||||
adapter._channel_team = {"C_FAKE": "T_FAKE"}
|
||||
adapter._platform_lock_scope = "slack-app-token"
|
||||
adapter._platform_lock_identity = "xapp-fake"
|
||||
adapter._app_token = "xapp-fake"
|
||||
adapter._proxy_url = "http://proxy.example.com:3128"
|
||||
adapter._bot_user_id = "U_BOT"
|
||||
|
||||
with patch("gateway.status.release_scoped_lock") as mock_release:
|
||||
await adapter.disconnect()
|
||||
|
||||
handler.close_async.assert_awaited_once_with()
|
||||
primary_client.close.assert_awaited_once_with()
|
||||
team_client.close.assert_awaited_once_with()
|
||||
assert socket_task.cancelled()
|
||||
assert adapter._app is None
|
||||
assert adapter._handler is None
|
||||
assert adapter._socket_mode_task is None
|
||||
assert adapter._team_clients == {}
|
||||
assert adapter._team_bot_user_ids == {}
|
||||
assert adapter._channel_team == {}
|
||||
assert adapter._bot_user_id is None
|
||||
assert adapter._app_token is None
|
||||
assert adapter._proxy_url is None
|
||||
mock_release.assert_called_once_with("slack-app-token", "xapp-fake")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestSlackSocketWatchdog
|
||||
@@ -802,6 +860,85 @@ class TestSlackSocketWatchdog:
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
# -- ping/pong staleness: heals the wedged transport that is_connected() misses --
|
||||
|
||||
def _adapter_with_fake_client(self, **client_attrs):
|
||||
adapter = SlackAdapter(PlatformConfig(enabled=True, token="xoxb-fake"))
|
||||
client = MagicMock()
|
||||
for key, value in client_attrs.items():
|
||||
setattr(client, key, value)
|
||||
adapter._handler = MagicMock(client=client)
|
||||
return adapter
|
||||
|
||||
def test_ping_pong_stale_when_last_ping_old(self):
|
||||
adapter = self._adapter_with_fake_client(
|
||||
ping_interval=30, last_ping_pong_time=time.time() - 1000
|
||||
)
|
||||
assert adapter._socket_ping_pong_stale() is True
|
||||
|
||||
def test_ping_pong_fresh_when_last_ping_recent(self):
|
||||
adapter = self._adapter_with_fake_client(
|
||||
ping_interval=30, last_ping_pong_time=time.time() - 5
|
||||
)
|
||||
assert adapter._socket_ping_pong_stale() is False
|
||||
|
||||
def test_ping_pong_none_within_grace_not_stale(self):
|
||||
adapter = self._adapter_with_fake_client(
|
||||
ping_interval=30, last_ping_pong_time=None
|
||||
)
|
||||
adapter._socket_handler_started_monotonic = time.monotonic()
|
||||
assert adapter._socket_ping_pong_stale() is False
|
||||
|
||||
def test_ping_pong_none_beyond_grace_is_stale(self):
|
||||
adapter = self._adapter_with_fake_client(
|
||||
ping_interval=30, last_ping_pong_time=None
|
||||
)
|
||||
adapter._socket_first_ping_grace_s = 0.0
|
||||
adapter._socket_handler_started_monotonic = time.monotonic() - 200
|
||||
assert adapter._socket_ping_pong_stale() is True
|
||||
|
||||
def test_ping_pong_no_handler_not_stale(self):
|
||||
adapter = SlackAdapter(PlatformConfig(enabled=True, token="xoxb-fake"))
|
||||
adapter._handler = None
|
||||
assert adapter._socket_ping_pong_stale() is False
|
||||
|
||||
def test_ping_pong_nonnumeric_attrs_not_stale(self):
|
||||
# A mocked/partial client (MagicMock attrs) must never trigger reconnect.
|
||||
adapter = SlackAdapter(PlatformConfig(enabled=True, token="xoxb-fake"))
|
||||
adapter._handler = MagicMock()
|
||||
assert adapter._socket_ping_pong_stale() is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_watchdog_reconnects_when_ping_pong_stale_despite_is_connected_true(self):
|
||||
adapter = SlackAdapter(PlatformConfig(enabled=True, token="xoxb-fake"))
|
||||
adapter._socket_watchdog_interval_s = 0.01
|
||||
factory, instances = self._make_fake_handler_factory()
|
||||
|
||||
with contextlib.ExitStack() as stack:
|
||||
for p in self._patch_stack(factory):
|
||||
stack.enter_context(p)
|
||||
|
||||
try:
|
||||
assert await adapter.connect() is True
|
||||
assert len(instances) == 1
|
||||
|
||||
# Transport lies: is_connected() stays True while ping/pong has
|
||||
# gone stale (the wedged "Session is closed" zombie).
|
||||
instances[0].client.is_connected = lambda: True
|
||||
instances[0].client.ping_interval = 30
|
||||
instances[0].client.last_ping_pong_time = time.time() - 1000
|
||||
|
||||
for _ in range(40):
|
||||
if len(instances) >= 2:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
assert len(instances) >= 2, "watchdog did not heal wedged (lying) transport"
|
||||
assert instances[0].closed is True
|
||||
assert adapter._handler is instances[-1]
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestSlackProxyBehavior
|
||||
@@ -1346,6 +1483,55 @@ class TestBangPrefixCommands:
|
||||
# same thread.
|
||||
assert msg_event.source.thread_id == "1111111111.000001"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_queue_survives_first_thread_context_backfill(self, adapter):
|
||||
"""Backfill stays out of command text while remaining available."""
|
||||
adapter._has_active_session_for_thread = MagicMock(return_value=False)
|
||||
adapter._fetch_thread_context = AsyncMock(
|
||||
return_value=(
|
||||
"[Slack thread context — earlier messages]\n"
|
||||
"Alice: prior request\n"
|
||||
"[End of thread context]\n\n"
|
||||
)
|
||||
)
|
||||
adapter._fetch_thread_parent_text = AsyncMock(return_value="prior request")
|
||||
|
||||
evt = self._make_event(
|
||||
"!queue follow up after the current task",
|
||||
thread_ts="1111111111.000001",
|
||||
)
|
||||
await adapter._handle_slack_message(evt)
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "/queue follow up after the current task"
|
||||
assert msg_event.message_type == MessageType.COMMAND
|
||||
assert msg_event.get_command() == "queue"
|
||||
assert msg_event.get_command_args() == "follow up after the current task"
|
||||
assert msg_event.channel_context.startswith("[Slack thread context")
|
||||
assert "prior request" in msg_event.channel_context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_command_thread_backfill_uses_channel_context(self, adapter):
|
||||
"""Normal thread text remains separate without losing its backfill."""
|
||||
adapter._has_active_session_for_thread = MagicMock(return_value=False)
|
||||
adapter._fetch_thread_context = AsyncMock(
|
||||
return_value="[Slack thread context]\nAlice: earlier note\n"
|
||||
)
|
||||
adapter._fetch_thread_parent_text = AsyncMock(return_value="earlier note")
|
||||
|
||||
evt = self._make_event(
|
||||
"follow up",
|
||||
thread_ts="1111111111.000001",
|
||||
)
|
||||
await adapter._handle_slack_message(evt)
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "follow up"
|
||||
assert msg_event.message_type == MessageType.TEXT
|
||||
assert msg_event.channel_context == (
|
||||
"[Slack thread context]\nAlice: earlier note\n"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_unknown_token_passes_through_unchanged(self, adapter):
|
||||
"""``!nice work`` is just a casual message — must NOT be rewritten."""
|
||||
@@ -3334,6 +3520,90 @@ class TestThreadReplyHandling:
|
||||
msg_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "Follow-up question"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_reply_routes_when_parent_mentioned_bot(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""A plain thread reply should route when the thread parent mentioned
|
||||
the bot (#24848) — e.g. parent says '<@bot> check this and ask me
|
||||
before running', a later bare 'run' reply must wake the bot even
|
||||
with no session and no in-memory mention tracking (restart-safe)."""
|
||||
mock_session_store._entries = {}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=False
|
||||
)
|
||||
mock_session_store.get_session_metadata = MagicMock(return_value="")
|
||||
adapter_with_session_store._app.client.conversations_replies = AsyncMock(
|
||||
side_effect=[
|
||||
# _bot_authored_thread_root miss path → full context fetch
|
||||
# (parent is human-authored, so check 4 fails).
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"ts": "123.000",
|
||||
"user": "U_USER",
|
||||
"text": "<@U_BOT> check this and ask me for run",
|
||||
},
|
||||
],
|
||||
},
|
||||
# Any later fetch (cold-start context) reuses cache or refetches.
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"ts": "123.000",
|
||||
"user": "U_USER",
|
||||
"text": "<@U_BOT> check this and ask me for run",
|
||||
},
|
||||
{"ts": "123.456", "user": "U_USER", "text": "run"},
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
adapter_with_session_store._user_name_cache = {("T_TEAM", "U_USER"): "Kai Yi"}
|
||||
|
||||
event = {
|
||||
"text": "run",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.456",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
}
|
||||
await adapter_with_session_store._handle_slack_message(event)
|
||||
|
||||
adapter_with_session_store.handle_message.assert_called_once()
|
||||
msg_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "run"
|
||||
# Cold-start context carries the parent so the agent sees the ask.
|
||||
assert "check this and ask me for run" in msg_event.channel_context
|
||||
# Thread remembered so later replies skip the parent fetch.
|
||||
assert "123.000" in adapter_with_session_store._mentioned_threads
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_top_level_mention_registers_thread_for_replies(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""A TOP-LEVEL @mention starts a thread (session-scoped thread_ts
|
||||
falls back to the message ts); replies to it must auto-trigger, so
|
||||
the synthetic root is registered in _mentioned_threads (#24848)."""
|
||||
mock_session_store._entries = {}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=False
|
||||
)
|
||||
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "<@U_BOT> kick off the deploy checklist",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "555.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
|
||||
adapter_with_session_store.handle_message.assert_called_once()
|
||||
assert "555.000" in adapter_with_session_store._mentioned_threads
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_reply_with_mention_strips_bot_id(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
@@ -3359,6 +3629,168 @@ class TestThreadReplyHandling:
|
||||
assert "<@U_BOT>" not in msg_event.text
|
||||
assert msg_event.text == "thanks for the help"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_active_thread_explicit_mention_refreshes_context_delta(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""Explicit @mention on an active thread must re-fetch the thread and
|
||||
inject only the delta past the stored watermark, as part of the NEW
|
||||
turn (channel_context) — never rewriting prior history (#23918)."""
|
||||
mock_session_store._entries = {"any": MagicMock()}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=True
|
||||
)
|
||||
# Persisted watermark: session has consumed up to 123.100.
|
||||
metadata = {"slack_thread_watermark:C123:123.000": "123.100"}
|
||||
mock_session_store.get_session_metadata = MagicMock(
|
||||
side_effect=lambda sk, k, d=None: metadata.get(k, d)
|
||||
)
|
||||
mock_session_store.set_session_metadata = MagicMock(
|
||||
side_effect=lambda sk, k, v: metadata.__setitem__(k, v) or True
|
||||
)
|
||||
adapter_with_session_store._app.client.conversations_replies = AsyncMock(
|
||||
return_value={
|
||||
"messages": [
|
||||
{"ts": "123.000", "user": "U_PARENT", "text": "Original question"},
|
||||
{"ts": "123.100", "user": "U_USER", "text": "Old context"},
|
||||
{"ts": "123.200", "user": "U_OTHER", "text": "Fresh update"},
|
||||
{"ts": "123.456", "user": "U_USER", "text": "<@U_BOT> what changed?"},
|
||||
]
|
||||
}
|
||||
)
|
||||
adapter_with_session_store._user_name_cache = {
|
||||
("T_TEAM", "U_PARENT"): "Parent",
|
||||
("T_TEAM", "U_USER"): "User",
|
||||
("T_TEAM", "U_OTHER"): "Other",
|
||||
}
|
||||
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "<@U_BOT> what changed?",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.456",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
|
||||
adapter_with_session_store._app.client.conversations_replies.assert_awaited_once()
|
||||
msg_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
# Delta arrives as new-turn channel_context, not baked into text.
|
||||
assert msg_event.text == "what changed?"
|
||||
assert "Fresh update" in msg_event.channel_context
|
||||
# Already-consumed messages must NOT be re-injected.
|
||||
assert "Old context" not in msg_event.channel_context
|
||||
# Watermark advanced to the trigger ts.
|
||||
assert metadata["slack_thread_watermark:C123:123.000"] == "123.456"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_active_thread_unmentioned_reply_does_not_refetch(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""Unmentioned replies in active threads keep the existing behavior:
|
||||
no thread re-fetch, no context injection (once the one-shot restart
|
||||
rehydration check has found no watermark)."""
|
||||
mock_session_store._entries = {"any": MagicMock()}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=True
|
||||
)
|
||||
# No persisted watermark → rehydration check is a no-op.
|
||||
mock_session_store.get_session_metadata = MagicMock(return_value="")
|
||||
adapter_with_session_store._app.client.conversations_replies = AsyncMock()
|
||||
adapter_with_session_store._fetch_thread_parent_text = AsyncMock(
|
||||
return_value=""
|
||||
)
|
||||
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "Follow-up without mention",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.456",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
|
||||
adapter_with_session_store.handle_message.assert_called_once()
|
||||
adapter_with_session_store._app.client.conversations_replies.assert_not_called()
|
||||
msg_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert msg_event.channel_context is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restart_rehydrates_thread_delta_once(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
):
|
||||
"""After a gateway restart (fresh adapter instance, persisted session
|
||||
+ watermark), the FIRST ordinary thread reply injects messages the
|
||||
session missed while the gateway was down — exactly once. Subsequent
|
||||
replies do not re-fetch."""
|
||||
mock_session_store._entries = {"any": MagicMock()}
|
||||
adapter_with_session_store._has_active_session_for_thread = MagicMock(
|
||||
return_value=True
|
||||
)
|
||||
# Persisted watermark survives the restart via the session store.
|
||||
metadata = {"slack_thread_watermark:C123:123.000": "123.100"}
|
||||
mock_session_store.get_session_metadata = MagicMock(
|
||||
side_effect=lambda sk, k, d=None: metadata.get(k, d)
|
||||
)
|
||||
mock_session_store.set_session_metadata = MagicMock(
|
||||
side_effect=lambda sk, k, v: metadata.__setitem__(k, v) or True
|
||||
)
|
||||
adapter_with_session_store._app.client.conversations_replies = AsyncMock(
|
||||
return_value={
|
||||
"messages": [
|
||||
{"ts": "123.000", "user": "U_PARENT", "text": "Original question"},
|
||||
{"ts": "123.100", "user": "U_USER", "text": "Old context"},
|
||||
{"ts": "123.200", "user": "U_OTHER", "text": "Missed while down"},
|
||||
{"ts": "123.456", "user": "U_USER", "text": "please continue"},
|
||||
]
|
||||
}
|
||||
)
|
||||
adapter_with_session_store._user_name_cache = {
|
||||
("T_TEAM", "U_PARENT"): "Parent",
|
||||
("T_TEAM", "U_USER"): "User",
|
||||
("T_TEAM", "U_OTHER"): "Other",
|
||||
}
|
||||
|
||||
# Fresh adapter instance == empty _thread_rehydration_checked, which
|
||||
# is exactly the post-restart state.
|
||||
assert adapter_with_session_store._thread_rehydration_checked == set()
|
||||
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "please continue",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.456",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
|
||||
first_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert first_event.text == "please continue"
|
||||
assert "Missed while down" in first_event.channel_context
|
||||
assert "Old context" not in first_event.channel_context
|
||||
assert metadata["slack_thread_watermark:C123:123.000"] == "123.456"
|
||||
|
||||
# Second ordinary reply: no re-fetch, no injection.
|
||||
adapter_with_session_store.handle_message.reset_mock()
|
||||
adapter_with_session_store._app.client.conversations_replies.reset_mock()
|
||||
await adapter_with_session_store._handle_slack_message({
|
||||
"text": "and another thing",
|
||||
"user": "U_USER",
|
||||
"channel": "C123",
|
||||
"ts": "123.500",
|
||||
"thread_ts": "123.000",
|
||||
"channel_type": "channel",
|
||||
"team": "T_TEAM",
|
||||
})
|
||||
adapter_with_session_store._app.client.conversations_replies.assert_not_called()
|
||||
second_event = adapter_with_session_store.handle_message.call_args[0][0]
|
||||
assert second_event.channel_context is None
|
||||
# Watermark keeps advancing in steady state.
|
||||
assert metadata["slack_thread_watermark:C123:123.000"] == "123.500"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_top_level_message_requires_mention_even_with_session(
|
||||
self, adapter_with_session_store, mock_session_store
|
||||
@@ -5192,3 +5624,64 @@ class TestThreadContextAppMessages:
|
||||
)
|
||||
|
||||
assert "hello" in content # the real message survives; empty bot msg dropped
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Missing-credential handling — fatal-error contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMissingCredentials:
|
||||
"""Missing SLACK_BOT_TOKEN or SLACK_APP_TOKEN must set a non-retryable fatal error."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_bot_token_sets_fatal_error(self):
|
||||
"""When SLACK_BOT_TOKEN is absent from both config and env, connect()
|
||||
must set fatal_error with code 'missing_slack_bot_token' and retryable=False."""
|
||||
config = PlatformConfig(enabled=True, token=None) # no bot token
|
||||
adapter = SlackAdapter(config)
|
||||
|
||||
fatal_errors = []
|
||||
|
||||
def capture_fatal(code, message, *, retryable):
|
||||
fatal_errors.append({"code": code, "message": message, "retryable": retryable})
|
||||
|
||||
with (
|
||||
patch.object(adapter, "_set_fatal_error", side_effect=capture_fatal),
|
||||
patch.dict(os.environ, {}, clear=True),
|
||||
):
|
||||
result = await adapter.connect()
|
||||
|
||||
assert result is False
|
||||
assert len(fatal_errors) == 1
|
||||
assert fatal_errors[0]["code"] == "missing_slack_bot_token"
|
||||
assert fatal_errors[0]["retryable"] is False
|
||||
assert "SLACK_BOT_TOKEN" in fatal_errors[0]["message"]
|
||||
assert "hermes gateway setup" in fatal_errors[0]["message"].lower() or ".env" in fatal_errors[0]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_app_token_sets_fatal_error(self):
|
||||
"""When SLACK_APP_TOKEN is absent but SLACK_BOT_TOKEN is present,
|
||||
connect() must set fatal_error with code 'missing_slack_app_token'
|
||||
and retryable=False."""
|
||||
config = PlatformConfig(enabled=True, token="xoxb-fake")
|
||||
adapter = SlackAdapter(config)
|
||||
|
||||
fatal_errors = []
|
||||
|
||||
def capture_fatal(code, message, *, retryable):
|
||||
fatal_errors.append({"code": code, "message": message, "retryable": retryable})
|
||||
|
||||
with (
|
||||
patch.object(adapter, "_set_fatal_error", side_effect=capture_fatal),
|
||||
patch.dict(os.environ, {"SLACK_BOT_TOKEN": "xoxb-fake"}, clear=True),
|
||||
):
|
||||
result = await adapter.connect()
|
||||
|
||||
assert result is False
|
||||
assert len(fatal_errors) == 1
|
||||
assert fatal_errors[0]["code"] == "missing_slack_app_token"
|
||||
assert fatal_errors[0]["retryable"] is False
|
||||
assert "SLACK_APP_TOKEN" in fatal_errors[0]["message"]
|
||||
assert "hermes gateway setup" in fatal_errors[0]["message"].lower() or ".env" in fatal_errors[0]["message"]
|
||||
|
||||
|
||||
@@ -514,24 +514,28 @@ class TestSlackThreadContext:
|
||||
assert "<@U_BOT>" not in context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_bot_messages(self):
|
||||
"""Self-bot child replies are skipped to avoid circular context,
|
||||
but non-self bots (e.g. cron posts, third-party integrations) are kept.
|
||||
async def test_includes_self_bot_replies_as_assistant_on_cold_start(self):
|
||||
"""Cold-start contract (issue #38861): self-bot replies in the thread
|
||||
must be included in the context, labelled with an ``[assistant]``
|
||||
prefix so the agent can reconstruct its own prior turns. This method
|
||||
only runs on the cold-start path (guarded at the call site by
|
||||
``_has_active_session_for_thread``) — when an active session exists,
|
||||
the session history already carries those replies, so there is no
|
||||
risk of circular duplication.
|
||||
|
||||
Regression guard for the fix in _fetch_thread_context: previously ALL
|
||||
bot messages were dropped, which lost context when the bot was replying
|
||||
to a cron-posted thread parent."""
|
||||
Third-party bots (e.g. deploy notifications) must still be kept,
|
||||
attributed to their display name."""
|
||||
adapter = _make_adapter()
|
||||
mock_client = adapter._team_clients["T1"]
|
||||
mock_client.conversations_replies = AsyncMock(return_value={
|
||||
"messages": [
|
||||
{"ts": "1000.0", "user": "U1", "text": "Parent"},
|
||||
# Self-bot reply -> must be skipped (circular)
|
||||
# Self-bot reply -> kept on cold-start, prefixed [assistant]
|
||||
{
|
||||
"ts": "1000.1",
|
||||
"bot_id": "B_SELF",
|
||||
"user": "U_BOT",
|
||||
"text": "Previous bot self-reply (should be skipped)",
|
||||
"text": "Previous bot self-reply",
|
||||
},
|
||||
# Third-party bot child -> kept (useful context)
|
||||
{
|
||||
@@ -552,10 +556,13 @@ class TestSlackThreadContext:
|
||||
channel_id="C1", thread_ts="1000.0", current_ts="1000.2", team_id="T1"
|
||||
)
|
||||
|
||||
assert "Previous bot self-reply" not in context
|
||||
assert "Alice: Parent" in context
|
||||
# Third-party bot message must now be included
|
||||
# Self-bot reply must now be included with [assistant] label
|
||||
assert "[assistant] Previous bot self-reply" in context
|
||||
# Third-party bot message must still be included
|
||||
assert "Deploy succeeded" in context
|
||||
# The [assistant] label must NOT leak to user messages
|
||||
assert "[assistant] Alice" not in context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_thread(self):
|
||||
@@ -751,14 +758,19 @@ class TestSlackThreadContext:
|
||||
|
||||
assert "Incident triggered" in text
|
||||
assert "https://example.example/incident/42" in text
|
||||
"""Parent (non-self bot) is kept, self-bot child replies are dropped,
|
||||
user replies are kept."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_thread_context_includes_self_bot_replies_with_assistant_label(self):
|
||||
"""Cold-start: parent (non-self bot) kept with [thread parent],
|
||||
self-bot child replies kept with [assistant] label, user replies
|
||||
kept unchanged. The cold-start path is the ONLY caller of this
|
||||
method; circular-context risk does not apply here (see #38861)."""
|
||||
adapter = _make_adapter()
|
||||
mock_client = adapter._team_clients["T1"]
|
||||
mock_client.conversations_replies = AsyncMock(return_value={
|
||||
"messages": [
|
||||
{"ts": "1000.0", "bot_id": "B_CRON", "text": "Cron summary"},
|
||||
# Self-bot child reply -> excluded
|
||||
# Self-bot child reply -> kept with [assistant] label
|
||||
{
|
||||
"ts": "1000.1",
|
||||
"bot_id": "B_SELF",
|
||||
@@ -779,7 +791,8 @@ class TestSlackThreadContext:
|
||||
|
||||
assert "Cron summary" in context
|
||||
assert "[thread parent]" in context
|
||||
assert "Previous self reply" not in context
|
||||
# Self-bot child reply is now kept with the [assistant] label
|
||||
assert "[assistant] Previous self reply" in context
|
||||
assert "Follow-up question" in context
|
||||
assert "Current" not in context
|
||||
|
||||
@@ -808,7 +821,7 @@ class TestSlackThreadContext:
|
||||
"team": "T2",
|
||||
"text": "Cross-workspace bot reply",
|
||||
},
|
||||
# Self-bot for T2 — must be skipped
|
||||
# Self-bot for T2 — kept with the [assistant] label
|
||||
{
|
||||
"ts": "2000.2",
|
||||
"bot_id": "B_SELF_T2",
|
||||
@@ -827,7 +840,12 @@ class TestSlackThreadContext:
|
||||
|
||||
assert "Parent T2" in context
|
||||
assert "Cross-workspace bot reply" in context
|
||||
assert "Own T2 bot reply" not in context
|
||||
# T2's own self-bot reply is kept with the [assistant] label
|
||||
# (cold-start path includes self-replies; see #38861). The
|
||||
# per-workspace filter still applies: this assertion confirms
|
||||
# we use T2's bot id, not T1's, when deciding what counts as
|
||||
# self-bot.
|
||||
assert "[assistant] Own T2 bot reply" in context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_thread_context_current_ts_excluded(self):
|
||||
@@ -933,6 +951,34 @@ class TestSessionKeyFix:
|
||||
)
|
||||
assert result is False
|
||||
|
||||
def test_stale_session_returns_false(self):
|
||||
"""A session key that exists but would be rolled by the reset policy
|
||||
must NOT count as active — otherwise the reset-time first turn skips
|
||||
the thread-history reseed (#55239)."""
|
||||
adapter = _make_adapter()
|
||||
|
||||
class Store:
|
||||
config = MagicMock()
|
||||
config.group_sessions_per_user = False
|
||||
config.thread_sessions_per_user = False
|
||||
_entries = {
|
||||
"agent:main:slack:group:C1:1000.0": MagicMock()
|
||||
}
|
||||
|
||||
def _ensure_loaded(self):
|
||||
return None
|
||||
|
||||
def _should_reset(self, entry, source):
|
||||
return "idle"
|
||||
|
||||
adapter._session_store = Store()
|
||||
|
||||
result = adapter._has_active_session_for_thread(
|
||||
channel_id="C1", thread_ts="1000.0", user_id="U123"
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Thread engagement — bot-started threads & mentioned threads
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Tests for Slack Socket Mode dedup TTL (#4777).
|
||||
|
||||
Slack replays un-acked Socket Mode events when the websocket reconnects.
|
||||
The replay can land several minutes after the original; the dedup window
|
||||
must outlast that gap so the redelivered event is suppressed instead of
|
||||
producing a second bot reply. Regression for the 300s-default bug where
|
||||
replays >5 min later slipped through.
|
||||
|
||||
Follows the slack-bolt mocking pattern from test_slack_mention.py.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
def _ensure_slack_mock():
|
||||
if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"):
|
||||
return
|
||||
|
||||
slack_bolt = MagicMock()
|
||||
slack_bolt.async_app.AsyncApp = MagicMock
|
||||
slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock
|
||||
|
||||
slack_sdk = MagicMock()
|
||||
slack_sdk.web.async_client.AsyncWebClient = MagicMock
|
||||
|
||||
for name, mod in [
|
||||
("slack_bolt", slack_bolt),
|
||||
("slack_bolt.async_app", slack_bolt.async_app),
|
||||
("slack_bolt.adapter", slack_bolt.adapter),
|
||||
("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode),
|
||||
("slack_bolt.adapter.socket_mode.async_handler", slack_bolt.adapter.socket_mode.async_handler),
|
||||
("slack_sdk", slack_sdk),
|
||||
("slack_sdk.web", slack_sdk.web),
|
||||
("slack_sdk.web.async_client", slack_sdk.web.async_client),
|
||||
]:
|
||||
sys.modules.setdefault(name, mod)
|
||||
|
||||
|
||||
_ensure_slack_mock()
|
||||
|
||||
import plugins.platforms.slack.adapter as _slack_mod # noqa: E402
|
||||
|
||||
_slack_mod.SLACK_AVAILABLE = True
|
||||
|
||||
from gateway.platforms.helpers import MessageDeduplicator # noqa: E402
|
||||
from plugins.platforms.slack.adapter import _slack_dedup_ttl_seconds # noqa: E402
|
||||
|
||||
|
||||
def test_default_ttl_outlasts_slack_reconnect_redelivery_window():
|
||||
# The whole point of the fix: the window must be much longer than the
|
||||
# ~6 min reconnect-redelivery gap that caused the duplicate reply.
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
assert _slack_dedup_ttl_seconds() >= 1800.0
|
||||
|
||||
|
||||
def test_env_override_is_respected():
|
||||
with patch.dict(os.environ, {"SLACK_DEDUP_TTL_SECONDS": "120"}, clear=True):
|
||||
assert _slack_dedup_ttl_seconds() == 120.0
|
||||
|
||||
|
||||
def test_invalid_env_falls_back_to_default():
|
||||
with patch.dict(os.environ, {"SLACK_DEDUP_TTL_SECONDS": "not-a-number"}, clear=True):
|
||||
assert _slack_dedup_ttl_seconds() >= 1800.0
|
||||
with patch.dict(os.environ, {"SLACK_DEDUP_TTL_SECONDS": "0"}, clear=True):
|
||||
assert _slack_dedup_ttl_seconds() >= 1800.0
|
||||
|
||||
|
||||
def test_redelivery_six_minutes_later_is_suppressed():
|
||||
"""A replay 6 min after first processing must be treated as duplicate."""
|
||||
dedup = MessageDeduplicator(ttl_seconds=_slack_dedup_ttl_seconds())
|
||||
event_ts = "1733382960.001500"
|
||||
|
||||
# First delivery — recorded now.
|
||||
assert dedup.is_duplicate(event_ts) is False
|
||||
# Simulate the entry being stamped 6 minutes ago (reconnect redelivery gap).
|
||||
dedup._seen[event_ts] = time.time() - 360
|
||||
# Redelivery of the SAME event must still be caught.
|
||||
assert dedup.is_duplicate(event_ts) is True
|
||||
|
||||
|
||||
def test_old_default_300s_would_have_missed_it():
|
||||
"""Pins the regression: the prior 300s window let the replay through."""
|
||||
dedup = MessageDeduplicator(ttl_seconds=300)
|
||||
event_ts = "1733382960.001500"
|
||||
assert dedup.is_duplicate(event_ts) is False
|
||||
dedup._seen[event_ts] = time.time() - 360 # 6 min ago, past 300s TTL
|
||||
# Demonstrates the bug: replay treated as new → second reply.
|
||||
assert dedup.is_duplicate(event_ts) is False
|
||||
@@ -0,0 +1,158 @@
|
||||
"""
|
||||
Tests for Slack inbound mention humanization + bot identity grounding.
|
||||
|
||||
Slack delivers user mentions as opaque IDs (``<@U123>``). Passing those to the
|
||||
agent raw leaves it unable to tell one participant from another — or from
|
||||
itself — so it can misread a mention of a human as a self-mention and reply to
|
||||
messages addressed to that person (the reported "bot thinks it's @someone-else"
|
||||
bug). Two cooperating fixes:
|
||||
|
||||
* ``_humanize_user_mentions`` rewrites ``<@UID>`` → ``@DisplayName`` (the
|
||||
Slack equivalent of Discord's ``clean_content``).
|
||||
* ``_build_identity_prompt`` returns an ephemeral system-prompt line naming
|
||||
the bot's own Slack handle so the agent has a positive "that's me" anchor.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock slack-bolt if not installed (same pattern as test_slack_mention.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _ensure_slack_mock():
|
||||
if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"):
|
||||
return
|
||||
|
||||
slack_bolt = MagicMock()
|
||||
slack_bolt.async_app.AsyncApp = MagicMock
|
||||
slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock
|
||||
|
||||
slack_sdk = MagicMock()
|
||||
slack_sdk.web.async_client.AsyncWebClient = MagicMock
|
||||
|
||||
for name, mod in [
|
||||
("slack_bolt", slack_bolt),
|
||||
("slack_bolt.async_app", slack_bolt.async_app),
|
||||
("slack_bolt.adapter", slack_bolt.adapter),
|
||||
("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode),
|
||||
("slack_bolt.adapter.socket_mode.async_handler",
|
||||
slack_bolt.adapter.socket_mode.async_handler),
|
||||
("slack_sdk", slack_sdk),
|
||||
("slack_sdk.web", slack_sdk.web),
|
||||
("slack_sdk.web.async_client", slack_sdk.web.async_client),
|
||||
]:
|
||||
sys.modules.setdefault(name, mod)
|
||||
sys.modules.setdefault("aiohttp", MagicMock())
|
||||
|
||||
|
||||
_ensure_slack_mock()
|
||||
|
||||
import plugins.platforms.slack.adapter as _slack_mod # noqa: E402
|
||||
_slack_mod.SLACK_AVAILABLE = True
|
||||
|
||||
from plugins.platforms.slack.adapter import SlackAdapter # noqa: E402
|
||||
|
||||
|
||||
def _make_adapter():
|
||||
# object.__new__ skips __init__ (heavy setup) — established slack-test pattern.
|
||||
return object.__new__(SlackAdapter)
|
||||
|
||||
|
||||
def _adapter_with_names(names):
|
||||
"""Adapter whose _resolve_user_name returns from a fixed UID→name map."""
|
||||
adapter = _make_adapter()
|
||||
|
||||
async def _resolve(user_id, chat_id="", team_id=""):
|
||||
return names.get(user_id, user_id)
|
||||
|
||||
adapter._resolve_user_name = _resolve # type: ignore[assignment]
|
||||
return adapter
|
||||
|
||||
|
||||
# ----- _humanize_user_mentions -------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_humanizes_single_mention():
|
||||
adapter = _adapter_with_names({"U07ALICE": "Alice Example"})
|
||||
out = await adapter._humanize_user_mentions(
|
||||
"<@U07ALICE> I think thread is prob the right default", chat_id="C1"
|
||||
)
|
||||
assert out == "@Alice Example I think thread is prob the right default"
|
||||
assert "<@" not in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_humanizes_multiple_distinct_mentions():
|
||||
adapter = _adapter_with_names(
|
||||
{"U07ALICE": "Alice Example", "U07BOB": "Bob Example"}
|
||||
)
|
||||
out = await adapter._humanize_user_mentions(
|
||||
"hey <@U07ALICE> and <@U07BOB>", chat_id="C1"
|
||||
)
|
||||
assert out == "hey @Alice Example and @Bob Example"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_labelled_mention_form():
|
||||
# Slack sometimes sends <@UID|handle>; only the ID drives resolution.
|
||||
adapter = _adapter_with_names({"U07ALICE": "Alice Example"})
|
||||
out = await adapter._humanize_user_mentions("<@U07ALICE|alice> hi", chat_id="C1")
|
||||
assert out == "@Alice Example hi"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_mention_all_replaced():
|
||||
adapter = _adapter_with_names({"U07ALICE": "Alice Example"})
|
||||
out = await adapter._humanize_user_mentions(
|
||||
"<@U07ALICE> ping <@U07ALICE>", chat_id="C1"
|
||||
)
|
||||
assert out == "@Alice Example ping @Alice Example"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unresolvable_mention_falls_back_to_id():
|
||||
# Resolution returns the bare ID; keep the message intact, don't empty it.
|
||||
adapter = _adapter_with_names({})
|
||||
out = await adapter._humanize_user_mentions("<@U07GHOST> hi", chat_id="C1")
|
||||
assert out == "@U07GHOST hi"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_mentions_returns_unchanged():
|
||||
adapter = _adapter_with_names({"U07ALICE": "Alice Example"})
|
||||
out = await adapter._humanize_user_mentions("plain text, no pings", chat_id="C1")
|
||||
assert out == "plain text, no pings"
|
||||
|
||||
|
||||
# ----- _build_identity_prompt --------------------------------------------------
|
||||
|
||||
def test_identity_prompt_names_the_bot():
|
||||
adapter = _make_adapter()
|
||||
adapter._bot_display_name = "TestBot"
|
||||
adapter._team_bot_names = {}
|
||||
prompt = adapter._build_identity_prompt(team_id="T1")
|
||||
assert "@TestBot" in prompt
|
||||
# Must instruct that another participant's mention is not a self-mention.
|
||||
assert "not a mention of you" in prompt
|
||||
|
||||
|
||||
def test_identity_prompt_prefers_per_team_name():
|
||||
adapter = _make_adapter()
|
||||
adapter._bot_display_name = "PrimaryBot"
|
||||
adapter._team_bot_names = {"T2": "WorkspaceTwoBot"}
|
||||
prompt = adapter._build_identity_prompt(team_id="T2")
|
||||
assert "@WorkspaceTwoBot" in prompt
|
||||
assert "PrimaryBot" not in prompt
|
||||
|
||||
|
||||
def test_identity_prompt_empty_when_name_unknown():
|
||||
# Before connect (no name resolved) the prompt must be empty, not a
|
||||
# half-formed line — callers skip injecting an empty string.
|
||||
adapter = _make_adapter()
|
||||
adapter._bot_display_name = None
|
||||
adapter._team_bot_names = {}
|
||||
assert adapter._build_identity_prompt(team_id="T1") == ""
|
||||
@@ -0,0 +1,381 @@
|
||||
"""
|
||||
Tests for Slack Socket Mode teardown (issue #46990).
|
||||
|
||||
slack_sdk's SocketModeClient.connect() is an unconditional retry loop that
|
||||
swallows connection errors and never checks the client's ``closed`` flag. If a
|
||||
task is still inside that loop when the client's shared aiohttp session is
|
||||
closed, it keeps retrying forever and logs
|
||||
``Failed to connect (error: Session is closed); Retrying...`` against a session
|
||||
that can never work again.
|
||||
|
||||
These tests pin the ordering and cleanup that keep old-client background work
|
||||
from outliving a teardown.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock the slack-bolt package if it's not installed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ensure_slack_mock():
|
||||
"""Install mock slack modules so SlackAdapter can be imported."""
|
||||
if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"):
|
||||
return # Real library installed
|
||||
|
||||
slack_bolt = MagicMock()
|
||||
slack_bolt.async_app.AsyncApp = MagicMock
|
||||
slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock
|
||||
|
||||
slack_sdk = MagicMock()
|
||||
slack_sdk.web.async_client.AsyncWebClient = MagicMock
|
||||
|
||||
for name, mod in [
|
||||
("slack_bolt", slack_bolt),
|
||||
("slack_bolt.async_app", slack_bolt.async_app),
|
||||
("slack_bolt.adapter", slack_bolt.adapter),
|
||||
("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode),
|
||||
(
|
||||
"slack_bolt.adapter.socket_mode.async_handler",
|
||||
slack_bolt.adapter.socket_mode.async_handler,
|
||||
),
|
||||
("slack_sdk", slack_sdk),
|
||||
("slack_sdk.web", slack_sdk.web),
|
||||
("slack_sdk.web.async_client", slack_sdk.web.async_client),
|
||||
]:
|
||||
sys.modules.setdefault(name, mod)
|
||||
|
||||
sys.modules.setdefault("aiohttp", MagicMock())
|
||||
|
||||
|
||||
_ensure_slack_mock()
|
||||
|
||||
import plugins.platforms.slack.adapter as _slack_mod # noqa: E402
|
||||
|
||||
_slack_mod.SLACK_AVAILABLE = True
|
||||
|
||||
from plugins.platforms.slack.adapter import SlackAdapter # noqa: E402
|
||||
from gateway.config import PlatformConfig # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal stand-ins for the slack_sdk objects involved in teardown
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Stands in for the ``aiohttp.ClientSession`` SocketModeClient holds."""
|
||||
|
||||
def __init__(self, client=None) -> None:
|
||||
self.closed = False
|
||||
self.reachable = False
|
||||
self.ws_connect_after_close = 0
|
||||
self._client = client
|
||||
self.live_tasks_at_close: list = []
|
||||
|
||||
async def ws_connect(self):
|
||||
if self.closed:
|
||||
# This is the exact failure recorded in #46990.
|
||||
self.ws_connect_after_close += 1
|
||||
raise RuntimeError("Session is closed")
|
||||
if not self.reachable:
|
||||
raise ConnectionError("connection refused")
|
||||
return object()
|
||||
|
||||
async def close(self) -> None:
|
||||
# Record which client tasks were still alive at the instant the shared
|
||||
# session went away. Anything listed here could be inside connect().
|
||||
if self._client is not None:
|
||||
self.live_tasks_at_close = self._client.live_task_names()
|
||||
self.closed = True
|
||||
# Closing a real session performs I/O and yields control back to the
|
||||
# loop, which is what gives a surviving retry task a chance to run.
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
class _FakeSocketModeClient:
|
||||
"""Mirrors the parts of SocketModeClient that matter during teardown."""
|
||||
|
||||
_TASK_ATTRS = ("message_processor", "current_session_monitor", "message_receiver")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.aiohttp_client_session = _FakeSession(self)
|
||||
self.closed = False
|
||||
self.close_should_raise = False
|
||||
self.message_processor = None
|
||||
self.current_session_monitor = None
|
||||
self.message_receiver = None
|
||||
|
||||
def live_task_names(self) -> list:
|
||||
return [
|
||||
attr
|
||||
for attr in self._TASK_ATTRS
|
||||
if getattr(self, attr) is not None and not getattr(self, attr).done()
|
||||
]
|
||||
|
||||
async def connect_to_new_endpoint(self) -> None:
|
||||
# monitor_current_session() (on staleness) and receive_messages() (on a
|
||||
# CLOSE frame) both reach connect() through here, independently.
|
||||
await self.connect()
|
||||
|
||||
async def monitor_current_session(self) -> None:
|
||||
while not self.closed:
|
||||
await asyncio.sleep(0.001)
|
||||
await self.connect_to_new_endpoint()
|
||||
|
||||
async def connect(self) -> None:
|
||||
# Mirrors SocketModeClient.connect(): ``while True`` with a broad
|
||||
# ``except Exception``, so neither the closed flag nor a closed session
|
||||
# ends the loop.
|
||||
while True:
|
||||
try:
|
||||
await self.aiohttp_client_session.ws_connect()
|
||||
return
|
||||
except Exception:
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
async def close(self) -> None:
|
||||
self.closed = True
|
||||
if self.close_should_raise:
|
||||
# SocketModeClient.close() calls disconnect() before it cancels its
|
||||
# background tasks. A broken session makes disconnect() raise, so
|
||||
# the SDK never reaches those cancel() calls at all.
|
||||
raise RuntimeError("Session is closed")
|
||||
for task in (
|
||||
self.message_processor,
|
||||
self.current_session_monitor,
|
||||
self.message_receiver,
|
||||
):
|
||||
if task is not None:
|
||||
# The SDK requests cancellation but never awaits it.
|
||||
task.cancel()
|
||||
await self.aiohttp_client_session.close()
|
||||
|
||||
|
||||
class _FakeHandler:
|
||||
"""Stands in for AsyncSocketModeHandler."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.client = _FakeSocketModeClient()
|
||||
|
||||
async def start_async(self) -> None:
|
||||
await self.client.connect()
|
||||
await asyncio.sleep(float("inf"))
|
||||
|
||||
async def close_async(self) -> None:
|
||||
await self.client.close()
|
||||
|
||||
|
||||
async def _spin() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def adapter():
|
||||
config = PlatformConfig(enabled=True, token="xoxb-fake-token")
|
||||
a = SlackAdapter(config)
|
||||
a._app = MagicMock()
|
||||
a._app_token = "xapp-fake"
|
||||
a._proxy_url = None
|
||||
a._running = True
|
||||
a.handle_message = AsyncMock()
|
||||
return a
|
||||
|
||||
|
||||
def _attach(adapter, handler):
|
||||
"""Wire a handler into the adapter the way _start_socket_mode_handler does."""
|
||||
adapter._handler = handler
|
||||
task = asyncio.create_task(handler.start_async())
|
||||
adapter._socket_mode_task = task
|
||||
return task
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSocketModeTeardown:
|
||||
@pytest.mark.asyncio
|
||||
async def test_socket_task_stops_before_session_is_closed(self, adapter):
|
||||
"""The socket task must be stopped before close_async() kills the session.
|
||||
|
||||
The task is parked in the SDK's connect() retry loop, which is the state
|
||||
#46990 describes. If teardown closes the shared session first, that loop
|
||||
wakes up and retries against a session that is already gone.
|
||||
"""
|
||||
handler = _FakeHandler()
|
||||
task = _attach(adapter, handler)
|
||||
# Let the task settle into the retry loop.
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await adapter._stop_socket_mode_handler()
|
||||
# Give anything that survived a chance to make itself known.
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
session = handler.client.aiohttp_client_session
|
||||
assert session.ws_connect_after_close == 0, (
|
||||
"the old socket task retried against a closed session "
|
||||
f"{session.ws_connect_after_close} time(s) after close_async()"
|
||||
)
|
||||
assert task.done(), "the old socket task outlived teardown"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_background_tasks_do_not_outlive_teardown(self, adapter):
|
||||
"""Old-client background tasks must be cancelled even if close_async() fails.
|
||||
|
||||
SocketModeClient.close() cancels message_processor,
|
||||
current_session_monitor and message_receiver only after disconnect()
|
||||
returns, so a raising disconnect() leaves all three running. The adapter
|
||||
logs and moves on, so it has to clean them up itself.
|
||||
"""
|
||||
handler = _FakeHandler()
|
||||
client = handler.client
|
||||
client.close_should_raise = True
|
||||
client.message_processor = asyncio.create_task(_spin())
|
||||
client.current_session_monitor = asyncio.create_task(_spin())
|
||||
client.message_receiver = asyncio.create_task(_spin())
|
||||
|
||||
_attach(adapter, handler)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await adapter._stop_socket_mode_handler()
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
for name in (
|
||||
"message_processor",
|
||||
"current_session_monitor",
|
||||
"message_receiver",
|
||||
):
|
||||
assert getattr(client, name).done(), f"{name} outlived teardown"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_tasks_are_dead_before_the_session_closes(self, adapter):
|
||||
"""Nothing may still be inside connect() when the shared session closes.
|
||||
|
||||
monitor_current_session() and receive_messages() each reach
|
||||
connect_to_new_endpoint() on their own, and connect() rebinds
|
||||
current_session_monitor and message_receiver to fresh tasks on success.
|
||||
The live task set therefore changes across the awaits inside
|
||||
SocketModeClient.close(), so cancelling from a snapshot taken partway
|
||||
through races a moving target. Everything has to be stopped before the
|
||||
session is closed. See slackapi/python-slack-sdk#1913.
|
||||
"""
|
||||
handler = _FakeHandler()
|
||||
client = handler.client
|
||||
client.message_processor = asyncio.create_task(_spin())
|
||||
client.current_session_monitor = asyncio.create_task(
|
||||
client.monitor_current_session()
|
||||
)
|
||||
client.message_receiver = asyncio.create_task(client.monitor_current_session())
|
||||
|
||||
_attach(adapter, handler)
|
||||
# Let both reconnect loops settle inside connect().
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await adapter._stop_socket_mode_handler()
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
session = client.aiohttp_client_session
|
||||
assert session.live_tasks_at_close == [], (
|
||||
"client tasks were still running when the shared session was closed: "
|
||||
f"{session.live_tasks_at_close}"
|
||||
)
|
||||
assert session.ws_connect_after_close == 0, (
|
||||
"a client task retried against a closed session "
|
||||
f"{session.ws_connect_after_close} time(s)"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_clears_adapter_state(self, adapter):
|
||||
"""Teardown always drops its references, even when close_async() raises."""
|
||||
handler = _FakeHandler()
|
||||
handler.client.close_should_raise = True
|
||||
_attach(adapter, handler)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await adapter._stop_socket_mode_handler()
|
||||
|
||||
assert adapter._handler is None
|
||||
assert adapter._socket_mode_task is None
|
||||
|
||||
|
||||
class TestSocketModeRestart:
|
||||
@pytest.mark.asyncio
|
||||
async def test_restart_stops_old_handler_before_starting_new_one(self, adapter):
|
||||
"""A reconnect must fully retire the old handler before replacing it."""
|
||||
old = _FakeHandler()
|
||||
old_task = _attach(adapter, old)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
started: list[str] = []
|
||||
|
||||
def _fake_start() -> None:
|
||||
started.append("started")
|
||||
assert old_task.done(), (
|
||||
"the replacement handler was created while the old socket task "
|
||||
"was still running"
|
||||
)
|
||||
|
||||
with patch.object(adapter, "_start_socket_mode_handler", _fake_start):
|
||||
await adapter._restart_socket_mode("transport disconnected")
|
||||
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
assert started == ["started"]
|
||||
assert old.client.aiohttp_client_session.ws_connect_after_close == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_watchdog_restarts_when_socket_task_stops(self, adapter):
|
||||
"""The existing watchdog triggers still fire after the teardown change."""
|
||||
done_task = MagicMock()
|
||||
done_task.done.return_value = True
|
||||
adapter._socket_mode_task = done_task
|
||||
|
||||
reasons: list[str] = []
|
||||
|
||||
async def _fake_restart(reason: str) -> None:
|
||||
reasons.append(reason)
|
||||
adapter._running = False
|
||||
|
||||
adapter._restart_socket_mode = _fake_restart
|
||||
adapter._socket_transport_connected = AsyncMock(return_value=None)
|
||||
adapter._socket_watchdog_interval_s = 0.01
|
||||
|
||||
await adapter._socket_watchdog_loop()
|
||||
|
||||
assert reasons == ["socket task stopped"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_watchdog_restarts_when_transport_disconnected(self, adapter):
|
||||
"""A transport that reports itself down still triggers a reconnect."""
|
||||
live_task = MagicMock()
|
||||
live_task.done.return_value = False
|
||||
adapter._socket_mode_task = live_task
|
||||
adapter._handler = MagicMock()
|
||||
|
||||
reasons: list[str] = []
|
||||
|
||||
async def _fake_restart(reason: str) -> None:
|
||||
reasons.append(reason)
|
||||
adapter._running = False
|
||||
|
||||
adapter._restart_socket_mode = _fake_restart
|
||||
adapter._socket_transport_connected = AsyncMock(return_value=False)
|
||||
adapter._socket_watchdog_interval_s = 0.01
|
||||
|
||||
await adapter._socket_watchdog_loop()
|
||||
|
||||
assert reasons == ["transport disconnected"]
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Regression tests for #63530 — Slack adapter drops human replies in
|
||||
threads whose root was posted by the bot via direct chat.postMessage
|
||||
(outside the gateway's send() path).
|
||||
|
||||
Background: the adapter's wake-decision at the un-mentioned branch in
|
||||
_handle_slack_message uses three checks:
|
||||
|
||||
1. thread_ts ∈ _bot_message_ts (only populated by send() / files_upload_v2)
|
||||
2. thread_ts ∈ _mentioned_threads (only populated on @mention)
|
||||
3. _has_active_session_for_thread(...) (survives restarts)
|
||||
|
||||
When a skill posts a triage message into a Slack thread via the Web API
|
||||
directly (chat.postMessage, no gateway run), the bot's own ts is NOT
|
||||
recorded in _bot_message_ts. A human reply in that thread, without an
|
||||
@-mention and without an existing session, falls through all three
|
||||
checks and is silently dropped. The same gap opens after a gateway
|
||||
restart: _bot_message_ts is process memory, so threads the bot started
|
||||
before the restart no longer wake it.
|
||||
|
||||
Fix: a 4th check — was the thread root authored by the bot? Root
|
||||
authorship is derived from the Slack API (conversations.replies), so it
|
||||
survives restarts, unlike the in-memory ts set. The wake decision is
|
||||
extracted into _should_wake_on_unmentioned_message so it's directly
|
||||
testable without spinning up Slack.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# Mock slack-bolt / slack-sdk the same way test_slack_mention.py does.
|
||||
def _ensure_slack_mock():
|
||||
if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"):
|
||||
return
|
||||
slack_bolt = MagicMock()
|
||||
slack_bolt.async_app.AsyncApp = MagicMock
|
||||
slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock
|
||||
slack_sdk = MagicMock()
|
||||
slack_sdk.web.async_client.AsyncWebClient = MagicMock
|
||||
for name, mod in [
|
||||
("slack_bolt", slack_bolt),
|
||||
("slack_bolt.async_app", slack_bolt.async_app),
|
||||
("slack_bolt.adapter", slack_bolt.adapter),
|
||||
("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode),
|
||||
(
|
||||
"slack_bolt.adapter.socket_mode.async_handler",
|
||||
slack_bolt.adapter.socket_mode.async_handler,
|
||||
),
|
||||
("slack_sdk", slack_sdk),
|
||||
("slack_sdk.web", slack_sdk.web),
|
||||
("slack_sdk.web.async_client", slack_sdk.web.async_client),
|
||||
]:
|
||||
sys.modules.setdefault(name, mod)
|
||||
sys.modules.setdefault("aiohttp", MagicMock())
|
||||
|
||||
|
||||
_ensure_slack_mock()
|
||||
|
||||
import plugins.platforms.slack.adapter as _slack_mod # noqa: E402
|
||||
|
||||
_slack_mod.SLACK_AVAILABLE = True
|
||||
|
||||
from plugins.platforms.slack.adapter import ( # noqa: E402
|
||||
SlackAdapter,
|
||||
_ThreadContextCache,
|
||||
)
|
||||
|
||||
from gateway.config import Platform, PlatformConfig # noqa: E402
|
||||
|
||||
|
||||
BOT_USER_ID = "U_BOT_OWN"
|
||||
CHANNEL_ID = "C_incident"
|
||||
USER_ID = "U_engineer"
|
||||
THREAD_TS = "1700000000.000100"
|
||||
|
||||
|
||||
def _make_adapter(bot_authored_root: bool = False):
|
||||
"""Build a bare SlackAdapter with the wake-decision state controlled.
|
||||
|
||||
None of the 3 legacy in-memory checks pass by default: the bot didn't
|
||||
send via gateway, the thread wasn't @-mentioned, and there is no active
|
||||
session — exactly the post-restart / outside-send state.
|
||||
"""
|
||||
adapter = object.__new__(SlackAdapter)
|
||||
adapter.platform = Platform.SLACK
|
||||
adapter.config = PlatformConfig(
|
||||
enabled=True,
|
||||
extra={"require_mention": True, "strict_mention": False},
|
||||
)
|
||||
adapter._bot_user_id = BOT_USER_ID
|
||||
adapter._team_bot_user_ids = {}
|
||||
adapter._bot_message_ts = set()
|
||||
adapter._mentioned_threads = set()
|
||||
adapter._thread_context_cache = {}
|
||||
|
||||
adapter._has_active_session_for_thread = lambda **kw: False
|
||||
# Mock _fetch_thread_context so the miss-path doesn't make a real
|
||||
# Slack API call. Tests that need a populated cache pre-populate
|
||||
# _thread_context_cache directly.
|
||||
adapter._fetch_thread_context = AsyncMock(return_value="")
|
||||
# The 4th-check helper is mocked so wake-decision tests can control
|
||||
# its result without setting up the full cache path. Helper-specific
|
||||
# tests call the real method via the class instead.
|
||||
adapter._bot_authored_thread_root = AsyncMock(return_value=bot_authored_root)
|
||||
|
||||
return adapter
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _should_wake_on_unmentioned_message — composes all 4 checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_false_when_not_thread_reply():
|
||||
"""A top-level channel message (no thread_ts) should never wake the bot
|
||||
when require_mention is true — unchanged by this fix."""
|
||||
adapter = _make_adapter(bot_authored_root=True)
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=None,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=False,
|
||||
)
|
||||
assert wake is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_false_when_all_four_checks_miss():
|
||||
"""All four checks miss (no bot-message, no mention, no session, no
|
||||
bot-authored root) → wake decision is False."""
|
||||
adapter = _make_adapter(bot_authored_root=False)
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_true_when_bot_authored_thread_root():
|
||||
"""The new behavior (#63530): a human reply in a thread whose root was
|
||||
authored by the bot via direct chat.postMessage (outside gateway send)
|
||||
wakes the bot even though none of the legacy 3 checks pass — including
|
||||
after a restart, when _bot_message_ts is empty."""
|
||||
adapter = _make_adapter(bot_authored_root=True)
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is True, (
|
||||
"human reply in a thread whose root was bot-posted (not via gateway "
|
||||
"send) should wake the bot — #63530"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_true_when_legacy_check_1_hits():
|
||||
"""Regression guard: _bot_message_ts hit still wakes (additive check)."""
|
||||
adapter = _make_adapter(bot_authored_root=False)
|
||||
adapter._bot_message_ts = {THREAD_TS}
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_true_when_legacy_check_2_hits():
|
||||
"""Regression guard: _mentioned_threads hit still wakes (additive)."""
|
||||
adapter = _make_adapter(bot_authored_root=False)
|
||||
adapter._mentioned_threads = {THREAD_TS}
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wake_decision_returns_true_when_legacy_check_3_hits():
|
||||
"""Regression guard: an active session still wakes (additive)."""
|
||||
adapter = _make_adapter(bot_authored_root=False)
|
||||
adapter._has_active_session_for_thread = lambda **kw: True
|
||||
wake = await adapter._should_wake_on_unmentioned_message(
|
||||
event_thread_ts=THREAD_TS,
|
||||
channel_id=CHANNEL_ID,
|
||||
user_id=USER_ID,
|
||||
is_thread_reply=True,
|
||||
)
|
||||
assert wake is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _bot_authored_thread_root — the API-derived, restart-surviving check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_true_from_cache():
|
||||
"""Cache hit whose parent_user_id matches the bot's user_id → True."""
|
||||
adapter = _make_adapter()
|
||||
adapter._thread_context_cache = {
|
||||
f"{CHANNEL_ID}:{THREAD_TS}:": _ThreadContextCache(
|
||||
content="[Thread context — prior messages...]",
|
||||
fetched_at=0,
|
||||
message_count=1,
|
||||
parent_text="triage analysis",
|
||||
parent_user_id=BOT_USER_ID,
|
||||
),
|
||||
}
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS
|
||||
)
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_false_for_human_authored_root():
|
||||
"""A human-authored root must return False even on a cache hit — guards
|
||||
against waking on any thread reply just because the cache is warm."""
|
||||
adapter = _make_adapter()
|
||||
adapter._thread_context_cache = {
|
||||
f"{CHANNEL_ID}:{THREAD_TS}:": _ThreadContextCache(
|
||||
content="[Thread context — prior messages...]",
|
||||
fetched_at=0,
|
||||
message_count=1,
|
||||
parent_text="someone else's message",
|
||||
parent_user_id="U_other_user",
|
||||
),
|
||||
}
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS
|
||||
)
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_false_on_empty_thread_ts():
|
||||
"""Defensive: empty thread_ts short-circuits to False without any
|
||||
cache lookup or network call."""
|
||||
adapter = _make_adapter()
|
||||
result = await SlackAdapter._bot_authored_thread_root(adapter, CHANNEL_ID, "")
|
||||
assert result is False
|
||||
adapter._fetch_thread_context.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_fetches_on_cache_miss():
|
||||
"""Cache miss → _fetch_thread_context runs; a successful fetch that
|
||||
populates parent_user_id with the bot's id yields True. This is the
|
||||
restart path: fresh process, empty caches, root authorship recovered
|
||||
from the Slack API."""
|
||||
adapter = _make_adapter()
|
||||
|
||||
async def _fake_fetch(channel_id, thread_ts, current_ts, team_id=""):
|
||||
adapter._thread_context_cache[f"{channel_id}:{thread_ts}:{team_id}"] = (
|
||||
_ThreadContextCache(
|
||||
content="ctx",
|
||||
fetched_at=0,
|
||||
message_count=1,
|
||||
parent_text="bot-posted root",
|
||||
parent_user_id=BOT_USER_ID,
|
||||
)
|
||||
)
|
||||
return "ctx"
|
||||
|
||||
adapter._fetch_thread_context = AsyncMock(side_effect=_fake_fetch)
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS
|
||||
)
|
||||
assert result is True
|
||||
adapter._fetch_thread_context.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_false_when_fetch_fails():
|
||||
"""Fetch failure (empty result, nothing cached) → False, no wake."""
|
||||
adapter = _make_adapter()
|
||||
adapter._fetch_thread_context = AsyncMock(return_value="")
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS
|
||||
)
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bot_authored_thread_root_uses_per_team_bot_id():
|
||||
"""Multi-workspace: the comparison must use the team's bot user id,
|
||||
not the primary workspace's."""
|
||||
adapter = _make_adapter()
|
||||
adapter._team_bot_user_ids = {"T2": "U_BOT_T2"}
|
||||
adapter._thread_context_cache = {
|
||||
f"{CHANNEL_ID}:{THREAD_TS}:T2": _ThreadContextCache(
|
||||
content="ctx",
|
||||
fetched_at=0,
|
||||
message_count=1,
|
||||
parent_text="root",
|
||||
parent_user_id="U_BOT_T2",
|
||||
),
|
||||
}
|
||||
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS, team_id="T2"
|
||||
)
|
||||
assert result is True
|
||||
# And the primary bot id must NOT match in that workspace.
|
||||
adapter._thread_context_cache[f"{CHANNEL_ID}:{THREAD_TS}:T2"].parent_user_id = (
|
||||
BOT_USER_ID
|
||||
)
|
||||
result = await SlackAdapter._bot_authored_thread_root(
|
||||
adapter, CHANNEL_ID, THREAD_TS, team_id="T2"
|
||||
)
|
||||
assert result is False
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Tests for Telegram connect() non-retryable fatal error on missing credentials.
|
||||
|
||||
When Telegram has no bot token or no python-telegram-bot installed, connect()
|
||||
must set a non-retryable fatal error so the gateway does not queue it for
|
||||
background reconnection (#31049).
|
||||
"""
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from gateway.config import PlatformConfig
|
||||
|
||||
|
||||
def _ensure_telegram_mock():
|
||||
if "telegram" in sys.modules and hasattr(sys.modules["telegram"], "__file__"):
|
||||
return
|
||||
|
||||
telegram_mod = MagicMock()
|
||||
telegram_mod.ext.ContextTypes.DEFAULT_TYPE = type(None)
|
||||
telegram_mod.constants.ParseMode.MARKDOWN_V2 = "MarkdownV2"
|
||||
telegram_mod.constants.ChatType.GROUP = "group"
|
||||
telegram_mod.constants.ChatType.SUPERGROUP = "supergroup"
|
||||
telegram_mod.constants.ChatType.CHANNEL = "channel"
|
||||
telegram_mod.constants.ChatType.PRIVATE = "private"
|
||||
|
||||
telegram_mod.error.NetworkError = type("NetworkError", (OSError,), {})
|
||||
telegram_mod.error.TimedOut = type("TimedOut", (OSError,), {})
|
||||
telegram_mod.error.BadRequest = type("BadRequest", (Exception,), {})
|
||||
|
||||
for name in ("telegram", "telegram.ext", "telegram.constants", "telegram.request"):
|
||||
sys.modules.setdefault(name, telegram_mod)
|
||||
sys.modules.setdefault("telegram.error", telegram_mod.error)
|
||||
|
||||
|
||||
_ensure_telegram_mock()
|
||||
|
||||
import plugins.platforms.telegram.adapter as telegram_mod # noqa: E402
|
||||
from plugins.platforms.telegram.adapter import TelegramAdapter # noqa: E402
|
||||
|
||||
|
||||
class TestTelegramUnconfiguredNonRetryable:
|
||||
"""Verify that missing dependency/token sets a non-retryable fatal error."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_telegram_lib_sets_non_retryable_fatal(self, monkeypatch):
|
||||
"""connect() with python-telegram-bot unavailable → non-retryable fatal error."""
|
||||
adapter = TelegramAdapter(PlatformConfig(enabled=True, token="fake"))
|
||||
monkeypatch.setattr(telegram_mod, "TELEGRAM_AVAILABLE", False)
|
||||
result = await adapter.connect()
|
||||
assert result is False
|
||||
assert adapter.has_fatal_error is True
|
||||
assert adapter.fatal_error_retryable is False
|
||||
assert adapter.fatal_error_code == "missing_dependency"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_bot_token_sets_non_retryable_fatal(self, monkeypatch):
|
||||
"""connect() with empty token → non-retryable fatal error."""
|
||||
monkeypatch.setattr(telegram_mod, "TELEGRAM_AVAILABLE", True)
|
||||
adapter = TelegramAdapter(PlatformConfig(enabled=True, token=""))
|
||||
result = await adapter.connect()
|
||||
assert result is False
|
||||
assert adapter.has_fatal_error is True
|
||||
assert adapter.fatal_error_retryable is False
|
||||
assert adapter.fatal_error_code == "missing_credentials"
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Tests for the dashboard plugin-catalog surface in hermes_cli.web_server.
|
||||
|
||||
Covers:
|
||||
- GET /api/dashboard/plugins/catalog — entry serialization, installed-state
|
||||
merge (via the ``.hermes-catalog.json`` sidecar), removed list exposure.
|
||||
- POST /api/dashboard/agent-plugins/install — removed-blocklist refusal for
|
||||
raw identifiers AND catalog names, catalog_name resolution to a pinned-ref
|
||||
install, sidecar write.
|
||||
- /api/dashboard/plugins/hub — ``removed_reason`` annotation on rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
VALID_SHA = "38fe0fb53eff98d477f807432e965429e665ca33"
|
||||
OTHER_SHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
|
||||
def _write_entry(catalog_dir: Path, name: str, **overrides) -> dict:
|
||||
data = {
|
||||
"name": name,
|
||||
"repo": f"https://github.com/example/{name}",
|
||||
"sha": VALID_SHA,
|
||||
"description": f"Test entry {name}.",
|
||||
"maintainer": "Example",
|
||||
"tier": "official",
|
||||
"docs_url": f"https://example.com/docs/{name}",
|
||||
"capabilities": {
|
||||
"provides_tools": ["tool_a"],
|
||||
"provides_hooks": ["hook_b"],
|
||||
"provides_middleware": [],
|
||||
"requires_env": ["EXAMPLE_API_KEY"],
|
||||
},
|
||||
}
|
||||
data.update(overrides)
|
||||
catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
(catalog_dir / f"{name}.yaml").write_text(
|
||||
yaml.safe_dump(data), encoding="utf-8"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def _write_removed(catalog_dir: Path, removed: list) -> None:
|
||||
catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
(catalog_dir / "removed.yaml").write_text(
|
||||
yaml.safe_dump({"removed": removed}), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def _make_installed_plugin(name: str, sidecar: dict | None = None) -> Path:
|
||||
"""Drop a minimal plugin dir under the isolated HERMES_HOME."""
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
plugin_dir = get_hermes_home() / "plugins" / name
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_dir / "plugin.yaml").write_text(
|
||||
yaml.safe_dump({"name": name, "version": "1.0", "description": "x"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
if sidecar is not None:
|
||||
(plugin_dir / ".hermes-catalog.json").write_text(
|
||||
json.dumps(sidecar), encoding="utf-8"
|
||||
)
|
||||
return plugin_dir
|
||||
|
||||
|
||||
class TestDashboardPluginCatalog:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _setup(self, monkeypatch, tmp_path, _isolate_hermes_home):
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip("fastapi/starlette not installed")
|
||||
|
||||
import hermes_state
|
||||
from hermes_constants import get_hermes_home
|
||||
from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN
|
||||
|
||||
monkeypatch.setattr(
|
||||
hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db"
|
||||
)
|
||||
|
||||
self.catalog_dir = tmp_path / "catalog"
|
||||
self.catalog_dir.mkdir()
|
||||
monkeypatch.setenv("HERMES_PLUGIN_CATALOG_DIR", str(self.catalog_dir))
|
||||
|
||||
self.client = TestClient(app)
|
||||
self.client.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN
|
||||
|
||||
# ── GET /api/dashboard/plugins/catalog ──────────────────────────────
|
||||
|
||||
def test_catalog_endpoint_requires_token(self):
|
||||
from starlette.testclient import TestClient
|
||||
from hermes_cli.web_server import app
|
||||
|
||||
unauth = TestClient(app)
|
||||
resp = unauth.get("/api/dashboard/plugins/catalog")
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_catalog_endpoint_shape(self):
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
_write_removed(
|
||||
self.catalog_dir,
|
||||
[{"name": "bad-plugin", "repo": "https://github.com/evil/bad-plugin",
|
||||
"reason": "exfiltrated env vars", "date": "2026-07-02"}],
|
||||
)
|
||||
|
||||
resp = self.client.get("/api/dashboard/plugins/catalog")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
|
||||
assert "generated_at" in data
|
||||
assert isinstance(data["entries"], list) and len(data["entries"]) == 1
|
||||
entry = data["entries"][0]
|
||||
assert entry["name"] == "alpha-plugin"
|
||||
assert entry["repo"] == "https://github.com/example/alpha-plugin"
|
||||
assert entry["sha"] == VALID_SHA
|
||||
assert entry["sha_short"] == VALID_SHA[:7]
|
||||
assert entry["tier"] == "official"
|
||||
assert entry["maintainer"] == "Example"
|
||||
assert entry["docs_url"] == "https://example.com/docs/alpha-plugin"
|
||||
assert entry["capabilities"]["provides_tools"] == ["tool_a"]
|
||||
assert entry["capabilities"]["requires_env"] == ["EXAMPLE_API_KEY"]
|
||||
assert "tool_a" in entry["capability_summary"]
|
||||
# Not installed → degraded install state.
|
||||
assert entry["installed"] is False
|
||||
assert entry["installed_sha"] is None
|
||||
assert entry["update_available"] is False
|
||||
assert entry["runtime_status"] is None
|
||||
|
||||
assert len(data["removed"]) == 1
|
||||
removed = data["removed"][0]
|
||||
assert removed["name"] == "bad-plugin"
|
||||
assert removed["reason"] == "exfiltrated env vars"
|
||||
|
||||
def test_catalog_installed_state_merge_with_sidecar(self):
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
_make_installed_plugin(
|
||||
"alpha-plugin",
|
||||
sidecar={
|
||||
"catalog_name": "alpha-plugin",
|
||||
"repo": "https://github.com/example/alpha-plugin",
|
||||
"sha": OTHER_SHA,
|
||||
"installed_at": "2026-07-01T00:00:00Z",
|
||||
"tier": "official",
|
||||
},
|
||||
)
|
||||
|
||||
resp = self.client.get("/api/dashboard/plugins/catalog")
|
||||
assert resp.status_code == 200
|
||||
entry = resp.json()["entries"][0]
|
||||
assert entry["installed"] is True
|
||||
assert entry["installed_sha"] == OTHER_SHA
|
||||
assert entry["update_available"] is True
|
||||
assert entry["runtime_status"] == "inactive"
|
||||
|
||||
def test_catalog_installed_no_sidecar_degrades_to_null_sha(self):
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
_make_installed_plugin("alpha-plugin", sidecar=None)
|
||||
|
||||
resp = self.client.get("/api/dashboard/plugins/catalog")
|
||||
entry = resp.json()["entries"][0]
|
||||
assert entry["installed"] is True
|
||||
assert entry["installed_sha"] is None
|
||||
assert entry["update_available"] is False
|
||||
|
||||
def test_catalog_installed_same_sha_no_update(self):
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
_make_installed_plugin(
|
||||
"alpha-plugin",
|
||||
sidecar={
|
||||
"catalog_name": "alpha-plugin",
|
||||
"repo": "https://github.com/example/alpha-plugin",
|
||||
"sha": VALID_SHA,
|
||||
"installed_at": "2026-07-01T00:00:00Z",
|
||||
"tier": "official",
|
||||
},
|
||||
)
|
||||
|
||||
entry = self.client.get("/api/dashboard/plugins/catalog").json()["entries"][0]
|
||||
assert entry["installed"] is True
|
||||
assert entry["installed_sha"] == VALID_SHA
|
||||
assert entry["update_available"] is False
|
||||
|
||||
# ── POST /api/dashboard/agent-plugins/install ───────────────────────
|
||||
|
||||
def test_install_refuses_removed_raw_identifier(self):
|
||||
_write_removed(
|
||||
self.catalog_dir,
|
||||
[{"name": "bad-plugin", "repo": "https://github.com/evil/bad-plugin",
|
||||
"reason": "exfiltrated env vars", "date": "2026-07-02"}],
|
||||
)
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": "https://github.com/evil/bad-plugin"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "exfiltrated env vars" in resp.json()["detail"]
|
||||
|
||||
def test_install_refuses_removed_catalog_name(self):
|
||||
_write_removed(
|
||||
self.catalog_dir,
|
||||
[{"name": "bad-plugin", "reason": "policy violation",
|
||||
"date": "2026-07-02"}],
|
||||
)
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": "", "catalog_name": "bad-plugin"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "policy violation" in resp.json()["detail"]
|
||||
|
||||
def test_install_unknown_catalog_name_is_400(self):
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": "", "catalog_name": "does-not-exist"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "does-not-exist" in resp.json()["detail"]
|
||||
|
||||
def test_install_missing_identifier_and_catalog_name_is_400(self):
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": ""},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_catalog_name_install_resolves_pinned_ref_and_writes_sidecar(
|
||||
self, monkeypatch, tmp_path
|
||||
):
|
||||
from hermes_constants import get_hermes_home
|
||||
import hermes_cli.plugins_cmd as plugins_cmd
|
||||
|
||||
_write_entry(self.catalog_dir, "alpha-plugin")
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_core(identifier, *, force, ref=None, skip_removed_check=False):
|
||||
captured["identifier"] = identifier
|
||||
captured["ref"] = ref
|
||||
target = get_hermes_home() / "plugins" / "alpha-plugin"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
(target / "plugin.yaml").write_text(
|
||||
yaml.safe_dump({"name": "alpha-plugin"}), encoding="utf-8"
|
||||
)
|
||||
return target, {"name": "alpha-plugin"}, "alpha-plugin"
|
||||
|
||||
monkeypatch.setattr(plugins_cmd, "_install_plugin_core", fake_core)
|
||||
|
||||
resp = self.client.post(
|
||||
"/api/dashboard/agent-plugins/install",
|
||||
json={"identifier": "", "catalog_name": "alpha-plugin",
|
||||
"enable": False},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["ok"] is True
|
||||
assert body["plugin_name"] == "alpha-plugin"
|
||||
|
||||
assert captured["ref"] == VALID_SHA
|
||||
assert captured["identifier"].startswith(
|
||||
"https://github.com/example/alpha-plugin"
|
||||
)
|
||||
|
||||
sidecar_path = (
|
||||
get_hermes_home() / "plugins" / "alpha-plugin" / ".hermes-catalog.json"
|
||||
)
|
||||
assert sidecar_path.is_file()
|
||||
sidecar = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||
assert sidecar["catalog_name"] == "alpha-plugin"
|
||||
assert sidecar["repo"] == "https://github.com/example/alpha-plugin"
|
||||
assert sidecar["sha"] == VALID_SHA
|
||||
assert sidecar["tier"] == "official"
|
||||
assert sidecar["installed_at"]
|
||||
|
||||
# ── /api/dashboard/plugins/hub removed_reason annotation ─────────────
|
||||
|
||||
def test_hub_rows_annotated_with_removed_reason(self):
|
||||
_write_removed(
|
||||
self.catalog_dir,
|
||||
[{"name": "bad-plugin", "reason": "supply chain incident",
|
||||
"date": "2026-07-02"}],
|
||||
)
|
||||
_make_installed_plugin("bad-plugin")
|
||||
_make_installed_plugin("good-plugin")
|
||||
|
||||
resp = self.client.get("/api/dashboard/plugins/hub")
|
||||
assert resp.status_code == 200
|
||||
rows = {r["name"]: r for r in resp.json()["plugins"]}
|
||||
assert rows["bad-plugin"]["removed_reason"] == "supply chain incident"
|
||||
assert rows["good-plugin"]["removed_reason"] is None
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Behavior tests for scripts/validate_plugin_catalog.py.
|
||||
|
||||
The script is the no-install structural validator used by the plugin-catalog
|
||||
admission CI: it must run with only stdlib + pyyaml, take file paths or a
|
||||
directory, exit 0/1, and support --json machine output. These tests exercise
|
||||
the CLI contract via subprocess (the same way CI invokes it).
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = REPO_ROOT / "scripts" / "validate_plugin_catalog.py"
|
||||
|
||||
VALID_ENTRY = {
|
||||
"name": "example-plugin",
|
||||
"repo": "https://github.com/NousResearch/hermes-example-plugins",
|
||||
"sha": "38fe0fb53eff98d477f807432e965429e665ca33",
|
||||
"subdir": "",
|
||||
"description": "One-line description.",
|
||||
"maintainer": "NousResearch",
|
||||
"tier": "official",
|
||||
"requires_hermes": ">=0.19",
|
||||
"docs_url": "",
|
||||
"platforms": [],
|
||||
"capabilities": {
|
||||
"provides_tools": ["example_tool"],
|
||||
"provides_hooks": [],
|
||||
"provides_middleware": [],
|
||||
"requires_env": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_entry(tmp_path: Path, data: dict, filename: str | None = None) -> Path:
|
||||
name = filename or f"{data.get('name', 'entry')}.yaml"
|
||||
path = tmp_path / name
|
||||
path.write_text(yaml.safe_dump(data), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def run_validator(*args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(SCRIPT), *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
# ── valid input ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_valid_entry_passes(tmp_path):
|
||||
path = write_entry(tmp_path, VALID_ENTRY)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_valid_entry_without_optional_fields_passes(tmp_path):
|
||||
entry = {
|
||||
"name": "minimal-plugin",
|
||||
"repo": "https://github.com/example/minimal",
|
||||
"sha": "a" * 40,
|
||||
"description": "Minimal.",
|
||||
"maintainer": "someone",
|
||||
}
|
||||
path = write_entry(tmp_path, entry)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
# ── each malformed field fails with a pointed error ────────────────────
|
||||
|
||||
|
||||
def _expect_error(tmp_path, mutation: dict, expected_substring: str, drop: str = ""):
|
||||
entry = {**VALID_ENTRY, **mutation}
|
||||
if drop:
|
||||
entry.pop(drop, None)
|
||||
path = write_entry(tmp_path, entry, filename="entry.yaml")
|
||||
result = run_validator(str(path))
|
||||
combined = result.stdout + result.stderr
|
||||
assert result.returncode == 1, combined
|
||||
assert expected_substring in combined, combined
|
||||
assert "entry.yaml" in combined, combined
|
||||
|
||||
|
||||
def test_bad_name_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"name": "Bad Name!"}, "name")
|
||||
|
||||
|
||||
def test_name_too_long_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"name": "x" * 65}, "name")
|
||||
|
||||
|
||||
def test_non_https_repo_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"repo": "git@github.com:evil/x.git"}, "repo")
|
||||
|
||||
|
||||
def test_short_sha_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"sha": "abc123"}, "sha")
|
||||
|
||||
|
||||
def test_non_hex_sha_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"sha": "z" * 40}, "sha")
|
||||
|
||||
|
||||
def test_bad_tier_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"tier": "platinum"}, "tier")
|
||||
|
||||
|
||||
def test_empty_description_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"description": ""}, "description")
|
||||
|
||||
|
||||
def test_empty_maintainer_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"maintainer": ""}, "maintainer")
|
||||
|
||||
|
||||
def test_missing_required_field_fails(tmp_path):
|
||||
_expect_error(tmp_path, {}, "sha", drop="sha")
|
||||
|
||||
|
||||
def test_capabilities_value_not_a_list_fails(tmp_path):
|
||||
_expect_error(
|
||||
tmp_path,
|
||||
{"capabilities": {"provides_tools": "not-a-list"}},
|
||||
"provides_tools",
|
||||
)
|
||||
|
||||
|
||||
def test_capabilities_list_of_non_strings_fails(tmp_path):
|
||||
_expect_error(
|
||||
tmp_path,
|
||||
{"capabilities": {"requires_env": [1, 2]}},
|
||||
"requires_env",
|
||||
)
|
||||
|
||||
|
||||
def test_bad_requires_hermes_spec_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"requires_hermes": "banana"}, "requires_hermes")
|
||||
|
||||
|
||||
def test_comma_separated_requires_hermes_passes(tmp_path):
|
||||
entry = {**VALID_ENTRY, "requires_hermes": ">=0.19, <2.0"}
|
||||
path = write_entry(tmp_path, entry)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_unknown_platform_fails(tmp_path):
|
||||
_expect_error(tmp_path, {"platforms": ["linux", "amiga"]}, "platforms")
|
||||
|
||||
|
||||
def test_entry_not_a_mapping_fails(tmp_path):
|
||||
path = tmp_path / "entry.yaml"
|
||||
path.write_text("- just\n- a\n- list\n", encoding="utf-8")
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 1
|
||||
assert "mapping" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
# ── unknown top-level keys warn but do not fail ────────────────────────
|
||||
|
||||
|
||||
def test_unknown_key_warns_but_passes(tmp_path):
|
||||
entry = {**VALID_ENTRY, "future_field": "hello"}
|
||||
path = write_entry(tmp_path, entry)
|
||||
result = run_validator(str(path))
|
||||
combined = result.stdout + result.stderr
|
||||
assert result.returncode == 0, combined
|
||||
assert "future_field" in combined
|
||||
assert "warning" in combined.lower()
|
||||
|
||||
|
||||
# ── removed.yaml shape ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_valid_removed_yaml_passes(tmp_path):
|
||||
path = tmp_path / "removed.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"removed": [
|
||||
{
|
||||
"name": "some-plugin",
|
||||
"repo": "https://github.com/evil/some-plugin",
|
||||
"reason": "Exfiltrated env vars",
|
||||
"date": "2026-07-02",
|
||||
}
|
||||
]
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_removed_yaml_not_a_list_fails(tmp_path):
|
||||
path = tmp_path / "removed.yaml"
|
||||
path.write_text(yaml.safe_dump({"removed": "nope"}), encoding="utf-8")
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 1
|
||||
assert "removed" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
def test_removed_item_missing_name_fails(tmp_path):
|
||||
path = tmp_path / "removed.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump({"removed": [{"reason": "bad", "date": "2026-01-01"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = run_validator(str(path))
|
||||
assert result.returncode == 1
|
||||
assert "name" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
# ── --json machine output ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_json_output_shape_on_failure(tmp_path):
|
||||
bad = write_entry(tmp_path, {**VALID_ENTRY, "sha": "short"}, filename="bad.yaml")
|
||||
result = run_validator("--json", str(bad))
|
||||
assert result.returncode == 1
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["ok"] is False
|
||||
assert isinstance(payload["files"], list)
|
||||
entry = next(f for f in payload["files"] if f["path"].endswith("bad.yaml"))
|
||||
assert entry["ok"] is False
|
||||
assert any("sha" in e for e in entry["errors"])
|
||||
|
||||
|
||||
def test_json_output_shape_on_success_with_warning(tmp_path):
|
||||
good = write_entry(tmp_path, {**VALID_ENTRY, "future_field": 1})
|
||||
result = run_validator("--json", str(good))
|
||||
assert result.returncode == 0
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["ok"] is True
|
||||
(entry,) = payload["files"]
|
||||
assert entry["ok"] is True
|
||||
assert entry["errors"] == []
|
||||
assert any("future_field" in w for w in entry["warnings"])
|
||||
|
||||
|
||||
# ── directory mode ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_directory_mode_validates_all_entries_and_removed(tmp_path):
|
||||
write_entry(tmp_path, VALID_ENTRY)
|
||||
write_entry(tmp_path, {**VALID_ENTRY, "name": "bad-one", "sha": "nope"})
|
||||
(tmp_path / "removed.yaml").write_text(
|
||||
yaml.safe_dump({"removed": [{"name": "gone", "reason": "test"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = run_validator(str(tmp_path))
|
||||
combined = result.stdout + result.stderr
|
||||
assert result.returncode == 1
|
||||
assert "bad-one.yaml" in combined
|
||||
# the valid entry and removed.yaml must not produce errors
|
||||
assert combined.count("ERROR") == combined.count("bad-one.yaml: ERROR")
|
||||
|
||||
|
||||
def test_directory_mode_all_valid_exits_zero(tmp_path):
|
||||
write_entry(tmp_path, VALID_ENTRY)
|
||||
(tmp_path / "removed.yaml").write_text(
|
||||
yaml.safe_dump({"removed": []}), encoding="utf-8"
|
||||
)
|
||||
result = run_validator(str(tmp_path))
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
@@ -0,0 +1,325 @@
|
||||
"""Tests for the messages_fts_cjk CJK-bigram index (salvaged from PR #65544).
|
||||
|
||||
Builds the loadable tokenizer from native/fts5_cjk/fts5_cjk.c on the fly;
|
||||
skips when no C toolchain / extension loading is available.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import FTS_CJK_STALE_KEY, SessionDB
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SRC = REPO / "native" / "fts5_cjk" / "fts5_cjk.c"
|
||||
VENDOR = REPO / "native" / "fts5_cjk" / "vendor"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cjk_so(tmp_path_factory):
|
||||
if shutil.which("gcc") is None or not SRC.exists():
|
||||
pytest.skip("no C toolchain / tokenizer source")
|
||||
out = tmp_path_factory.mktemp("fts5cjk") / "libfts5_cjk.so"
|
||||
try:
|
||||
subprocess.run(
|
||||
["gcc", "-shared", "-fPIC", "-O2", f"-I{VENDOR}", str(SRC),
|
||||
"-o", str(out)],
|
||||
check=True, capture_output=True, text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
pytest.skip(f"tokenizer build failed: {e.stderr[:200]}")
|
||||
# Loadability probe (extension loading may be disabled in this build).
|
||||
probe = sqlite3.connect(":memory:")
|
||||
try:
|
||||
probe.enable_load_extension(True)
|
||||
probe.load_extension(str(out))
|
||||
except Exception as e:
|
||||
pytest.skip(f"extension loading unavailable: {e}")
|
||||
finally:
|
||||
probe.close()
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(cjk_so, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
assert d._fts_cjk_loaded, "tokenizer must load on the writer connection"
|
||||
assert d._fts_cjk_available, "fresh DB must be born with the cjk index"
|
||||
d.create_session(session_id="s1", source="cli", model="m")
|
||||
d.append_message("s1", role="user", content="웅기가 shared default 프로필을 요청했다")
|
||||
d.append_message("s1", role="assistant", content="일본 MCP 후보 우선순위 정리했습니다")
|
||||
d.append_message("s1", role="user", content="graphiti daemon looks healthy")
|
||||
d.append_message("s1", role="tool", content="일본 tool output blob", tool_name="terminal")
|
||||
yield d
|
||||
d.close()
|
||||
|
||||
|
||||
def test_two_char_korean_hits_cjk_index(db):
|
||||
rows = db.search_messages("웅기", limit=10)
|
||||
assert rows and "웅기" in rows[0]["snippet"]
|
||||
rows = db.search_messages("일본", limit=10)
|
||||
assert rows
|
||||
|
||||
|
||||
def test_mixed_and_ascii_queries(db):
|
||||
assert db.search_messages("graphiti", limit=10)
|
||||
assert db.search_messages('"shared default" AND 웅기', limit=10)
|
||||
assert db.search_messages("우선순위", limit=10)
|
||||
|
||||
|
||||
def test_no_false_positive_across_words(db):
|
||||
# 기가/했다 exist inside runs; a bigram crossing a word boundary must not.
|
||||
assert db.search_messages("다프로", limit=10) == []
|
||||
|
||||
|
||||
def test_lone_single_cjk_char_routes_like(db):
|
||||
# 1-char CJK terms keep LIKE substring semantics (bigram index only
|
||||
# holds unigrams for isolated chars). "가" appears inside 웅기가.
|
||||
assert db._describe_search_path("가") == "like_scan"
|
||||
rows = db.search_messages("가", limit=10)
|
||||
assert rows, "LIKE fallback must still find substring matches"
|
||||
|
||||
|
||||
def test_tool_role_filter_routes_like(db):
|
||||
# Tool rows are excluded from the cjk index; role_filter=['tool'] CJK
|
||||
# queries take the LIKE route and still find tool output.
|
||||
rows = db.search_messages("일본", role_filter=["tool"], limit=10)
|
||||
assert rows and all(r["role"] == "tool" for r in rows)
|
||||
|
||||
|
||||
def test_triggers_mirror_updates_and_deletes(db):
|
||||
db.append_message("s1", role="user", content="자바스크립트 리팩토링")
|
||||
assert db.search_messages("리팩토링", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"UPDATE messages SET content = '파이썬 리라이트' WHERE content LIKE '%리팩토링%'"
|
||||
)
|
||||
db._conn.commit()
|
||||
assert db.search_messages("리팩토링", limit=10) == []
|
||||
assert db.search_messages("리라이트", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute("DELETE FROM messages WHERE content = '파이썬 리라이트'")
|
||||
db._conn.commit()
|
||||
assert db.search_messages("리라이트", limit=10) == []
|
||||
|
||||
|
||||
def test_rewound_rows_hidden_from_cjk_search(db):
|
||||
db.append_message("s1", role="user", content="되돌리기 대상 메시지")
|
||||
assert db.search_messages("되돌리기", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"UPDATE messages SET active = 0, compacted = 0 "
|
||||
"WHERE content LIKE '%되돌리기%'"
|
||||
)
|
||||
db._conn.commit()
|
||||
assert db.search_messages("되돌리기", limit=10) == []
|
||||
assert db.search_messages("되돌리기", include_inactive=True, limit=10)
|
||||
|
||||
|
||||
def test_config_toggle_disables_cjk(cjk_so, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "0")
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
try:
|
||||
assert not d._fts_cjk_loaded
|
||||
assert not d._fts_cjk_available
|
||||
# No cjk objects created at all.
|
||||
with d._lock:
|
||||
row = d._conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name = 'messages_fts_cjk'"
|
||||
).fetchone()
|
||||
assert row is None
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_no_extension_no_cjk_objects(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "nonexistent.so"))
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
try:
|
||||
assert not d._fts_cjk_loaded
|
||||
assert not d._fts_cjk_available
|
||||
d.create_session(session_id="s1", source="cli", model="m")
|
||||
d.append_message("s1", role="user", content="일본 MCP 정리")
|
||||
# Trigram/LIKE routing still answers.
|
||||
assert d.search_messages("일본", limit=10)
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_tokenizer_loss_self_heals_and_optimize_rebuilds(cjk_so, tmp_path, monkeypatch):
|
||||
"""Full stale lifecycle: capable open → tokenizer-less open (drops
|
||||
triggers, breadcrumbs) → rows written in the gap → capable open again
|
||||
(index NOT served) → optimize-storage rebuilds → search complete."""
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
db_path = tmp_path / "state.db"
|
||||
|
||||
d1 = SessionDB(db_path=db_path)
|
||||
assert d1._fts_cjk_available
|
||||
d1.create_session(session_id="s1", source="cli", model="m")
|
||||
d1.append_message("s1", role="user", content="첫번째 메시지")
|
||||
d1.close()
|
||||
|
||||
# Tokenizer-less open: triggers dropped, breadcrumb set, writes fine.
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "gone.so"))
|
||||
d2 = SessionDB(db_path=db_path)
|
||||
assert not d2._fts_cjk_loaded
|
||||
assert not d2._fts_cjk_available
|
||||
d2.append_message("s1", role="user", content="틈새에 쓰인 메시지")
|
||||
assert d2.get_meta(FTS_CJK_STALE_KEY) == "1"
|
||||
with d2._lock:
|
||||
trigs = d2._conn.execute(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='trigger' "
|
||||
"AND name LIKE 'messages_fts_cjk%'"
|
||||
).fetchone()[0]
|
||||
assert trigs == 0
|
||||
# Search still answers via trigram/LIKE.
|
||||
assert d2.search_messages("틈새", limit=10)
|
||||
d2.close()
|
||||
|
||||
# Capable open again: stale index must NOT be served.
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d3 = SessionDB(db_path=db_path)
|
||||
assert d3._fts_cjk_loaded
|
||||
assert not d3._fts_cjk_available, "stale index must not serve reads"
|
||||
assert d3.fts_optimize_available(), "optimize must offer the rebuild"
|
||||
# Search still complete through the legacy routes meanwhile.
|
||||
assert d3.search_messages("틈새", limit=10)
|
||||
|
||||
result = d3.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d3._fts_cjk_available
|
||||
assert d3.get_meta(FTS_CJK_STALE_KEY) is None
|
||||
assert d3.fts_cjk_rebuild_status() is None
|
||||
# Both the pre-gap and in-gap rows are now searchable via the index.
|
||||
assert d3._describe_search_path("틈새") == "fts_cjk"
|
||||
assert d3.search_messages("첫번째", limit=10)
|
||||
assert d3.search_messages("틈새", limit=10)
|
||||
d3.close()
|
||||
|
||||
|
||||
def test_existing_v23_db_gains_cjk_via_optimize(cjk_so, tmp_path, monkeypatch):
|
||||
"""A v23 DB created BEFORE the extension existed: next capable open
|
||||
creates the index with backfill markers; optimize-storage backfills."""
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "absent.so"))
|
||||
db_path = tmp_path / "state.db"
|
||||
d1 = SessionDB(db_path=db_path)
|
||||
d1.create_session(session_id="s1", source="cli", model="m")
|
||||
for i in range(10):
|
||||
d1.append_message("s1", role="user", content=f"기존 메시지 {i}")
|
||||
d1.close()
|
||||
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d2 = SessionDB(db_path=db_path)
|
||||
assert d2._fts_cjk_loaded
|
||||
# Backfill pending — index not served yet, old rows not indexed.
|
||||
assert not d2._fts_cjk_available
|
||||
st = d2.fts_cjk_rebuild_status()
|
||||
assert st is not None and st["pending"]
|
||||
assert d2.fts_optimize_available()
|
||||
# NEW rows are indexed live by the id-gated triggers even mid-backfill.
|
||||
d2.append_message("s1", role="user", content="새로운 메시지")
|
||||
# Search answers via legacy routes meanwhile.
|
||||
assert d2.search_messages("기존", limit=10)
|
||||
|
||||
result = d2.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d2._fts_cjk_available
|
||||
assert d2.fts_cjk_rebuild_status() is None
|
||||
assert d2._describe_search_path("기존") == "fts_cjk"
|
||||
rows = d2.search_messages("기존", limit=20)
|
||||
assert len(rows) == 10
|
||||
assert d2.search_messages("새로운", limit=10)
|
||||
d2.close()
|
||||
|
||||
|
||||
def test_legacy_v22_optimize_lands_on_cjk(cjk_so, tmp_path, monkeypatch):
|
||||
"""A legacy inline-FTS (pre-v23) DB optimized on a tokenizer-capable
|
||||
host comes out with BOTH the v23 external-content layout AND a complete
|
||||
cjk index in the same run."""
|
||||
import time as _time
|
||||
|
||||
from hermes_state import SCHEMA_SQL
|
||||
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
db_path = tmp_path / "state.db"
|
||||
|
||||
# Hand-build a genuine legacy inline DB (single-column messages_fts).
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
conn.executescript("""
|
||||
DROP TABLE IF EXISTS messages_fts;
|
||||
DROP TABLE IF EXISTS messages_fts_trigram;
|
||||
DROP VIEW IF EXISTS messages_fts_trigram_src;
|
||||
CREATE VIRTUAL TABLE messages_fts USING fts5(content);
|
||||
CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, content) VALUES (new.id, COALESCE(new.content,''));
|
||||
END;
|
||||
""")
|
||||
conn.execute("DELETE FROM schema_version")
|
||||
conn.execute("INSERT INTO schema_version (version) VALUES (10)")
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (id, source, started_at) VALUES ('s1', 'cli', ?)",
|
||||
(_time.time(),),
|
||||
)
|
||||
for role, content in (
|
||||
("user", "레거시 일본 메시지"),
|
||||
("assistant", "legacy english reply"),
|
||||
("tool", "레거시 tool output"),
|
||||
):
|
||||
conn.execute(
|
||||
"INSERT INTO messages (session_id, timestamp, role, content) "
|
||||
"VALUES ('s1', ?, ?, ?)",
|
||||
(_time.time(), role, content),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
d = SessionDB(db_path=db_path)
|
||||
try:
|
||||
assert d.fts_optimize_available()
|
||||
# Legacy DB: cjk index deliberately not created at open (the legacy
|
||||
# branch of _init_schema doesn't touch v23 surfaces).
|
||||
assert not d._fts_cjk_available
|
||||
|
||||
result = d.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d._fts_cjk_available
|
||||
assert d.fts_cjk_rebuild_status() is None
|
||||
assert d._describe_search_path("일본") == "fts_cjk"
|
||||
assert d.search_messages("일본", limit=10)
|
||||
assert d.search_messages("legacy english", limit=10)
|
||||
with d._lock:
|
||||
idx = d._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages_fts_cjk"
|
||||
).fetchone()[0]
|
||||
non_tool = d._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
assert idx == non_tool
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_fresh_db_index_counts_exclude_tool_rows(db):
|
||||
with db._lock:
|
||||
idx = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages_fts_cjk"
|
||||
).fetchone()[0]
|
||||
non_tool = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
assert idx == non_tool
|
||||
|
||||
|
||||
def test_integrity_after_lifecycle(db):
|
||||
db.append_message("s1", role="user", content="무결성 검사")
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(messages_fts_cjk) "
|
||||
"VALUES('integrity-check')"
|
||||
)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Tests for the session-search slow-query log (salvaged from PR #65544)."""
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import SessionDB
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(tmp_path):
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
d.create_session(session_id="s1", source="cli", model="m")
|
||||
d.append_message("s1", role="user", content="hello graphiti 일본 MCP 정리")
|
||||
yield d
|
||||
d.close()
|
||||
|
||||
|
||||
def test_slow_log_emitted_at_zero_threshold(db, monkeypatch, caplog):
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "0")
|
||||
with caplog.at_level(logging.INFO, logger="hermes_state"):
|
||||
rows = db.search_messages("graphiti", limit=5)
|
||||
assert rows
|
||||
slow = [r for r in caplog.records if "slow session search" in r.getMessage()]
|
||||
assert slow, "threshold 0 must log every search"
|
||||
msg = slow[0].getMessage()
|
||||
assert "path=" in msg and "rows=1" in msg
|
||||
|
||||
|
||||
def test_no_log_under_threshold(db, monkeypatch, caplog):
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "60000")
|
||||
with caplog.at_level(logging.INFO, logger="hermes_state"):
|
||||
db.search_messages("graphiti", limit=5)
|
||||
assert not [r for r in caplog.records if "slow session search" in r.getMessage()]
|
||||
|
||||
|
||||
def test_path_attribution(db):
|
||||
# Without the cjk tokenizer loaded, routing matches the pre-cjk shape.
|
||||
assert db._describe_search_path("graphiti OR neo4j") == "fts5"
|
||||
assert db._describe_search_path("우선순위 캘린더") == "trigram"
|
||||
assert db._describe_search_path("일본 MCP") == "like_scan"
|
||||
|
||||
|
||||
def test_path_attribution_cjk_available(db):
|
||||
# With the bigram index available, CJK queries (including 2-char terms)
|
||||
# route to fts_cjk; lone 1-char CJK runs keep the LIKE route.
|
||||
db._fts_cjk_available = True
|
||||
try:
|
||||
assert db._describe_search_path("일본 MCP") == "fts_cjk"
|
||||
assert db._describe_search_path("우선순위 캘린더") == "fts_cjk"
|
||||
assert db._describe_search_path("가 alone") == "like_scan"
|
||||
assert db._describe_search_path("graphiti OR neo4j") == "fts5"
|
||||
finally:
|
||||
db._fts_cjk_available = False
|
||||
|
||||
|
||||
def test_results_unchanged_by_wrapper(db, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "0")
|
||||
rows = db.search_messages("graphiti", limit=5)
|
||||
assert rows and rows[0]["session_id"] == "s1"
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Tests for website/scripts/extract-plugins.py.
|
||||
|
||||
Behavioral contracts for the /docs/plugins catalog extractor:
|
||||
|
||||
1. Reads ``plugin-catalog/*.yaml`` entries (skipping ``removed.yaml``) and
|
||||
emits ``plugins.json`` rows carrying name/repo/sha/tier/capabilities plus
|
||||
a synthesized ``hermes plugins install <name>`` command.
|
||||
2. Entries missing any of name/repo/sha are skipped (logged, not fatal).
|
||||
3. A missing ``plugin-catalog/`` directory degrades gracefully: empty
|
||||
catalog list, zero counts in the meta sidecar, exit 0 — the docs build
|
||||
must stay green before the catalog directory lands on main.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
EXTRACT = REPO_ROOT / "website" / "scripts" / "extract-plugins.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mod():
|
||||
spec = importlib.util.spec_from_file_location("extract_plugins", EXTRACT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _write_entry(catalog_dir: Path, name: str, **overrides) -> Path:
|
||||
import yaml
|
||||
|
||||
entry = {
|
||||
"name": name,
|
||||
"repo": f"https://github.com/example/{name}",
|
||||
"sha": "38fe0fb53eff98d477f807432e965429e665ca33",
|
||||
"description": f"{name} does things.",
|
||||
"maintainer": "Example",
|
||||
"tier": "community",
|
||||
}
|
||||
entry.update(overrides)
|
||||
# Drop keys explicitly set to None so tests can simulate missing fields.
|
||||
entry = {k: v for k, v in entry.items() if v is not None}
|
||||
path = catalog_dir / f"{name}.yaml"
|
||||
path.write_text(yaml.safe_dump(entry), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Entry loading + validation
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def test_valid_entry_is_extracted_with_install_command(mod, tmp_path):
|
||||
catalog = tmp_path / "plugin-catalog"
|
||||
catalog.mkdir()
|
||||
_write_entry(
|
||||
catalog,
|
||||
"example-plugin",
|
||||
tier="official",
|
||||
docs_url="https://example.com/docs",
|
||||
requires_hermes=">=0.19",
|
||||
platforms=["linux"],
|
||||
capabilities={
|
||||
"provides_tools": ["do_thing"],
|
||||
"provides_hooks": ["on_start"],
|
||||
"provides_middleware": [],
|
||||
"requires_env": ["EXAMPLE_TOKEN"],
|
||||
},
|
||||
)
|
||||
|
||||
entries = mod.load_catalog_entries(catalog)
|
||||
|
||||
assert len(entries) == 1
|
||||
e = entries[0]
|
||||
assert e["name"] == "example-plugin"
|
||||
assert e["repo"] == "https://github.com/example/example-plugin"
|
||||
assert e["sha"] == "38fe0fb53eff98d477f807432e965429e665ca33"
|
||||
assert e["shaShort"] == "38fe0fb"
|
||||
assert e["tier"] == "official"
|
||||
assert e["maintainer"] == "Example"
|
||||
assert e["requiresHermes"] == ">=0.19"
|
||||
assert e["platforms"] == ["linux"]
|
||||
assert e["docsUrl"] == "https://example.com/docs"
|
||||
assert e["capabilities"]["providesTools"] == ["do_thing"]
|
||||
assert e["capabilities"]["providesHooks"] == ["on_start"]
|
||||
assert e["capabilities"]["requiresEnv"] == ["EXAMPLE_TOKEN"]
|
||||
assert e["installCommand"] == "hermes plugins install example-plugin"
|
||||
|
||||
|
||||
def test_entries_missing_required_fields_are_skipped(mod, tmp_path, capsys):
|
||||
catalog = tmp_path / "plugin-catalog"
|
||||
catalog.mkdir()
|
||||
_write_entry(catalog, "good-plugin")
|
||||
_write_entry(catalog, "no-sha", sha=None)
|
||||
_write_entry(catalog, "no-repo", repo=None)
|
||||
|
||||
entries = mod.load_catalog_entries(catalog)
|
||||
|
||||
assert [e["name"] for e in entries] == ["good-plugin"]
|
||||
err = capsys.readouterr().err
|
||||
assert "no-sha" in err
|
||||
assert "no-repo" in err
|
||||
|
||||
|
||||
def test_removed_yaml_is_not_treated_as_an_entry(mod, tmp_path):
|
||||
catalog = tmp_path / "plugin-catalog"
|
||||
catalog.mkdir()
|
||||
_write_entry(catalog, "kept-plugin")
|
||||
(catalog / "removed.yaml").write_text(
|
||||
"removed:\n - name: evil-plugin\n repo: https://github.com/evil/x\n"
|
||||
' reason: "bad"\n date: "2026-07-02"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
entries = mod.load_catalog_entries(catalog)
|
||||
assert [e["name"] for e in entries] == ["kept-plugin"]
|
||||
assert mod.count_removed(catalog) == 1
|
||||
|
||||
|
||||
def test_unknown_tier_normalizes_to_community(mod, tmp_path):
|
||||
catalog = tmp_path / "plugin-catalog"
|
||||
catalog.mkdir()
|
||||
_write_entry(catalog, "weird-tier", tier="platinum")
|
||||
|
||||
entries = mod.load_catalog_entries(catalog)
|
||||
assert entries[0]["tier"] == "community"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Full run: outputs + graceful degradation
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def test_main_writes_catalog_and_meta(mod, tmp_path):
|
||||
catalog = tmp_path / "plugin-catalog"
|
||||
catalog.mkdir()
|
||||
_write_entry(catalog, "alpha", tier="official")
|
||||
_write_entry(catalog, "beta")
|
||||
(catalog / "removed.yaml").write_text(
|
||||
"removed:\n - name: gone\n", encoding="utf-8"
|
||||
)
|
||||
out_dir = tmp_path / "api"
|
||||
|
||||
rc = mod.main(catalog_dir=catalog, output_dir=out_dir)
|
||||
|
||||
assert rc == 0
|
||||
plugins = json.loads((out_dir / "plugins.json").read_text(encoding="utf-8"))
|
||||
meta = json.loads((out_dir / "plugins-meta.json").read_text(encoding="utf-8"))
|
||||
assert [p["name"] for p in plugins] == ["alpha", "beta"]
|
||||
assert meta["total"] == 2
|
||||
assert meta["byTier"] == {"official": 1, "community": 1}
|
||||
assert meta["removedCount"] == 1
|
||||
assert meta["generatedAt"]
|
||||
|
||||
|
||||
def test_missing_catalog_dir_degrades_to_empty_outputs_exit_zero(mod, tmp_path):
|
||||
out_dir = tmp_path / "api"
|
||||
|
||||
rc = mod.main(catalog_dir=tmp_path / "does-not-exist", output_dir=out_dir)
|
||||
|
||||
assert rc == 0
|
||||
plugins = json.loads((out_dir / "plugins.json").read_text(encoding="utf-8"))
|
||||
meta = json.loads((out_dir / "plugins-meta.json").read_text(encoding="utf-8"))
|
||||
assert plugins == []
|
||||
assert meta["total"] == 0
|
||||
assert meta["byTier"] == {"official": 0, "community": 0}
|
||||
assert meta["removedCount"] == 0
|
||||
|
||||
|
||||
def test_script_exits_zero_as_subprocess_when_catalog_missing(tmp_path):
|
||||
"""CLI contract: the deploy step runs the script hard (no `|| true`);
|
||||
it must exit 0 even when plugin-catalog/ hasn't landed yet."""
|
||||
out_dir = tmp_path / "api"
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(EXTRACT),
|
||||
"--catalog-dir",
|
||||
str(tmp_path / "missing"),
|
||||
"--output-dir",
|
||||
str(out_dir),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert (out_dir / "plugins.json").exists()
|
||||
assert (out_dir / "plugins-meta.json").exists()
|
||||
@@ -402,6 +402,21 @@ export const en: Translations = {
|
||||
versionBadge: "Version",
|
||||
showInSidebar: "Show in sidebar",
|
||||
hideFromSidebar: "Hide from sidebar",
|
||||
catalogHeading: "Plugin catalog",
|
||||
catalogHint:
|
||||
"Curated, Nous-reviewed plugins pinned to exact commits. Install from here for supply-chain-safe versions.",
|
||||
catalogSearchPlaceholder: "Search catalog...",
|
||||
catalogEmpty: "No catalog entries match.",
|
||||
catalogEmptyDocsLink: "Learn about Hermes plugins",
|
||||
catalogInstallBtn: "Install",
|
||||
catalogInstalledBadge: "Installed ✓",
|
||||
catalogUpdateBtn: "Update available",
|
||||
catalogRemovedBadge: "Removed",
|
||||
catalogConfirmTitle: "Install this plugin?",
|
||||
catalogConfirmInstallNote:
|
||||
"Plugins install disabled; enable it after install to activate.",
|
||||
catalogRequiresEnv: "Requires env",
|
||||
removedFromCatalog: "Removed from catalog",
|
||||
},
|
||||
|
||||
skills: {
|
||||
|
||||
@@ -351,6 +351,20 @@ export interface Translations {
|
||||
versionBadge: string;
|
||||
showInSidebar: string;
|
||||
hideFromSidebar: string;
|
||||
// Catalog section (en-only fallback convention — optional keys).
|
||||
catalogHeading?: string;
|
||||
catalogHint?: string;
|
||||
catalogSearchPlaceholder?: string;
|
||||
catalogEmpty?: string;
|
||||
catalogEmptyDocsLink?: string;
|
||||
catalogInstallBtn?: string;
|
||||
catalogInstalledBadge?: string;
|
||||
catalogUpdateBtn?: string;
|
||||
catalogRemovedBadge?: string;
|
||||
catalogConfirmTitle?: string;
|
||||
catalogConfirmInstallNote?: string;
|
||||
catalogRequiresEnv?: string;
|
||||
removedFromCatalog?: string;
|
||||
};
|
||||
|
||||
// ── Profiles page ──
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { api } from "./api";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function jsonFetchMock(body: unknown = { ok: true }) {
|
||||
return vi.fn<typeof fetch>(
|
||||
async () =>
|
||||
new Response(JSON.stringify(body), {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
status: 200,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe("api.getPluginsCatalog", () => {
|
||||
it("fetches the dashboard plugins catalog endpoint", async () => {
|
||||
vi.stubGlobal("window", {});
|
||||
|
||||
const fetchMock = jsonFetchMock({ entries: [], removed: [], generated_at: "" });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const result = await api.getPluginsCatalog();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"/api/dashboard/plugins/catalog",
|
||||
expect.objectContaining({ credentials: "include" }),
|
||||
);
|
||||
expect(result.entries).toEqual([]);
|
||||
expect(result.removed).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("api.installAgentPlugin with catalog_name", () => {
|
||||
it("posts catalog_name through to the install endpoint", async () => {
|
||||
vi.stubGlobal("window", {});
|
||||
|
||||
const fetchMock = jsonFetchMock({ ok: true, plugin_name: "alpha-plugin" });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await api.installAgentPlugin({
|
||||
identifier: "",
|
||||
catalog_name: "alpha-plugin",
|
||||
enable: false,
|
||||
});
|
||||
|
||||
const [url, init] = fetchMock.mock.calls[0]!;
|
||||
expect(url).toBe("/api/dashboard/agent-plugins/install");
|
||||
const body = JSON.parse(String((init as RequestInit).body));
|
||||
expect(body.catalog_name).toBe("alpha-plugin");
|
||||
expect(body.identifier).toBe("");
|
||||
expect(body.enable).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -921,6 +921,9 @@ export const api = {
|
||||
|
||||
getPluginsHub: () => fetchJSON<PluginsHubResponse>("/api/dashboard/plugins/hub"),
|
||||
|
||||
getPluginsCatalog: () =>
|
||||
fetchJSON<CatalogResponse>("/api/dashboard/plugins/catalog"),
|
||||
|
||||
installAgentPlugin: (body: AgentPluginInstallRequest) =>
|
||||
fetchJSON<AgentPluginInstallResponse>("/api/dashboard/agent-plugins/install", {
|
||||
method: "POST",
|
||||
@@ -2506,6 +2509,8 @@ export interface HubAgentPluginRow {
|
||||
auth_required: boolean;
|
||||
auth_command: string;
|
||||
user_hidden: boolean;
|
||||
/** Reason string when this plugin is on the catalog removed blocklist. */
|
||||
removed_reason?: string | null;
|
||||
}
|
||||
|
||||
export interface PluginsHubProviders {
|
||||
@@ -2525,6 +2530,8 @@ export interface AgentPluginInstallRequest {
|
||||
identifier: string;
|
||||
force?: boolean;
|
||||
enable?: boolean;
|
||||
/** Install by curated-catalog name (resolves repo + pinned SHA server-side). */
|
||||
catalog_name?: string;
|
||||
}
|
||||
|
||||
export interface AgentPluginInstallResponse {
|
||||
@@ -2537,6 +2544,48 @@ export interface AgentPluginInstallResponse {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ── Plugin catalog types ───────────────────────────────────────────────
|
||||
|
||||
export interface CatalogCapabilities {
|
||||
provides_tools: string[];
|
||||
provides_hooks: string[];
|
||||
provides_middleware: string[];
|
||||
requires_env: string[];
|
||||
}
|
||||
|
||||
export interface CatalogEntry {
|
||||
name: string;
|
||||
description: string;
|
||||
repo: string;
|
||||
sha: string;
|
||||
sha_short: string;
|
||||
tier: "official" | "community";
|
||||
maintainer: string;
|
||||
requires_hermes: string;
|
||||
platforms: string[];
|
||||
capabilities: CatalogCapabilities;
|
||||
docs_url: string;
|
||||
capability_summary: string;
|
||||
/** Installed-state merge (computed server-side). */
|
||||
installed: boolean;
|
||||
installed_sha: string | null;
|
||||
update_available: boolean;
|
||||
runtime_status: "disabled" | "enabled" | "inactive" | null;
|
||||
}
|
||||
|
||||
export interface CatalogRemovedEntry {
|
||||
name: string;
|
||||
repo: string;
|
||||
reason: string;
|
||||
date: string;
|
||||
}
|
||||
|
||||
export interface CatalogResponse {
|
||||
entries: CatalogEntry[];
|
||||
removed: CatalogRemovedEntry[];
|
||||
generated_at: string;
|
||||
}
|
||||
|
||||
export interface AgentPluginUpdateResponse {
|
||||
ok: boolean;
|
||||
name?: string;
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { ExternalLink, RefreshCw, Trash2, Eye, EyeOff } from "lucide-react";
|
||||
import type { Translations } from "@/i18n/types";
|
||||
import { Link } from "react-router-dom";
|
||||
import { api } from "@/lib/api";
|
||||
import type {
|
||||
CatalogEntry,
|
||||
CatalogRemovedEntry,
|
||||
CatalogResponse,
|
||||
HubAgentPluginRow,
|
||||
MemoryProviderConfig,
|
||||
MemoryProviderField,
|
||||
@@ -281,6 +284,11 @@ function MemoryProviderSetupHint({
|
||||
export default function PluginsPage() {
|
||||
const [hub, setHub] = useState<PluginsHubResponse | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [catalog, setCatalog] = useState<CatalogResponse | null>(null);
|
||||
const [catalogLoading, setCatalogLoading] = useState(true);
|
||||
const [catalogSearch, setCatalogSearch] = useState("");
|
||||
const [catalogConfirm, setCatalogConfirm] = useState<CatalogEntry | null>(null);
|
||||
const [catalogBusy, setCatalogBusy] = useState<string | null>(null);
|
||||
const [installId, setInstallId] = useState("");
|
||||
const [installForce, setInstallForce] = useState(false);
|
||||
const [installEnable, setInstallEnable] = useState(true);
|
||||
@@ -316,9 +324,17 @@ export default function PluginsPage() {
|
||||
.catch(() => showToast(t.common.loading, "error"));
|
||||
}, [showToast, t.common.loading]);
|
||||
|
||||
const loadCatalog = useCallback(() => {
|
||||
return api
|
||||
.getPluginsCatalog()
|
||||
.then(setCatalog)
|
||||
.catch(() => setCatalog(null));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void loadHub().finally(() => setLoading(false));
|
||||
}, [loadHub]);
|
||||
void loadCatalog().finally(() => setCatalogLoading(false));
|
||||
}, [loadHub, loadCatalog]);
|
||||
|
||||
useEffect(() => {
|
||||
const provider = memorySel === MEMORY_PROVIDER_BUILTIN ? "" : memorySel;
|
||||
@@ -391,6 +407,27 @@ export default function PluginsPage() {
|
||||
}
|
||||
};
|
||||
|
||||
const onCatalogInstall = async (entry: CatalogEntry) => {
|
||||
setCatalogConfirm(null);
|
||||
setCatalogBusy(entry.name);
|
||||
try {
|
||||
const r = await api.installAgentPlugin({
|
||||
identifier: "",
|
||||
catalog_name: entry.name,
|
||||
force: entry.installed,
|
||||
enable: false,
|
||||
});
|
||||
showToast(`${r.plugin_name ?? entry.name} installed`, "success");
|
||||
if ((r.missing_env?.length ?? 0) > 0)
|
||||
showToast(`${t.pluginsPage.missingEnvWarn} ${r.missing_env!.join(", ")}`, "error");
|
||||
await Promise.all([loadHub(), loadCatalog()]);
|
||||
} catch (e) {
|
||||
showToast(e instanceof Error ? e.message : "Install failed", "error");
|
||||
} finally {
|
||||
setCatalogBusy(null);
|
||||
}
|
||||
};
|
||||
|
||||
const onRescan = useCallback(async () => {
|
||||
setRescanBusy(true);
|
||||
try {
|
||||
@@ -506,6 +543,27 @@ export default function PluginsPage() {
|
||||
|
||||
const rows = hub?.plugins ?? [];
|
||||
const providers = hub?.providers;
|
||||
|
||||
const catalogEntries = useMemo(() => {
|
||||
const entries = catalog?.entries ?? [];
|
||||
const q = catalogSearch.trim().toLowerCase();
|
||||
if (!q) return entries;
|
||||
return entries.filter((entry) =>
|
||||
[
|
||||
entry.name,
|
||||
entry.description,
|
||||
entry.maintainer,
|
||||
...entry.capabilities.provides_tools,
|
||||
].some((haystack) => haystack.toLowerCase().includes(q)),
|
||||
);
|
||||
}, [catalog, catalogSearch]);
|
||||
|
||||
const removedByName = useMemo(() => {
|
||||
const map = new Map<string, CatalogRemovedEntry>();
|
||||
for (const r of catalog?.removed ?? []) map.set(r.name, r);
|
||||
return map;
|
||||
}, [catalog]);
|
||||
|
||||
const selectedMemoryName = memorySel === MEMORY_PROVIDER_BUILTIN ? "" : memorySel;
|
||||
const selectedMemoryInfo = selectedMemoryName
|
||||
? providers?.memory_options.find((provider) => provider.name === selectedMemoryName)
|
||||
@@ -822,6 +880,59 @@ export default function PluginsPage() {
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<div className="flex flex-col gap-3" data-testid="plugin-catalog-section">
|
||||
|
||||
<h3 className="font-mondwest text-display text-xs tracking-[0.12em] text-text-secondary">
|
||||
{t.pluginsPage.catalogHeading ?? "Plugin catalog"}
|
||||
</h3>
|
||||
|
||||
<p className="text-xs tracking-[0.06em] text-text-tertiary">
|
||||
{t.pluginsPage.catalogHint ??
|
||||
"Curated, Nous-reviewed plugins pinned to exact commits."}
|
||||
</p>
|
||||
|
||||
<Input
|
||||
className="max-w-md"
|
||||
placeholder={t.pluginsPage.catalogSearchPlaceholder ?? "Search catalog..."}
|
||||
value={catalogSearch}
|
||||
onChange={(e) => setCatalogSearch(e.target.value)}
|
||||
aria-label={t.pluginsPage.catalogSearchPlaceholder ?? "Search catalog..."}
|
||||
/>
|
||||
|
||||
{catalogLoading ? (
|
||||
<div className="flex items-center gap-2 py-4 text-xs text-text-tertiary">
|
||||
<Spinner />
|
||||
<span>{t.common.loading}</span>
|
||||
</div>
|
||||
) : catalogEntries.length === 0 ? (
|
||||
<p className="text-xs text-text-tertiary">
|
||||
{t.pluginsPage.catalogEmpty ?? "No catalog entries match."}{" "}
|
||||
<a
|
||||
className="underline"
|
||||
href="https://hermes-agent.nousresearch.com/docs/plugins"
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
{t.pluginsPage.catalogEmptyDocsLink ?? "Learn about Hermes plugins"}
|
||||
</a>
|
||||
</p>
|
||||
) : (
|
||||
<ul className="flex flex-col gap-3">
|
||||
{catalogEntries.map((entry) => (
|
||||
<li key={entry.name}>
|
||||
<CatalogEntryCard
|
||||
busy={catalogBusy === entry.name}
|
||||
entry={entry}
|
||||
onInstall={() => setCatalogConfirm(entry)}
|
||||
removed={removedByName.get(entry.name) ?? null}
|
||||
t={t}
|
||||
/>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-3">
|
||||
|
||||
<h3 className="font-mondwest text-display text-xs tracking-[0.12em] text-text-secondary">
|
||||
@@ -896,6 +1007,30 @@ export default function PluginsPage() {
|
||||
|
||||
<Toast toast={toast} />
|
||||
<PluginSlot name="plugins:bottom" />
|
||||
|
||||
<ConfirmDialog
|
||||
open={catalogConfirm !== null}
|
||||
onCancel={() => setCatalogConfirm(null)}
|
||||
onConfirm={() => {
|
||||
if (catalogConfirm) void onCatalogInstall(catalogConfirm);
|
||||
}}
|
||||
title={t.pluginsPage.catalogConfirmTitle ?? "Install this plugin?"}
|
||||
description={
|
||||
catalogConfirm
|
||||
? [
|
||||
catalogConfirm.capability_summary,
|
||||
catalogConfirm.capabilities.requires_env.length
|
||||
? `${t.pluginsPage.catalogRequiresEnv ?? "Requires env"}: ${catalogConfirm.capabilities.requires_env.join(", ")}`
|
||||
: "",
|
||||
t.pluginsPage.catalogConfirmInstallNote ??
|
||||
"Plugins install disabled; enable it after install to activate.",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
: ""
|
||||
}
|
||||
confirmLabel={t.pluginsPage.catalogInstallBtn ?? "Install"}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -961,6 +1096,12 @@ function PluginRowCard(props: PluginRowCardProps) {
|
||||
{row.auth_required ? (
|
||||
<Badge tone="destructive">{t.pluginsPage.authRequired}</Badge>
|
||||
) : null}
|
||||
|
||||
{row.removed_reason ? (
|
||||
<Badge tone="destructive">
|
||||
{t.pluginsPage.catalogRemovedBadge ?? "Removed"}
|
||||
</Badge>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2 shrink-0">
|
||||
@@ -1070,6 +1211,12 @@ function PluginRowCard(props: PluginRowCardProps) {
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{row.removed_reason ? (
|
||||
<p className="border border-destructive/50 px-3 py-2 text-xs text-destructive">
|
||||
{t.pluginsPage.removedFromCatalog ?? "Removed from catalog"}: {row.removed_reason}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{dm?.slots?.length ? (
|
||||
|
||||
<p className="text-xs tracking-[0.05em] text-text-tertiary">
|
||||
@@ -1111,3 +1258,127 @@ function PluginRowCard(props: PluginRowCardProps) {
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
interface CatalogEntryCardProps {
|
||||
busy: boolean;
|
||||
entry: CatalogEntry;
|
||||
onInstall: () => void;
|
||||
removed: CatalogRemovedEntry | null;
|
||||
t: Translations;
|
||||
}
|
||||
|
||||
function CatalogEntryCard(props: CatalogEntryCardProps) {
|
||||
const { busy, entry, onInstall, removed, t } = props;
|
||||
|
||||
const caps = entry.capabilities;
|
||||
const chips: string[] = [];
|
||||
if (caps.provides_tools.length) chips.push(`${caps.provides_tools.length} tools`);
|
||||
if (caps.provides_hooks.length) chips.push(`${caps.provides_hooks.length} hooks`);
|
||||
if (caps.provides_middleware.length)
|
||||
chips.push(`${caps.provides_middleware.length} middleware`);
|
||||
if (caps.requires_env.length) chips.push(`env: ${caps.requires_env.join(", ")}`);
|
||||
|
||||
const isRemoved = removed !== null;
|
||||
|
||||
return (
|
||||
<Card className={cn(busy ? "opacity-70" : undefined)}>
|
||||
<CardContent className="flex flex-col gap-3 px-6 py-4">
|
||||
<div className="flex flex-wrap items-start justify-between gap-4">
|
||||
<div className="flex min-w-0 flex-1 flex-wrap items-center gap-3">
|
||||
<span className="truncate font-semibold">{entry.name}</span>
|
||||
|
||||
<Badge tone={entry.tier === "official" ? "success" : "secondary"}>
|
||||
{entry.tier}
|
||||
</Badge>
|
||||
|
||||
{entry.installed && entry.runtime_status ? (
|
||||
<Badge tone="outline">{entry.runtime_status}</Badge>
|
||||
) : null}
|
||||
|
||||
{isRemoved ? (
|
||||
<Badge tone="destructive">
|
||||
{t.pluginsPage.catalogRemovedBadge ?? "Removed"}
|
||||
</Badge>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2 shrink-0">
|
||||
{isRemoved ? null : entry.installed && !entry.update_available ? (
|
||||
<Badge tone="success">
|
||||
{t.pluginsPage.catalogInstalledBadge ?? "Installed ✓"}
|
||||
</Badge>
|
||||
) : (
|
||||
<Button disabled={busy} ghost size="sm" onClick={onInstall}>
|
||||
{busy ? <Spinner /> : null}
|
||||
{entry.update_available
|
||||
? t.pluginsPage.catalogUpdateBtn ?? "Update available"
|
||||
: t.pluginsPage.catalogInstallBtn ?? "Install"}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isRemoved ? (
|
||||
<p className="border border-destructive/50 px-3 py-2 text-xs text-destructive">
|
||||
{t.pluginsPage.removedFromCatalog ?? "Removed from catalog"}
|
||||
{removed.reason ? `: ${removed.reason}` : ""}
|
||||
{removed.date ? ` (${removed.date})` : ""}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{entry.description ? (
|
||||
<p className="min-w-0 w-full text-xs tracking-[0.06em] text-text-secondary break-words">
|
||||
{entry.description}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{chips.length ? (
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{chips.map((chip) => (
|
||||
<code
|
||||
key={chip}
|
||||
className="border border-border bg-background/40 px-2 py-1 font-mono text-[0.6875rem]"
|
||||
>
|
||||
{chip}
|
||||
</code>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="flex flex-wrap items-center gap-3 text-xs text-text-tertiary">
|
||||
<span>{entry.maintainer}</span>
|
||||
|
||||
<a
|
||||
className="inline-flex items-center gap-1 font-mono underline"
|
||||
href={`${entry.repo.replace(/\.git$/, "")}/tree/${entry.sha}`}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
{entry.sha_short}
|
||||
<ExternalLink className="h-3 w-3 opacity-65" />
|
||||
</a>
|
||||
|
||||
{entry.docs_url ? (
|
||||
<a
|
||||
className="inline-flex items-center gap-1 underline"
|
||||
href={entry.docs_url}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
docs
|
||||
<ExternalLink className="h-3 w-3 opacity-65" />
|
||||
</a>
|
||||
) : null}
|
||||
|
||||
{entry.requires_hermes ? (
|
||||
<span>hermes {entry.requires_hermes}</span>
|
||||
) : null}
|
||||
|
||||
{entry.platforms.length ? (
|
||||
<span>{entry.platforms.join(", ")}</span>
|
||||
) : null}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -744,6 +744,7 @@ All compression settings live in `config.yaml` (no environment variables).
|
||||
compression:
|
||||
enabled: true # Toggle compression on/off
|
||||
threshold: 0.50 # Compress at this % of context limit
|
||||
threshold_tokens: null # Absolute token cap (optional) — takes lower of ratio vs absolute
|
||||
target_ratio: 0.20 # Fraction of threshold to preserve as recent tail
|
||||
protect_last_n: 20 # Min recent messages to keep uncompressed
|
||||
protect_first_n: 3 # Non-system head messages pinned across compactions (0 = pin nothing)
|
||||
@@ -765,6 +766,8 @@ Older configs with `compression.summary_model`, `compression.summary_provider`,
|
||||
|
||||
`protect_first_n` controls how many **non-system** head messages are pinned across every compaction. Default `3` — the opening user/assistant exchange survives every summarizer pass so the original goal stays visible. On long-running rolling-compaction sessions where the opening turn is no longer relevant, set `protect_first_n: 0` to pin nothing but the system prompt + summary + tail. The system prompt itself is always preserved regardless of this setting.
|
||||
|
||||
`threshold_tokens` sets an optional **absolute token cap** for the compression trigger. When set, compression fires at the lower of the ratio-based `threshold` and this absolute count — so compression never fires later than the user's preferred token number regardless of which model is active. This solves the problem where switching between models with different context windows (e.g. 1M → 400K) shifts the absolute trigger point. The cap is clamped to the model's context length, so setting it higher than the model supports is safe — the ratio-based threshold is used instead. Default `null` (disabled — ratio-based threshold only). The cap survives model switches and fallback activations.
|
||||
|
||||
:::tip Gateway hot-reload of compression and context length
|
||||
As of recent releases, editing `model.context_length` or any `compression.*` key in `config.yaml` on a running gateway takes effect on the next message — no gateway restart, no `/reset`, no session rotation required. The cached-agent signature includes these keys, so the gateway transparently rebuilds the agent when it sees a change. API keys and tool/skill config still require the usual reload paths.
|
||||
:::
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
---
|
||||
sidebar_position: 13
|
||||
sidebar_label: "Plugin Catalog"
|
||||
title: "Plugin Catalog"
|
||||
description: "Browse and install reviewed, SHA-pinned Hermes plugins from the curated catalog"
|
||||
---
|
||||
|
||||
# Plugin Catalog
|
||||
|
||||
The plugin catalog is a curated, human-reviewed directory of Hermes plugins you
|
||||
can install by name with a single command:
|
||||
|
||||
```bash
|
||||
hermes plugins install <name>
|
||||
```
|
||||
|
||||
Browse it visually at **[/docs/plugins](/plugins)** — search, tier filters
|
||||
(Official / Community), capability chips, and copyable install commands for
|
||||
every entry.
|
||||
|
||||
The catalog complements — it does not replace — the existing
|
||||
[plugin system](plugins.md). Anything you can install from the catalog is a
|
||||
normal plugin under the hood; the catalog just adds discovery and a review
|
||||
layer on top.
|
||||
|
||||
## What's in an entry
|
||||
|
||||
Each catalog entry is a small YAML file in the
|
||||
[`plugin-catalog/`](https://github.com/NousResearch/hermes-agent/tree/main/plugin-catalog)
|
||||
directory of the hermes-agent repository, declaring:
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `name` | The catalog key you pass to `hermes plugins install` |
|
||||
| `repo` | The plugin's public git repository |
|
||||
| `sha` | The **exact 40-hex commit** that was reviewed — installs check out this pin, not a branch tip |
|
||||
| `tier` | `official` (maintained by NousResearch) or `community` |
|
||||
| `maintainer` | Who owns the plugin |
|
||||
| `capabilities` | Declared tools, hooks, middleware, and required env vars |
|
||||
| `requires_hermes` | Minimum Hermes version, e.g. `>=0.19` (optional) |
|
||||
| `platforms` | OS restrictions, empty = all (optional) |
|
||||
| `docs_url` | External documentation link (optional) |
|
||||
|
||||
## Trust model
|
||||
|
||||
The catalog is designed so you know exactly what you're installing:
|
||||
|
||||
- **Human-merged admission.** Every entry (and every pin update) lands via a
|
||||
pull request reviewed by a maintainer. Nothing enters the catalog
|
||||
automatically.
|
||||
- **Exact SHA pins.** Entries pin a specific commit, not a branch. A plugin
|
||||
author pushing new code to their repo does **not** change what the catalog
|
||||
installs — updating the pin requires another reviewed PR.
|
||||
- **Capability declarations.** Entries state up front which tools, hooks, and
|
||||
middleware the plugin provides and which environment variables (API keys
|
||||
etc.) it needs, so you can judge its blast radius before installing.
|
||||
- **Removed list.** Plugins pulled from the catalog (for example after a
|
||||
security incident) go on `plugin-catalog/removed.yaml` with a reason and
|
||||
date. The installer refuses to install anything on the removed list.
|
||||
- **Installed ≠ enabled.** Installing a catalog plugin puts it on disk; like
|
||||
any plugin it must still be enabled before it loads. See
|
||||
[Plugins → Enabling and disabling](plugins.md).
|
||||
|
||||
:::warning Catalog review is a point-in-time review
|
||||
A catalog entry means the pinned commit was looked at by a human, capability
|
||||
declarations were checked, and the repo met the submission bar. It is not a
|
||||
security audit, and it says nothing about other commits in the same
|
||||
repository. Review the code of anything you give credentials to.
|
||||
:::
|
||||
|
||||
## Installing from the catalog
|
||||
|
||||
```bash
|
||||
# Install a reviewed catalog entry by name (checks out the pinned SHA)
|
||||
hermes plugins install <name>
|
||||
|
||||
# Then enable it, as with any plugin
|
||||
hermes plugins enable <name>
|
||||
```
|
||||
|
||||
The install prompt shows the entry's capability summary — declared tools,
|
||||
hooks, and required env vars — before anything is cloned.
|
||||
|
||||
### Custom git URLs are different
|
||||
|
||||
`hermes plugins install <git-url>` still works for any repository, but it
|
||||
bypasses the catalog entirely:
|
||||
|
||||
- **No review** — you get whatever is at the branch tip, not a reviewed pin.
|
||||
- **A warning banner** is shown to make clear the code is unvetted.
|
||||
- The removed list is still consulted (a known-bad repo is refused by URL).
|
||||
|
||||
Use the git-URL path for your own plugins and repos you already trust; use the
|
||||
catalog for discovery.
|
||||
|
||||
## Submitting a plugin to the catalog
|
||||
|
||||
Submissions are pull requests that add one `plugin-catalog/<name>.yaml` file.
|
||||
The full checklist lives in the
|
||||
[plugin-catalog README](https://github.com/NousResearch/hermes-agent/tree/main/plugin-catalog);
|
||||
in short, an entry must be:
|
||||
|
||||
1. **Owner-submitted** — the PR author owns or maintains the plugin repo.
|
||||
2. **A public repository** — the `repo` URL is publicly cloneable.
|
||||
3. **Released** — the repo has real releases/tags, not just a default branch.
|
||||
4. **Passing validation** — the catalog validation GitHub Action is green on
|
||||
the PR (schema, SHA format, reachability).
|
||||
5. **Pinned to settled code** — the pinned SHA is at least **2 weeks old**, so
|
||||
the catalog never points at code pushed moments before review.
|
||||
|
||||
Pin updates (bumping `sha` to a newer commit) follow the same PR + review
|
||||
process.
|
||||
|
||||
## See also
|
||||
|
||||
- [Plugins](plugins.md) — the plugin system itself: manifest format, enabling,
|
||||
configuration
|
||||
- [Built-in Plugins](built-in-plugins.md) — plugins that ship with Hermes
|
||||
- [Build a Hermes Plugin](/developer-guide/plugins) — write your own
|
||||
- [Plugin Catalog page](/plugins) — the browsable catalog
|
||||
@@ -144,6 +144,11 @@ const config: Config = {
|
||||
label: 'Skills',
|
||||
position: 'left',
|
||||
},
|
||||
{
|
||||
to: '/plugins',
|
||||
label: 'Plugins',
|
||||
position: 'left',
|
||||
},
|
||||
{
|
||||
href: 'https://hermes-agent.nousresearch.com/',
|
||||
label: 'Download',
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract plugin-catalog entries into website/static/api/plugins.json.
|
||||
|
||||
Feeds the Plugin Catalog page at /docs/plugins (website/src/pages/plugins/).
|
||||
|
||||
Data source: ``plugin-catalog/*.yaml`` at the repo root — one YAML file per
|
||||
catalog entry (see plugin-catalog/README.md for the entry schema), plus
|
||||
``plugin-catalog/removed.yaml`` listing plugins pulled from the catalog.
|
||||
No network, no crawling: the catalog is human-merged data in the checkout.
|
||||
|
||||
Graceful degradation: when ``plugin-catalog/`` does not exist yet (the
|
||||
catalog PR may not have merged), we emit an EMPTY catalog list and zeroed
|
||||
meta counts and exit 0 so the docs build stays green. The page renders a
|
||||
"catalog is just getting started" state.
|
||||
|
||||
Outputs (both under website/static/api/, CDN-served at /docs/api/):
|
||||
|
||||
- ``plugins.json`` — list of catalog entries for the page
|
||||
- ``plugins-meta.json`` — counts by tier + generatedAt + removedCount
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_CATALOG_DIR = REPO_ROOT / "plugin-catalog"
|
||||
DEFAULT_OUTPUT_DIR = REPO_ROOT / "website" / "static" / "api"
|
||||
|
||||
CATALOG_TIERS = ("official", "community")
|
||||
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
print(f"[extract-plugins] {msg}", file=sys.stderr)
|
||||
|
||||
|
||||
def _str_list(value) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
return [value] if value.strip() else []
|
||||
if isinstance(value, list):
|
||||
return [str(x) for x in value if x]
|
||||
return []
|
||||
|
||||
|
||||
def _normalize_capabilities(raw) -> dict:
|
||||
raw = raw if isinstance(raw, dict) else {}
|
||||
return {
|
||||
"providesTools": _str_list(raw.get("provides_tools")),
|
||||
"providesHooks": _str_list(raw.get("provides_hooks")),
|
||||
"providesMiddleware": _str_list(raw.get("provides_middleware")),
|
||||
"requiresEnv": _str_list(raw.get("requires_env")),
|
||||
}
|
||||
|
||||
|
||||
def load_catalog_entries(catalog_dir: Path) -> list[dict]:
|
||||
"""Parse all ``*.yaml`` files (except removed.yaml) into page entries.
|
||||
|
||||
Entries missing any of name/repo/sha are skipped with a stderr log —
|
||||
a malformed community entry must never break the docs deploy.
|
||||
"""
|
||||
entries: list[dict] = []
|
||||
if not catalog_dir.is_dir():
|
||||
return entries
|
||||
|
||||
for path in sorted(catalog_dir.glob("*.yaml")):
|
||||
if path.name == "removed.yaml":
|
||||
continue
|
||||
try:
|
||||
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except (yaml.YAMLError, OSError) as e:
|
||||
_log(f"skipping {path.name}: unreadable YAML ({e})")
|
||||
continue
|
||||
if not isinstance(raw, dict):
|
||||
_log(f"skipping {path.name}: not a mapping")
|
||||
continue
|
||||
|
||||
name = str(raw.get("name") or "").strip()
|
||||
repo = str(raw.get("repo") or "").strip()
|
||||
sha = str(raw.get("sha") or "").strip().lower()
|
||||
missing = [
|
||||
field
|
||||
for field, value in (("name", name), ("repo", repo), ("sha", sha))
|
||||
if not value
|
||||
]
|
||||
if missing:
|
||||
_log(f"skipping {path.name}: missing required field(s) {', '.join(missing)}")
|
||||
continue
|
||||
if not SHA_RE.match(sha):
|
||||
_log(f"skipping {path.name} ({name}): sha is not a 40-hex commit pin")
|
||||
continue
|
||||
|
||||
tier = str(raw.get("tier") or "community").strip().lower()
|
||||
if tier not in CATALOG_TIERS:
|
||||
_log(f"{path.name} ({name}): unknown tier {tier!r}, treating as community")
|
||||
tier = "community"
|
||||
|
||||
entries.append({
|
||||
"name": name,
|
||||
"description": str(raw.get("description") or "").strip(),
|
||||
"repo": repo,
|
||||
"sha": sha,
|
||||
"shaShort": sha[:7],
|
||||
"tier": tier,
|
||||
"maintainer": str(raw.get("maintainer") or "").strip(),
|
||||
"requiresHermes": str(raw.get("requires_hermes") or "").strip(),
|
||||
"platforms": _str_list(raw.get("platforms")),
|
||||
"capabilities": _normalize_capabilities(raw.get("capabilities")),
|
||||
"docsUrl": str(raw.get("docs_url") or "").strip(),
|
||||
"installCommand": f"hermes plugins install {name}",
|
||||
})
|
||||
|
||||
entries.sort(key=lambda e: (0 if e["tier"] == "official" else 1, e["name"]))
|
||||
return entries
|
||||
|
||||
|
||||
def count_removed(catalog_dir: Path) -> int:
|
||||
"""Number of entries in plugin-catalog/removed.yaml (``removed:`` list)."""
|
||||
removed_path = catalog_dir / "removed.yaml"
|
||||
if not removed_path.is_file():
|
||||
return 0
|
||||
try:
|
||||
raw = yaml.safe_load(removed_path.read_text(encoding="utf-8"))
|
||||
except (yaml.YAMLError, OSError) as e:
|
||||
_log(f"could not read removed.yaml: {e}")
|
||||
return 0
|
||||
if not isinstance(raw, dict):
|
||||
return 0
|
||||
removed = raw.get("removed")
|
||||
return len(removed) if isinstance(removed, list) else 0
|
||||
|
||||
|
||||
def main(catalog_dir: Path = DEFAULT_CATALOG_DIR, output_dir: Path = DEFAULT_OUTPUT_DIR) -> int:
|
||||
if not catalog_dir.is_dir():
|
||||
_log(
|
||||
f"plugin-catalog directory not found at {catalog_dir}; "
|
||||
"emitting empty catalog (this is expected until the catalog lands)"
|
||||
)
|
||||
|
||||
entries = load_catalog_entries(catalog_dir)
|
||||
removed_count = count_removed(catalog_dir)
|
||||
|
||||
by_tier = Counter(e["tier"] for e in entries)
|
||||
meta = {
|
||||
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
||||
"total": len(entries),
|
||||
"byTier": {tier: by_tier.get(tier, 0) for tier in CATALOG_TIERS},
|
||||
"removedCount": removed_count,
|
||||
}
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_dir / "plugins.json", "w", encoding="utf-8") as f:
|
||||
json.dump(entries, f, separators=(",", ":"), ensure_ascii=False)
|
||||
with open(output_dir / "plugins-meta.json", "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f, separators=(",", ":"), ensure_ascii=False)
|
||||
|
||||
print(
|
||||
f"Extracted {len(entries)} plugin catalog entries "
|
||||
f"({meta['byTier']['official']} official, {meta['byTier']['community']} community, "
|
||||
f"{removed_count} removed) to {output_dir / 'plugins.json'}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--catalog-dir", type=Path, default=DEFAULT_CATALOG_DIR)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||||
args = parser.parse_args()
|
||||
sys.exit(main(catalog_dir=args.catalog_dir, output_dir=args.output_dir))
|
||||
@@ -32,7 +32,10 @@ const websiteDir = resolve(scriptDir, "..");
|
||||
const extractScript = join(scriptDir, "extract-skills.py");
|
||||
const llmsScript = join(scriptDir, "generate-llms-txt.py");
|
||||
const cronBlueprintsScript = join(scriptDir, "extract-automation-blueprints.py");
|
||||
const pluginsScript = join(scriptDir, "extract-plugins.py");
|
||||
const outputFile = join(websiteDir, "static", "api", "skills.json");
|
||||
const pluginsOutputFile = join(websiteDir, "static", "api", "plugins.json");
|
||||
const pluginsMetaOutputFile = join(websiteDir, "static", "api", "plugins-meta.json");
|
||||
const unifiedIndexFile = join(websiteDir, "static", "api", "skills-index.json");
|
||||
const UNIFIED_INDEX_URL =
|
||||
"https://hermes-agent.nousresearch.com/docs/api/skills-index.json";
|
||||
@@ -143,3 +146,22 @@ runPython(llmsScript, "generate-llms-txt.py");
|
||||
// 3) automation-blueprints-index.json — Automation Blueprints catalog page. Non-fatal; the page
|
||||
// renders an empty state if the generator can't run.
|
||||
runPython(cronBlueprintsScript, "extract-automation-blueprints.py");
|
||||
|
||||
// 4) plugins.json + plugins-meta.json — Plugin Catalog page. The script itself
|
||||
// degrades gracefully (empty catalog, exit 0) when plugin-catalog/ is absent;
|
||||
// if python3 is missing entirely, write the same empty fallback so the page
|
||||
// renders its "just getting started" state instead of a fetch error.
|
||||
if (!runPython(pluginsScript, "extract-plugins.py")) {
|
||||
mkdirSync(dirname(pluginsOutputFile), { recursive: true });
|
||||
writeFileSync(pluginsOutputFile, "[]\n");
|
||||
writeFileSync(
|
||||
pluginsMetaOutputFile,
|
||||
JSON.stringify({
|
||||
generatedAt: new Date().toISOString(),
|
||||
total: 0,
|
||||
byTier: { official: 0, community: 0 },
|
||||
removedCount: 0,
|
||||
}) + "\n",
|
||||
);
|
||||
console.warn("[prebuild] wrote empty plugins.json fallback");
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ const sidebars: SidebarsConfig = {
|
||||
'user-guide/features/skins',
|
||||
'user-guide/features/plugins',
|
||||
'user-guide/features/built-in-plugins',
|
||||
'user-guide/features/plugin-catalog',
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,576 @@
|
||||
import React, { useState, useMemo, useCallback, useRef, useEffect } from "react";
|
||||
import Layout from "@theme/Layout";
|
||||
import Link from "@docusaurus/Link";
|
||||
import styles from "./styles.module.css";
|
||||
|
||||
interface PluginCapabilities {
|
||||
providesTools?: string[];
|
||||
providesHooks?: string[];
|
||||
providesMiddleware?: string[];
|
||||
requiresEnv?: string[];
|
||||
}
|
||||
|
||||
interface CatalogPlugin {
|
||||
name: string;
|
||||
description: string;
|
||||
repo: string;
|
||||
sha: string;
|
||||
shaShort: string;
|
||||
tier: string;
|
||||
maintainer: string;
|
||||
requiresHermes?: string;
|
||||
platforms?: string[];
|
||||
capabilities?: PluginCapabilities;
|
||||
docsUrl?: string;
|
||||
installCommand: string;
|
||||
/** Lowercase pre-joined haystack for the search filter (built at load). */
|
||||
_search?: string;
|
||||
}
|
||||
|
||||
interface CatalogMeta {
|
||||
generatedAt?: string;
|
||||
total?: number;
|
||||
byTier?: Record<string, number>;
|
||||
removedCount?: number;
|
||||
}
|
||||
|
||||
// Routes Docusaurus serves the static API JSON from. `baseUrl` is `/docs/`,
|
||||
// `static/api/` ends up at `/docs/api/` — same pattern as the Skills Hub.
|
||||
const PLUGINS_URL = "/docs/api/plugins.json";
|
||||
const META_URL = "/docs/api/plugins-meta.json";
|
||||
|
||||
const CATALOG_README_URL =
|
||||
"https://github.com/NousResearch/hermes-agent/tree/main/plugin-catalog";
|
||||
|
||||
const TIER_CONFIG: Record<
|
||||
string,
|
||||
{ label: string; color: string; bg: string; border: string; icon: string }
|
||||
> = {
|
||||
official: {
|
||||
label: "Official",
|
||||
color: "#ffd700",
|
||||
bg: "rgba(255, 215, 0, 0.08)",
|
||||
border: "rgba(255, 215, 0, 0.25)",
|
||||
icon: "\u{2713}",
|
||||
},
|
||||
community: {
|
||||
label: "Community",
|
||||
color: "#94a3b8",
|
||||
bg: "rgba(148, 163, 184, 0.08)",
|
||||
border: "rgba(148, 163, 184, 0.2)",
|
||||
icon: "\u{2756}",
|
||||
},
|
||||
};
|
||||
|
||||
const TIER_ORDER = ["all", "official", "community"];
|
||||
|
||||
function formatRelativeTime(iso?: string): string | null {
|
||||
if (!iso) return null;
|
||||
const then = new Date(iso).getTime();
|
||||
if (!Number.isFinite(then)) return null;
|
||||
const diffMs = Date.now() - then;
|
||||
if (diffMs < 0) return "just now";
|
||||
const mins = Math.floor(diffMs / 60_000);
|
||||
if (mins < 1) return "just now";
|
||||
if (mins < 60) return `${mins} minute${mins === 1 ? "" : "s"} ago`;
|
||||
const hours = Math.floor(mins / 60);
|
||||
if (hours < 24) return `${hours} hour${hours === 1 ? "" : "s"} ago`;
|
||||
const days = Math.floor(hours / 24);
|
||||
if (days < 30) return `${days} day${days === 1 ? "" : "s"} ago`;
|
||||
const months = Math.floor(days / 30);
|
||||
return `${months} month${months === 1 ? "" : "s"} ago`;
|
||||
}
|
||||
|
||||
function highlightMatch(text: string, query: string): React.ReactNode {
|
||||
if (!query || !text) return text;
|
||||
const idx = text.toLowerCase().indexOf(query.toLowerCase());
|
||||
if (idx === -1) return text;
|
||||
return (
|
||||
<>
|
||||
{text.slice(0, idx)}
|
||||
<mark className={styles.highlight}>{text.slice(idx, idx + query.length)}</mark>
|
||||
{text.slice(idx + query.length)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function CopyButton({ text }: { text: string }) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
const onCopy = useCallback(
|
||||
(e: React.MouseEvent) => {
|
||||
e.stopPropagation();
|
||||
navigator.clipboard?.writeText(text).then(
|
||||
() => {
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 1500);
|
||||
},
|
||||
() => {},
|
||||
);
|
||||
},
|
||||
[text],
|
||||
);
|
||||
return (
|
||||
<button
|
||||
className={styles.copyBtn}
|
||||
onClick={onCopy}
|
||||
title="Copy install command"
|
||||
aria-label="Copy install command"
|
||||
>
|
||||
{copied ? (
|
||||
<svg viewBox="0 0 20 20" fill="currentColor" width="14" height="14">
|
||||
<path
|
||||
fillRule="evenodd"
|
||||
d="M16.704 4.153a.75.75 0 01.143 1.052l-8 10.5a.75.75 0 01-1.127.075l-4.5-4.5a.75.75 0 011.06-1.06l3.894 3.893 7.48-9.817a.75.75 0 011.05-.143z"
|
||||
clipRule="evenodd"
|
||||
/>
|
||||
</svg>
|
||||
) : (
|
||||
<svg viewBox="0 0 20 20" fill="currentColor" width="14" height="14">
|
||||
<path d="M7 3.5A1.5 1.5 0 018.5 2h3.879a1.5 1.5 0 011.06.44l3.122 3.12A1.5 1.5 0 0117 6.622V12.5a1.5 1.5 0 01-1.5 1.5h-1v-3.379a3 3 0 00-.879-2.121L10.5 5.379A3 3 0 008.379 4.5H7v-1z" />
|
||||
<path d="M4.5 6A1.5 1.5 0 003 7.5v9A1.5 1.5 0 004.5 18h7a1.5 1.5 0 001.5-1.5v-5.879a1.5 1.5 0 00-.44-1.06L9.44 6.439A1.5 1.5 0 008.378 6H4.5z" />
|
||||
</svg>
|
||||
)}
|
||||
<span className={styles.copyBtnLabel}>{copied ? "Copied" : "Copy"}</span>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function PluginCard({
|
||||
plugin,
|
||||
query,
|
||||
expanded,
|
||||
onToggle,
|
||||
style,
|
||||
}: {
|
||||
plugin: CatalogPlugin;
|
||||
query: string;
|
||||
expanded: boolean;
|
||||
onToggle: () => void;
|
||||
style?: React.CSSProperties;
|
||||
}) {
|
||||
const tier = TIER_CONFIG[plugin.tier] || TIER_CONFIG.community;
|
||||
const caps = plugin.capabilities || {};
|
||||
const toolCount = caps.providesTools?.length || 0;
|
||||
const hookCount = caps.providesHooks?.length || 0;
|
||||
const middlewareCount = caps.providesMiddleware?.length || 0;
|
||||
const pinUrl = `${plugin.repo.replace(/\.git$/, "").replace(/\/$/, "")}/tree/${plugin.sha}`;
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`${styles.card} ${expanded ? styles.cardExpanded : ""}`}
|
||||
onClick={onToggle}
|
||||
style={style}
|
||||
>
|
||||
<div className={styles.cardAccent} style={{ background: tier.color }} />
|
||||
|
||||
<div className={styles.cardInner}>
|
||||
<div className={styles.cardTop}>
|
||||
<span className={styles.cardIcon}>{"\u{1F50C}"}</span>
|
||||
<div className={styles.cardTitleGroup}>
|
||||
<h3 className={styles.cardTitle}>{highlightMatch(plugin.name, query)}</h3>
|
||||
<span
|
||||
className={styles.tierPill}
|
||||
style={{
|
||||
color: tier.color,
|
||||
background: tier.bg,
|
||||
borderColor: tier.border,
|
||||
}}
|
||||
>
|
||||
{tier.icon} {tier.label}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className={`${styles.cardDesc} ${expanded ? styles.cardDescFull : ""}`}>
|
||||
{highlightMatch(plugin.description || "No description available.", query)}
|
||||
</p>
|
||||
|
||||
<div className={styles.cardMeta}>
|
||||
{toolCount > 0 && (
|
||||
<span className={styles.capChip}>
|
||||
{toolCount} tool{toolCount === 1 ? "" : "s"}
|
||||
</span>
|
||||
)}
|
||||
{hookCount > 0 && (
|
||||
<span className={styles.capChip}>
|
||||
{hookCount} hook{hookCount === 1 ? "" : "s"}
|
||||
</span>
|
||||
)}
|
||||
{middlewareCount > 0 && (
|
||||
<span className={styles.capChip}>
|
||||
{middlewareCount} middleware
|
||||
</span>
|
||||
)}
|
||||
{caps.requiresEnv?.map((v) => (
|
||||
<code key={v} className={styles.envChip}>
|
||||
{v}
|
||||
</code>
|
||||
))}
|
||||
{plugin.platforms?.map((p) => (
|
||||
<span key={p} className={styles.platformPill}>
|
||||
{p === "macos" ? "\u{F8FF} macOS" : p === "linux" ? "\u{1F427} Linux" : p}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{expanded && (
|
||||
<div className={styles.cardDetail}>
|
||||
{plugin.maintainer && (
|
||||
<div className={styles.metaRow}>
|
||||
<span className={styles.metaLabel}>Maintainer</span>
|
||||
<span className={styles.metaValue}>{plugin.maintainer}</span>
|
||||
</div>
|
||||
)}
|
||||
{plugin.requiresHermes && (
|
||||
<div className={styles.metaRow}>
|
||||
<span className={styles.metaLabel}>Requires</span>
|
||||
<span className={styles.metaValue}>
|
||||
<code>hermes {plugin.requiresHermes}</code>
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div className={styles.metaRow}>
|
||||
<span className={styles.metaLabel}>Pinned</span>
|
||||
<span className={styles.metaValue}>
|
||||
<a
|
||||
href={pinUrl}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
className={styles.shaLink}
|
||||
title={plugin.sha}
|
||||
>
|
||||
<code>{plugin.shaShort}</code> ↗
|
||||
</a>
|
||||
</span>
|
||||
</div>
|
||||
{caps.providesTools?.length ? (
|
||||
<div className={styles.metaRow}>
|
||||
<span className={styles.metaLabel}>Tools</span>
|
||||
<span className={styles.chipList}>
|
||||
{caps.providesTools.map((t) => (
|
||||
<code key={t} className={styles.envChip}>
|
||||
{t}
|
||||
</code>
|
||||
))}
|
||||
</span>
|
||||
</div>
|
||||
) : null}
|
||||
<div className={styles.installHint}>
|
||||
<code>{plugin.installCommand}</code>
|
||||
<CopyButton text={plugin.installCommand} />
|
||||
</div>
|
||||
<div className={styles.cardLinks}>
|
||||
<a
|
||||
className={styles.docsLink}
|
||||
href={plugin.repo}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
Repository ↗
|
||||
</a>
|
||||
{plugin.docsUrl ? (
|
||||
<a
|
||||
className={styles.docsLink}
|
||||
href={plugin.docsUrl}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
Documentation ↗
|
||||
</a>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function StatCard({ value, label, color }: { value: number; label: string; color: string }) {
|
||||
return (
|
||||
<div className={styles.stat}>
|
||||
<span className={styles.statValue} style={{ color }}>
|
||||
{value}
|
||||
</span>
|
||||
<span className={styles.statLabel}>{label}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function buildSearchHaystack(p: CatalogPlugin): string {
|
||||
return [
|
||||
p.name,
|
||||
p.description,
|
||||
p.maintainer,
|
||||
p.tier,
|
||||
...(p.capabilities?.providesTools || []),
|
||||
...(p.capabilities?.providesHooks || []),
|
||||
...(p.capabilities?.requiresEnv || []),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export default function PluginCatalogPage() {
|
||||
const [data, setData] = useState<{ plugins: CatalogPlugin[]; meta: CatalogMeta } | null>(
|
||||
null,
|
||||
);
|
||||
const [loadError, setLoadError] = useState<string | null>(null);
|
||||
|
||||
const [search, setSearch] = useState("");
|
||||
const [tierFilter, setTierFilter] = useState("all");
|
||||
const [expandedCard, setExpandedCard] = useState<string | null>(null);
|
||||
const searchRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
(async () => {
|
||||
try {
|
||||
const [pl, mt] = await Promise.all([
|
||||
fetch(PLUGINS_URL).then((r) => {
|
||||
if (!r.ok) throw new Error(`plugins.json HTTP ${r.status}`);
|
||||
return r.json();
|
||||
}),
|
||||
fetch(META_URL).then((r) => (r.ok ? r.json() : {})).catch(() => ({})),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
const arr = Array.isArray(pl) ? (pl as CatalogPlugin[]) : [];
|
||||
for (const p of arr) p._search = buildSearchHaystack(p);
|
||||
setData({ plugins: arr, meta: mt || {} });
|
||||
} catch (err) {
|
||||
if (cancelled) return;
|
||||
setLoadError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === "/" && document.activeElement?.tagName !== "INPUT") {
|
||||
e.preventDefault();
|
||||
searchRef.current?.focus();
|
||||
}
|
||||
if (e.key === "Escape") {
|
||||
searchRef.current?.blur();
|
||||
setExpandedCard(null);
|
||||
}
|
||||
};
|
||||
window.addEventListener("keydown", handler);
|
||||
return () => window.removeEventListener("keydown", handler);
|
||||
}, []);
|
||||
|
||||
const allPlugins: CatalogPlugin[] = data?.plugins ?? [];
|
||||
const meta: CatalogMeta = data?.meta ?? {};
|
||||
|
||||
const filtered = useMemo(() => {
|
||||
const q = search.toLowerCase().trim();
|
||||
return allPlugins.filter((p) => {
|
||||
if (tierFilter !== "all" && p.tier !== tierFilter) return false;
|
||||
if (q) return (p._search || "").includes(q);
|
||||
return true;
|
||||
});
|
||||
}, [search, tierFilter, allPlugins]);
|
||||
|
||||
useEffect(() => {
|
||||
setExpandedCard(null);
|
||||
}, [search, tierFilter]);
|
||||
|
||||
const clearAll = useCallback(() => {
|
||||
setSearch("");
|
||||
setTierFilter("all");
|
||||
}, []);
|
||||
|
||||
const catalogEmpty = data !== null && allPlugins.length === 0;
|
||||
|
||||
return (
|
||||
<Layout
|
||||
title="Plugin Catalog"
|
||||
description="Browse reviewed, SHA-pinned plugins for Hermes Agent"
|
||||
>
|
||||
<div className={styles.page}>
|
||||
<header className={styles.hero}>
|
||||
<div className={styles.heroGlow} />
|
||||
<div className={styles.heroContent}>
|
||||
<p className={styles.heroEyebrow}>Hermes Agent</p>
|
||||
<h1 className={styles.heroTitle}>Plugin Catalog</h1>
|
||||
<nav className={styles.crossNav} aria-label="Catalog pages">
|
||||
<Link className={styles.crossNavLink} to="/skills">
|
||||
Skills
|
||||
</Link>
|
||||
<span className={`${styles.crossNavLink} ${styles.crossNavActive}`}>
|
||||
Plugins
|
||||
</span>
|
||||
</nav>
|
||||
<p className={styles.heroSub}>
|
||||
Reviewed, SHA-pinned plugins you can install with one command.
|
||||
{loadError && (
|
||||
<span style={{ color: "#f87171", marginLeft: 8 }}>
|
||||
· failed to load catalog ({loadError})
|
||||
</span>
|
||||
)}
|
||||
</p>
|
||||
{meta.generatedAt && !catalogEmpty && (
|
||||
<p className={styles.heroSub} style={{ fontSize: "0.85rem", opacity: 0.75 }}>
|
||||
Catalog refreshed{" "}
|
||||
<span title={meta.generatedAt}>
|
||||
{formatRelativeTime(meta.generatedAt) || "recently"}
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
{!catalogEmpty && (
|
||||
<div className={styles.statsRow}>
|
||||
<StatCard
|
||||
value={allPlugins.filter((p) => p.tier === "official").length}
|
||||
label="Official"
|
||||
color="#ffd700"
|
||||
/>
|
||||
<StatCard
|
||||
value={allPlugins.filter((p) => p.tier === "community").length}
|
||||
label="Community"
|
||||
color="#94a3b8"
|
||||
/>
|
||||
<StatCard value={meta.removedCount ?? 0} label="Removed" color="#f87171" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{!catalogEmpty && (
|
||||
<div className={styles.controlsBar}>
|
||||
<div className={styles.searchWrap}>
|
||||
<svg
|
||||
className={styles.searchIcon}
|
||||
viewBox="0 0 20 20"
|
||||
fill="currentColor"
|
||||
width="18"
|
||||
height="18"
|
||||
>
|
||||
<path
|
||||
fillRule="evenodd"
|
||||
d="M8 4a4 4 0 100 8 4 4 0 000-8zM2 8a6 6 0 1110.89 3.476l4.817 4.817a1 1 0 01-1.414 1.414l-4.816-4.816A6 6 0 012 8z"
|
||||
clipRule="evenodd"
|
||||
/>
|
||||
</svg>
|
||||
<input
|
||||
ref={searchRef}
|
||||
type="text"
|
||||
placeholder='Search plugins... (press "/" to focus)'
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
className={styles.searchInput}
|
||||
/>
|
||||
{search && (
|
||||
<button className={styles.clearBtn} onClick={() => setSearch("")}>
|
||||
<svg viewBox="0 0 20 20" fill="currentColor" width="16" height="16">
|
||||
<path
|
||||
fillRule="evenodd"
|
||||
d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z"
|
||||
clipRule="evenodd"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className={styles.tierPills}>
|
||||
{TIER_ORDER.map((tier) => {
|
||||
const active = tierFilter === tier;
|
||||
const conf = TIER_CONFIG[tier];
|
||||
const count =
|
||||
tier === "all"
|
||||
? allPlugins.length
|
||||
: allPlugins.filter((p) => p.tier === tier).length;
|
||||
return (
|
||||
<button
|
||||
key={tier}
|
||||
className={`${styles.tierBtn} ${active ? styles.tierBtnActive : ""}`}
|
||||
onClick={() => setTierFilter(tier)}
|
||||
style={
|
||||
active && conf
|
||||
? ({
|
||||
"--pill-color": conf.color,
|
||||
"--pill-bg": conf.bg,
|
||||
"--pill-border": conf.border,
|
||||
} as React.CSSProperties)
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{tier === "all" ? "All" : conf?.label || tier}
|
||||
<span className={styles.tierCount}>{count}</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<main className={styles.main}>
|
||||
{!data && !loadError ? (
|
||||
<div className={styles.empty}>
|
||||
<div className={styles.loadingSpinner} />
|
||||
<h3 className={styles.emptyTitle}>Loading the catalog…</h3>
|
||||
</div>
|
||||
) : catalogEmpty ? (
|
||||
<div className={styles.empty}>
|
||||
<div className={styles.emptyIcon}>{"\u{1F331}"}</div>
|
||||
<h3 className={styles.emptyTitle}>The catalog is just getting started</h3>
|
||||
<p className={styles.emptyDesc}>
|
||||
The plugin catalog is a curated, human-reviewed list of Hermes
|
||||
plugins — each entry pinned to an exact commit. Want yours listed?
|
||||
Submissions are open.
|
||||
</p>
|
||||
<div className={styles.emptyActions}>
|
||||
<a
|
||||
className={styles.emptyCta}
|
||||
href={CATALOG_README_URL}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
How to submit a plugin ↗
|
||||
</a>
|
||||
<Link className={styles.emptyCtaSecondary} to="/user-guide/features/plugin-catalog">
|
||||
Read the catalog docs
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
) : filtered.length > 0 ? (
|
||||
<div className={styles.grid}>
|
||||
{filtered.map((plugin, i) => {
|
||||
const key = `${plugin.tier}-${plugin.name}`;
|
||||
return (
|
||||
<PluginCard
|
||||
key={key}
|
||||
plugin={plugin}
|
||||
query={search}
|
||||
expanded={expandedCard === key}
|
||||
onToggle={() => setExpandedCard(expandedCard === key ? null : key)}
|
||||
style={{ animationDelay: `${Math.min(i, 20) * 25}ms` }}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
) : (
|
||||
<div className={styles.empty}>
|
||||
<div className={styles.emptyIcon}>{"\u{1F50D}"}</div>
|
||||
<h3 className={styles.emptyTitle}>No plugins found</h3>
|
||||
<p className={styles.emptyDesc}>
|
||||
Try a different search term or clear your filters.
|
||||
</p>
|
||||
<button className={styles.emptyReset} onClick={clearAll}>
|
||||
Reset all filters
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</main>
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,691 @@
|
||||
@import url("https://fonts.googleapis.com/css2?family=DM+Sans:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap");
|
||||
|
||||
.page {
|
||||
font-family: "DM Sans", -apple-system, BlinkMacSystemFont, sans-serif;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
.hero {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
padding: 4rem 2rem 2.5rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.heroGlow {
|
||||
position: absolute;
|
||||
top: -120px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: 600px;
|
||||
height: 400px;
|
||||
background: radial-gradient(
|
||||
ellipse at center,
|
||||
rgba(255, 215, 0, 0.07) 0%,
|
||||
transparent 70%
|
||||
);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.heroContent {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
max-width: 720px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.heroEyebrow {
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 0.75rem;
|
||||
letter-spacing: 0.15em;
|
||||
text-transform: uppercase;
|
||||
color: rgba(255, 215, 0, 0.5);
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
.heroTitle {
|
||||
font-size: 3rem;
|
||||
font-weight: 700;
|
||||
letter-spacing: -0.04em;
|
||||
line-height: 1.1;
|
||||
margin: 0 0 0.75rem;
|
||||
}
|
||||
|
||||
[data-theme="dark"] .heroTitle {
|
||||
color: #fafaf6;
|
||||
}
|
||||
|
||||
.heroSub {
|
||||
font-size: 1.05rem;
|
||||
color: var(--ifm-font-color-secondary, #9a968e);
|
||||
line-height: 1.5;
|
||||
margin: 0 0 1.5rem;
|
||||
}
|
||||
|
||||
/* Cross-nav between the Skills Hub and Plugin Catalog pages. */
|
||||
.crossNav {
|
||||
display: inline-flex;
|
||||
gap: 0.35rem;
|
||||
margin: 0 0 1.25rem;
|
||||
padding: 0.25rem;
|
||||
border: 1px solid rgba(255, 215, 0, 0.1);
|
||||
border-radius: 10px;
|
||||
background: rgba(255, 255, 255, 0.02);
|
||||
}
|
||||
|
||||
.crossNavLink {
|
||||
font-family: "DM Sans", sans-serif;
|
||||
font-size: 0.82rem;
|
||||
font-weight: 500;
|
||||
padding: 0.3rem 0.9rem;
|
||||
border-radius: 7px;
|
||||
color: var(--ifm-font-color-secondary, #9a968e);
|
||||
text-decoration: none;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.crossNavLink:hover {
|
||||
color: #ffd700;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.crossNavActive {
|
||||
background: rgba(255, 215, 0, 0.08);
|
||||
color: #ffd700;
|
||||
}
|
||||
|
||||
.statsRow {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
gap: 2.5rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.stat {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0.2rem;
|
||||
}
|
||||
|
||||
.statValue {
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 1.6rem;
|
||||
font-weight: 700;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.statLabel {
|
||||
font-size: 0.72rem;
|
||||
letter-spacing: 0.06em;
|
||||
text-transform: uppercase;
|
||||
color: var(--ifm-font-color-secondary, #9a968e);
|
||||
}
|
||||
|
||||
.controlsBar {
|
||||
position: sticky;
|
||||
top: 60px; /* below Docusaurus navbar */
|
||||
z-index: 50;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
align-items: center;
|
||||
padding: 1rem 2rem;
|
||||
backdrop-filter: blur(16px) saturate(1.4);
|
||||
border-bottom: 1px solid rgba(255, 215, 0, 0.06);
|
||||
}
|
||||
|
||||
[data-theme="dark"] .controlsBar {
|
||||
background: rgba(7, 7, 13, 0.85);
|
||||
}
|
||||
|
||||
.searchWrap {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
max-width: 560px;
|
||||
}
|
||||
|
||||
.searchIcon {
|
||||
position: absolute;
|
||||
left: 0.85rem;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
color: rgba(255, 215, 0, 0.35);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.searchInput {
|
||||
width: 100%;
|
||||
padding: 0.7rem 2.5rem 0.7rem 2.6rem;
|
||||
font-size: 0.95rem;
|
||||
font-family: "DM Sans", sans-serif;
|
||||
border: 1px solid rgba(255, 215, 0, 0.12);
|
||||
border-radius: 10px;
|
||||
background: rgba(15, 15, 24, 0.6);
|
||||
color: var(--ifm-font-color-base, #e8e4dc);
|
||||
outline: none;
|
||||
transition: border-color 0.2s, box-shadow 0.2s;
|
||||
}
|
||||
|
||||
.searchInput:focus {
|
||||
border-color: rgba(255, 215, 0, 0.4);
|
||||
box-shadow: 0 0 0 3px rgba(255, 215, 0, 0.06);
|
||||
}
|
||||
|
||||
.searchInput::placeholder {
|
||||
color: var(--ifm-font-color-secondary, #9a968e);
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
.clearBtn {
|
||||
position: absolute;
|
||||
right: 0.6rem;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--ifm-font-color-secondary);
|
||||
cursor: pointer;
|
||||
padding: 0.15rem;
|
||||
display: flex;
|
||||
opacity: 0.6;
|
||||
transition: opacity 0.15s;
|
||||
}
|
||||
|
||||
.clearBtn:hover {
|
||||
opacity: 1;
|
||||
color: #ffd700;
|
||||
}
|
||||
|
||||
/* Tier tabs: All / Official / Community (skills page's source-pill pattern). */
|
||||
.tierPills {
|
||||
display: flex;
|
||||
gap: 0.4rem;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.tierBtn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.35rem;
|
||||
padding: 0.35rem 0.75rem;
|
||||
border: 1px solid rgba(255, 255, 255, 0.07);
|
||||
border-radius: 20px;
|
||||
background: transparent;
|
||||
color: var(--ifm-font-color-secondary, #9a968e);
|
||||
font-family: "DM Sans", sans-serif;
|
||||
font-size: 0.8rem;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.tierBtn:hover {
|
||||
border-color: rgba(255, 255, 255, 0.15);
|
||||
color: var(--ifm-font-color-base);
|
||||
}
|
||||
|
||||
.tierBtnActive {
|
||||
border-color: var(--pill-border, rgba(255, 215, 0, 0.3));
|
||||
background: var(--pill-bg, rgba(255, 215, 0, 0.06));
|
||||
color: var(--pill-color, #ffd700);
|
||||
}
|
||||
|
||||
.tierCount {
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 0.68rem;
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
padding: 0.05rem 0.35rem;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.tierBtnActive .tierCount {
|
||||
background: rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.main {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
padding: 1.5rem 2rem 3rem;
|
||||
}
|
||||
|
||||
.grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(340px, 1fr));
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
@keyframes cardIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(8px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
.card {
|
||||
position: relative;
|
||||
border: 1px solid rgba(255, 255, 255, 0.05);
|
||||
border-radius: 10px;
|
||||
overflow: hidden;
|
||||
cursor: pointer;
|
||||
transition: border-color 0.2s, box-shadow 0.2s, transform 0.2s;
|
||||
animation: cardIn 0.35s ease both;
|
||||
}
|
||||
|
||||
[data-theme="dark"] .card {
|
||||
background: #0c0c16;
|
||||
}
|
||||
|
||||
.card:hover {
|
||||
border-color: rgba(255, 215, 0, 0.15);
|
||||
box-shadow: 0 4px 24px rgba(0, 0, 0, 0.3), 0 0 0 1px rgba(255, 215, 0, 0.05);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.cardExpanded {
|
||||
border-color: rgba(255, 215, 0, 0.2);
|
||||
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4), 0 0 0 1px rgba(255, 215, 0, 0.08);
|
||||
}
|
||||
|
||||
.cardAccent {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 3px;
|
||||
height: 100%;
|
||||
opacity: 0.5;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
|
||||
.card:hover .cardAccent {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.cardInner {
|
||||
padding: 1rem 1rem 0.85rem 1.15rem;
|
||||
}
|
||||
|
||||
.cardTop {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 0.6rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.cardIcon {
|
||||
font-size: 1.15rem;
|
||||
line-height: 1;
|
||||
flex-shrink: 0;
|
||||
margin-top: 0.1rem;
|
||||
opacity: 0.7;
|
||||
}
|
||||
|
||||
.cardTitleGroup {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 0.5rem;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.cardTitle {
|
||||
font-size: 0.92rem;
|
||||
font-weight: 600;
|
||||
line-height: 1.3;
|
||||
margin: 0;
|
||||
word-break: break-word;
|
||||
color: var(--ifm-font-color-base);
|
||||
}
|
||||
|
||||
.tierPill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 0.62rem;
|
||||
font-weight: 500;
|
||||
padding: 0.15rem 0.45rem;
|
||||
border-radius: 4px;
|
||||
border: 1px solid;
|
||||
white-space: nowrap;
|
||||
flex-shrink: 0;
|
||||
margin-top: 0.1rem;
|
||||
}
|
||||
|
||||
.cardDesc {
|
||||
font-size: 0.82rem;
|
||||
line-height: 1.55;
|
||||
color: var(--ifm-font-color-secondary, #9a968e);
|
||||
margin: 0 0 0.6rem;
|
||||
display: -webkit-box;
|
||||
-webkit-line-clamp: 2;
|
||||
-webkit-box-orient: vertical;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.cardDescFull {
|
||||
-webkit-line-clamp: unset;
|
||||
}
|
||||
|
||||
.cardMeta {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.35rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
/* Capability chips: tools/hooks/middleware counts. */
|
||||
.capChip {
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 0.66rem;
|
||||
padding: 0.15rem 0.45rem;
|
||||
border: 1px solid rgba(255, 215, 0, 0.12);
|
||||
border-radius: 3px;
|
||||
background: rgba(255, 215, 0, 0.04);
|
||||
color: rgba(255, 215, 0, 0.7);
|
||||
}
|
||||
|
||||
/* Required env var chips. */
|
||||
.envChip {
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 0.66rem;
|
||||
padding: 0.12rem 0.4rem;
|
||||
border: 1px solid rgba(255, 255, 255, 0.06);
|
||||
border-radius: 3px;
|
||||
background: rgba(255, 255, 255, 0.02);
|
||||
color: rgba(255, 215, 0, 0.6);
|
||||
}
|
||||
|
||||
.platformPill {
|
||||
font-size: 0.66rem;
|
||||
padding: 0.12rem 0.4rem;
|
||||
border-radius: 3px;
|
||||
background: rgba(96, 165, 250, 0.06);
|
||||
color: rgba(96, 165, 250, 0.8);
|
||||
border: 1px solid rgba(96, 165, 250, 0.1);
|
||||
}
|
||||
|
||||
.cardDetail {
|
||||
margin-top: 0.75rem;
|
||||
padding-top: 0.7rem;
|
||||
border-top: 1px solid rgba(255, 255, 255, 0.04);
|
||||
animation: cardIn 0.2s ease both;
|
||||
}
|
||||
|
||||
.metaRow {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 0.3rem;
|
||||
}
|
||||
|
||||
.metaLabel {
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 0.62rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
color: var(--ifm-font-color-secondary);
|
||||
opacity: 0.5;
|
||||
min-width: 4.5rem;
|
||||
padding-top: 0.15rem;
|
||||
}
|
||||
|
||||
.metaValue {
|
||||
font-size: 0.78rem;
|
||||
color: var(--ifm-font-color-base);
|
||||
}
|
||||
|
||||
.metaValue code {
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 0.72rem;
|
||||
background: rgba(255, 255, 255, 0.03);
|
||||
padding: 0.05rem 0.3rem;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.chipList {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.3rem;
|
||||
}
|
||||
|
||||
.shaLink {
|
||||
color: rgba(96, 165, 250, 0.9);
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.shaLink:hover {
|
||||
color: rgba(96, 165, 250, 1);
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.installHint {
|
||||
margin-top: 0.65rem;
|
||||
padding: 0.45rem 0.65rem;
|
||||
background: rgba(0, 0, 0, 0.25);
|
||||
border: 1px solid rgba(255, 215, 0, 0.06);
|
||||
border-radius: 5px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.installHint code {
|
||||
font-family: "JetBrains Mono", monospace;
|
||||
font-size: 0.72rem;
|
||||
color: rgba(255, 215, 0, 0.7);
|
||||
background: none;
|
||||
padding: 0;
|
||||
flex: 1;
|
||||
overflow-x: auto;
|
||||
white-space: nowrap;
|
||||
scrollbar-width: none;
|
||||
}
|
||||
|
||||
.installHint code::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.copyBtn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
flex-shrink: 0;
|
||||
padding: 0.2rem 0.45rem;
|
||||
border: 1px solid rgba(255, 215, 0, 0.18);
|
||||
border-radius: 4px;
|
||||
background: rgba(255, 215, 0, 0.06);
|
||||
color: rgba(255, 215, 0, 0.85);
|
||||
font-size: 0.68rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.copyBtn:hover {
|
||||
background: rgba(255, 215, 0, 0.14);
|
||||
color: rgba(255, 215, 0, 1);
|
||||
}
|
||||
|
||||
.copyBtnLabel {
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.cardLinks {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.cardLinks .docsLink {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.docsLink {
|
||||
display: block;
|
||||
margin-top: 0.65rem;
|
||||
padding: 0.45rem 0.65rem;
|
||||
border: 1px solid rgba(96, 165, 250, 0.2);
|
||||
border-radius: 5px;
|
||||
background: rgba(96, 165, 250, 0.06);
|
||||
color: rgba(96, 165, 250, 0.9);
|
||||
font-size: 0.78rem;
|
||||
text-decoration: none;
|
||||
text-align: center;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.docsLink:hover {
|
||||
background: rgba(96, 165, 250, 0.12);
|
||||
color: rgba(96, 165, 250, 1);
|
||||
border-color: rgba(96, 165, 250, 0.35);
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.highlight {
|
||||
background: rgba(255, 215, 0, 0.2);
|
||||
color: #ffd700;
|
||||
border-radius: 2px;
|
||||
padding: 0 1px;
|
||||
}
|
||||
|
||||
.loadingSpinner {
|
||||
width: 2.25rem;
|
||||
height: 2.25rem;
|
||||
margin: 0 auto 1rem;
|
||||
border: 3px solid rgba(255, 215, 0, 0.15);
|
||||
border-top-color: rgba(255, 215, 0, 0.7);
|
||||
border-radius: 50%;
|
||||
animation: pluginsSpin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes pluginsSpin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.empty {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 5rem 2rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.emptyIcon {
|
||||
font-size: 2.5rem;
|
||||
margin-bottom: 1rem;
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.emptyTitle {
|
||||
font-size: 1.1rem;
|
||||
font-weight: 600;
|
||||
margin: 0 0 0.5rem;
|
||||
color: var(--ifm-font-color-base);
|
||||
}
|
||||
|
||||
.emptyDesc {
|
||||
font-size: 0.85rem;
|
||||
color: var(--ifm-font-color-secondary);
|
||||
margin: 0 0 1.25rem;
|
||||
max-width: 480px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.emptyActions {
|
||||
display: flex;
|
||||
gap: 0.6rem;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.emptyCta {
|
||||
font-family: "DM Sans", sans-serif;
|
||||
font-size: 0.85rem;
|
||||
font-weight: 600;
|
||||
padding: 0.55rem 1.25rem;
|
||||
border: 1px solid rgba(255, 215, 0, 0.3);
|
||||
border-radius: 6px;
|
||||
background: rgba(255, 215, 0, 0.06);
|
||||
color: #ffd700;
|
||||
text-decoration: none;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.emptyCta:hover {
|
||||
background: rgba(255, 215, 0, 0.12);
|
||||
color: #ffd700;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.emptyCtaSecondary {
|
||||
font-family: "DM Sans", sans-serif;
|
||||
font-size: 0.85rem;
|
||||
padding: 0.55rem 1.25rem;
|
||||
border: 1px solid rgba(96, 165, 250, 0.25);
|
||||
border-radius: 6px;
|
||||
background: rgba(96, 165, 250, 0.05);
|
||||
color: rgba(96, 165, 250, 0.9);
|
||||
text-decoration: none;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.emptyCtaSecondary:hover {
|
||||
background: rgba(96, 165, 250, 0.1);
|
||||
color: rgba(96, 165, 250, 1);
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.emptyReset {
|
||||
font-family: "DM Sans", sans-serif;
|
||||
font-size: 0.85rem;
|
||||
padding: 0.5rem 1.25rem;
|
||||
border: 1px solid rgba(255, 215, 0, 0.25);
|
||||
border-radius: 6px;
|
||||
background: transparent;
|
||||
color: #ffd700;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.emptyReset:hover {
|
||||
background: rgba(255, 215, 0, 0.08);
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.hero {
|
||||
padding: 2.5rem 1.25rem 1.75rem;
|
||||
}
|
||||
|
||||
.heroTitle {
|
||||
font-size: 2rem;
|
||||
}
|
||||
|
||||
.statsRow {
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.statValue {
|
||||
font-size: 1.25rem;
|
||||
}
|
||||
|
||||
.controlsBar {
|
||||
padding: 0.75rem 1rem;
|
||||
}
|
||||
|
||||
.main {
|
||||
padding: 0.75rem 1rem 2rem;
|
||||
}
|
||||
|
||||
.grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import React, { useState, useMemo, useCallback, useRef, useEffect } from "react";
|
||||
import Layout from "@theme/Layout";
|
||||
import Link from "@docusaurus/Link";
|
||||
import styles from "./styles.module.css";
|
||||
|
||||
interface Skill {
|
||||
@@ -650,6 +651,14 @@ export default function SkillsDashboard() {
|
||||
<div className={styles.heroContent}>
|
||||
<p className={styles.heroEyebrow}>Hermes Agent</p>
|
||||
<h1 className={styles.heroTitle}>Skills Hub</h1>
|
||||
<nav className={styles.crossNav} aria-label="Catalog pages">
|
||||
<span className={`${styles.crossNavLink} ${styles.crossNavActive}`}>
|
||||
Skills
|
||||
</span>
|
||||
<Link className={styles.crossNavLink} to="/plugins">
|
||||
Plugins
|
||||
</Link>
|
||||
</nav>
|
||||
<p className={styles.heroSub}>
|
||||
Discover, search, and install from{" "}
|
||||
<strong className={styles.heroAccent}>
|
||||
|
||||
@@ -69,6 +69,38 @@
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
/* Cross-nav between the Skills Hub and Plugin Catalog pages. */
|
||||
.crossNav {
|
||||
display: inline-flex;
|
||||
gap: 0.35rem;
|
||||
margin: 0 0 1.25rem;
|
||||
padding: 0.25rem;
|
||||
border: 1px solid rgba(255, 215, 0, 0.1);
|
||||
border-radius: 10px;
|
||||
background: rgba(255, 255, 255, 0.02);
|
||||
}
|
||||
|
||||
.crossNavLink {
|
||||
font-family: "DM Sans", sans-serif;
|
||||
font-size: 0.82rem;
|
||||
font-weight: 500;
|
||||
padding: 0.3rem 0.9rem;
|
||||
border-radius: 7px;
|
||||
color: var(--ifm-font-color-secondary, #9a968e);
|
||||
text-decoration: none;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.crossNavLink:hover {
|
||||
color: #ffd700;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.crossNavActive {
|
||||
background: rgba(255, 215, 0, 0.08);
|
||||
color: #ffd700;
|
||||
}
|
||||
|
||||
.statsRow {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
|
||||
Reference in New Issue
Block a user