Revert "perf(docker-tests): blocking exec with DEVNULL for wait_for_container_ready"

This reverts commit 19ca99e863.

.github/workflows/docker.yml

@@ -143,9 +143,70 @@ jobs:
           OPENROUTER_API_KEY: ""
           OPENAI_API_KEY: ""
           NOUS_API_KEY: ""
+          # Profile every docker subprocess call so we can diagnose why
+          # the suite is slow on CI vs. local. Each per-file subprocess
+          # writes its own docker-test-profile-<pid>.json; the merge
+          # step below combines them into a single report.
+          HERMES_DOCKER_TEST_PROFILE: "1"
         run: |
           scripts/run_tests.sh tests/docker/ --file-timeout 600
 
+      - name: Merge docker test profiles
+        if: always()
+        run: |
+          python3 -c "
+          import json, glob, sys
+          files = sorted(glob.glob('docker-test-profile-*.json'))
+          if not files:
+              print('No per-PID profile files found — profiling may not have activated')
+              sys.exit(0)
+          merged = {'tests': [], 'summary': {}}
+          for f in files:
+              with open(f) as fh:
+                  data = json.load(fh)
+              merged['tests'].extend(data.get('tests', []))
+          # Recompute summary aggregates from merged tests.
+          subcmd_totals = {}
+          subcmd_counts = {}
+          total_sleep = 0
+          total_sleeps = 0
+          for t in merged['tests']:
+              for sub, info in t.get('by_subcommand', {}).items():
+                  subcmd_totals[sub] = subcmd_totals.get(sub, 0) + info['total_s']
+                  subcmd_counts[sub] = subcmd_counts.get(sub, 0) + info['count']
+              total_sleep += t.get('total_sleep_s', 0)
+              total_sleeps += t.get('sleep_count', 0)
+          total_docker = sum(subcmd_totals.values())
+          merged['summary'] = {
+              'total_tests': len(merged['tests']),
+              'total_calls': sum(subcmd_counts.values()),
+              'total_sleeps': total_sleeps,
+              'total_docker_s': round(total_docker, 3),
+              'total_sleep_s': round(total_sleep, 3),
+              'total_wall_s': round(total_docker + total_sleep, 3),
+              'by_subcommand': {
+                  sub: {
+                      'count': subcmd_counts[sub],
+                      'total_s': round(subcmd_totals[sub], 3),
+                      'avg_s': round(subcmd_totals[sub] / subcmd_counts[sub], 3) if subcmd_counts[sub] else 0,
+                  }
+                  for sub in sorted(subcmd_totals, key=lambda s: subcmd_totals[s], reverse=True)
+              },
+          }
+          with open('docker-test-profile.json', 'w') as fh:
+              json.dump(merged, fh, indent=2)
+          print(f'Merged {len(files)} per-PID profiles ({len(merged[\"tests\"])} tests, {total_docker:.0f}s docker + {total_sleep:.0f}s sleep = {total_docker + total_sleep:.0f}s wall)')
+          "
+
+      - name: Upload docker test profile
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: docker-test-profile-${{ matrix.arch }}
+          path: docker-test-profile.json
+          retention-days: 14
+          if-no-files-found: warn
+
   # ---------------------------------------------------------------------------
   # Stitch both per-arch digests into a single tagged multi-arch manifest.
   # This is a registry-side operation — no building, no layer re-push —

scripts/run_tests.sh

@@ -88,6 +88,8 @@ exec env -i \
   LC_ALL=C.UTF-8 \
   PYTHONHASHSEED=0 \
   ${HERMES_RUN_SLOW_PET_TESTS:+HERMES_RUN_SLOW_PET_TESTS="$HERMES_RUN_SLOW_PET_TESTS"} \
+  ${HERMES_DOCKER_TEST_PROFILE:+HERMES_DOCKER_TEST_PROFILE="$HERMES_DOCKER_TEST_PROFILE"} \
+  ${HERMES_DOCKER_PROFILE_OUT:+HERMES_DOCKER_PROFILE_OUT="$HERMES_DOCKER_PROFILE_OUT"} \
   ${EXTRA_PYTHONPATH:+PYTHONPATH="$EXTRA_PYTHONPATH"} \
   ${EXTRA_PYTEST_PLUGINS:+PYTEST_PLUGINS="$EXTRA_PYTEST_PLUGINS"} \
   "$PYTHON" "$SCRIPT_DIR/run_tests_parallel.py" "$@"

tests/docker/conftest.py

@@ -8,6 +8,10 @@ Override the image with ``HERMES_TEST_IMAGE`` env var to point at a pre-built
 image (faster local iteration); otherwise the ``built_image`` fixture builds
 the repo's Dockerfile once per session.
 
+Profiling: set ``HERMES_DOCKER_TEST_PROFILE=1`` to instrument every
+``subprocess.run`` that invokes ``docker``. A per-test breakdown of
+subcommand timings is written to ``docker-test-profile.json`` and a
+summary is printed to stderr at session end. See ``profiling.py``.
 """
 from __future__ import annotations
 
@@ -19,6 +23,8 @@ from collections.abc import Iterator
 
 import pytest
 
+pytest_plugins = ["tests.docker.profiling"]
+
 IMAGE_TAG = os.environ.get("HERMES_TEST_IMAGE", "hermes-agent-harness:latest")
 
 
@@ -82,6 +88,42 @@ def container_name(request) -> Iterator[str]:
     )
 
 
+@pytest.fixture(scope="module")
+def shared_container(built_image: str, request) -> Iterator[str]:
+    """A long-lived container shared across all tests in a module.
+
+    Starts one ``sleep infinity`` container, waits for s6 cont-init to
+    finish, yields the container name, and tears it down at module exit.
+    Tests that only need to *read* static image state (env vars, file
+    existence, immutable-permission checks, etc.) can share this instead
+    of each paying the full ``docker run`` + cont-init startup cost.
+
+    Tests that *mutate* container state (config changes, gateway starts,
+    restarts, etc.) should still use ``container_name`` + ``start_container``
+    for isolation.
+    """
+    safe = request.module.__name__.replace(".", "-")
+    name = f"hermes-shared-{safe}"
+    # Clean up any leftover from a prior run.
+    subprocess.run(
+        ["docker", "rm", "-f", name],
+        capture_output=True, timeout=10,
+    )
+    r = subprocess.run(
+        ["docker", "run", "-d", "--name", name, built_image, "sleep", "infinity"],
+        capture_output=True, text=True, timeout=60,
+    )
+    assert r.returncode == 0, f"docker run failed: {r.stderr}"
+    try:
+        wait_for_container_ready(name)
+        yield name
+    finally:
+        subprocess.run(
+            ["docker", "rm", "-f", name],
+            capture_output=True, timeout=10,
+        )
+
+
 # ---------------------------------------------------------------------------
 # docker_exec — default to the unprivileged hermes user
 # ---------------------------------------------------------------------------
@@ -153,6 +195,14 @@ def wait_for_container_ready(
     Raises ``TimeoutError`` if the container never becomes ready — much
     better than a fixed ``time.sleep()`` that either wastes time on fast
     machines or flakes on slow ones.
+
+    Note: an earlier iteration tried a single blocking ``docker exec``
+    with an in-container ``until`` loop to eliminate polling overhead.
+    That was a net regression on CI: the per-``docker exec`` connection
+    overhead on shared runners (~0.7–1s) is higher than the cost of
+    3–7 quick poll calls (0.27s each), so the blocking approach traded
+    fewer calls for longer per-call duration and lost. The poll loop
+    is kept because it's better suited to CI's docker exec cost profile.
     """
     end = time.monotonic() + deadline_s
     while time.monotonic() < end:

tests/docker/profiling.py

@@ -0,0 +1,416 @@
+"""Profiling plugin for docker integration tests.
+
+Activated by ``HERMES_DOCKER_TEST_PROFILE=1``. Instruments every
+``subprocess.run`` call whose argv starts with ``docker`` to measure
+wall-clock time, and also tracks ``time.sleep`` calls so we can see the
+full wall-clock picture — including the invisible sleep time in polling
+loops that doesn't show up as docker call duration.
+
+Outputs:
+  - JSON report at ``$HERMES_DOCKER_PROFILE_OUT`` (default:
+    ``docker-test-profile-{pid}.json`` in the repo root — per-PID
+    because run_tests_parallel.py spawns each test file in its own
+    subprocess).
+  - Console summary on stderr at session end.
+
+The plugin is a no-op when the env var is not set — zero overhead on
+normal test runs.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional
+
+import pytest
+
+_ACTIVE = bool(os.environ.get("HERMES_DOCKER_TEST_PROFILE"))
+
+# ---------------------------------------------------------------------------
+# Data model
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class TimedEvent:
+    """Base for any timed event in a test's lifecycle."""
+
+    duration_s: float
+    timestamp: float  # monotonic
+
+
+@dataclass
+class DockerCall(TimedEvent):
+    """One instrumented docker subprocess call."""
+
+    argv: list[str]
+    returncode: int
+
+
+@dataclass
+class SleepGap(TimedEvent):
+    """A time.sleep() that occurred between docker calls.
+
+    This captures the invisible "gap" time — the polling sleeps, the
+    Python logic between assertions, etc. Each SleepGap is attributed to
+    the test that was running when it occurred.
+    """
+
+    caller: str  # short description of who called sleep
+
+
+@dataclass
+class TestProfile:
+    """Per-test accumulation of docker calls and sleep gaps."""
+
+    name: str
+    calls: list[DockerCall] = field(default_factory=list)
+    sleeps: list[SleepGap] = field(default_factory=list)
+
+    @property
+    def total_docker_s(self) -> float:
+        return sum(c.duration_s for c in self.calls)
+
+    @property
+    def total_sleep_s(self) -> float:
+        return sum(s.duration_s for s in self.sleeps)
+
+    @property
+    def total_wall_s(self) -> float:
+        """Docker time + sleep time — the test's visible wall-clock cost."""
+        return self.total_docker_s + self.total_sleep_s
+
+    @property
+    def call_count(self) -> int:
+        return len(self.calls)
+
+    def by_subcommand(self) -> dict[str, list[DockerCall]]:
+        """Group calls by the first docker subcommand (run, exec, restart, ...)."""
+        groups: dict[str, list[DockerCall]] = defaultdict(list)
+        for c in self.calls:
+            sub = c.argv[1] if len(c.argv) > 1 else "?"
+            groups[sub].append(c)
+        return groups
+
+
+# ---------------------------------------------------------------------------
+# Session-level collector
+# ---------------------------------------------------------------------------
+
+
+class ProfileCollector:
+    """Singleton accumulator shared across the pytest session."""
+
+    def __init__(self) -> None:
+        self.tests: dict[str, TestProfile] = {}
+        self.current: Optional[TestProfile] = None
+        self._original_run: Any = None
+        self._original_sleep: Any = None
+        self._patched = False
+        self._last_docker_end: float = 0.0
+
+    def start_test(self, name: str) -> None:
+        self.current = TestProfile(name=name)
+        self.tests[name] = self.current
+        self._last_docker_end = 0.0
+
+    def end_test(self) -> None:
+        self.current = None
+
+    def record(self, call: DockerCall) -> None:
+        if self.current is not None:
+            self.current.calls.append(call)
+        self._last_docker_end = call.timestamp + call.duration_s
+
+    def record_sleep(self, gap: SleepGap) -> None:
+        if self.current is not None:
+            self.current.sleeps.append(gap)
+
+    def install_patch(self) -> None:
+        """Monkey-patch subprocess.run and time.sleep to capture timings."""
+        if self._patched:
+            return
+        import subprocess
+
+        self._original_run = subprocess.run
+        self._original_sleep = time.sleep
+        collector = self
+
+        def timed_run(*args: Any, **kwargs: Any) -> Any:
+            argv = list(args[0]) if args and args[0] else kwargs.get("args", [])
+            is_docker = bool(argv) and argv[0] == "docker"
+            if not is_docker:
+                return collector._original_run(*args, **kwargs)
+            t0 = time.monotonic()
+            result = collector._original_run(*args, **kwargs)
+            elapsed = time.monotonic() - t0
+            rc = getattr(result, "returncode", -1)
+            call = DockerCall(
+                duration_s=round(elapsed, 4),
+                timestamp=t0,
+                argv=[str(a) for a in argv],
+                returncode=rc,
+            )
+            collector.record(call)
+            return result
+
+        def timed_sleep(secs: float) -> None:
+            # Only track sleeps that happen between docker calls within a test.
+            # Short sleeps (< 0.05s) are probably just Python scheduling noise.
+            if collector.current is None or secs < 0.05:
+                return collector._original_sleep(secs)
+            t0 = time.monotonic()
+            collector._original_sleep(secs)
+            elapsed = time.monotonic() - t0
+            # Try to identify the caller for context
+            import traceback
+            stack = traceback.extract_stack(limit=4)
+            caller = ""
+            for frame in reversed(stack):
+                fname = frame.filename
+                if "conftest" in fname or "test_" in fname:
+                    caller = f"{Path(fname).name}:{frame.lineno}"
+                    break
+            gap = SleepGap(
+                duration_s=round(elapsed, 4),
+                timestamp=t0,
+                caller=caller,
+            )
+            collector.record_sleep(gap)
+
+        subprocess.run = timed_run
+        time.sleep = timed_sleep
+        self._patched = True
+
+    def uninstall_patch(self) -> None:
+        if not self._patched:
+            return
+        import subprocess
+
+        if self._original_run is not None:
+            subprocess.run = self._original_run
+        if self._original_sleep is not None:
+            time.sleep = self._original_sleep
+        self._patched = False
+
+    def build_report(self) -> dict[str, Any]:
+        """Build the JSON-serializable report dict."""
+        report: dict[str, Any] = {
+            "tests": [],
+            "summary": {},
+        }
+        all_docker_time = 0.0
+        all_sleep_time = 0.0
+        all_call_count = 0
+        all_sleep_count = 0
+        subcmd_totals: dict[str, float] = defaultdict(float)
+        subcmd_counts: dict[str, int] = defaultdict(int)
+
+        for _name, tp in sorted(
+            self.tests.items(), key=lambda x: x[1].total_wall_s, reverse=True
+        ):
+            by_sub = tp.by_subcommand()
+            test_entry: dict[str, Any] = {
+                "name": tp.name,
+                "total_docker_s": round(tp.total_docker_s, 3),
+                "total_sleep_s": round(tp.total_sleep_s, 3),
+                "total_wall_s": round(tp.total_wall_s, 3),
+                "call_count": tp.call_count,
+                "sleep_count": len(tp.sleeps),
+                "by_subcommand": {
+                    sub: {
+                        "count": len(calls),
+                        "total_s": round(sum(c.duration_s for c in calls), 3),
+                        "avg_s": round(
+                            sum(c.duration_s for c in calls) / len(calls), 3
+                        )
+                        if calls
+                        else 0,
+                        "max_s": round(max(c.duration_s for c in calls), 3)
+                        if calls
+                        else 0,
+                    }
+                    for sub, calls in sorted(
+                        by_sub.items(),
+                        key=lambda x: sum(c.duration_s for c in x[1]),
+                        reverse=True,
+                    )
+                },
+                "calls": [
+                    {
+                        "type": "docker",
+                        "argv": " ".join(c.argv[:8]),
+                        "duration_s": c.duration_s,
+                        "returncode": c.returncode,
+                    }
+                    for c in sorted(tp.calls, key=lambda x: x.duration_s, reverse=True)
+                ],
+                "sleeps": [
+                    {
+                        "type": "sleep",
+                        "duration_s": s.duration_s,
+                        "caller": s.caller,
+                    }
+                    for s in sorted(tp.sleeps, key=lambda x: x.duration_s, reverse=True)
+                ],
+            }
+            report["tests"].append(test_entry)
+            all_docker_time += tp.total_docker_s
+            all_sleep_time += tp.total_sleep_s
+            all_call_count += tp.call_count
+            all_sleep_count += len(tp.sleeps)
+            for sub, calls in by_sub.items():
+                subcmd_totals[sub] += sum(c.duration_s for c in calls)
+                subcmd_counts[sub] += len(calls)
+
+        report["summary"] = {
+            "total_tests": len(self.tests),
+            "total_docker_s": round(all_docker_time, 3),
+            "total_sleep_s": round(all_sleep_time, 3),
+            "total_wall_s": round(all_docker_time + all_sleep_time, 3),
+            "total_calls": all_call_count,
+            "total_sleeps": all_sleep_count,
+            "by_subcommand": {
+                sub: {
+                    "count": subcmd_counts[sub],
+                    "total_s": round(subcmd_totals[sub], 3),
+                    "avg_s": round(subcmd_totals[sub] / subcmd_counts[sub], 3)
+                    if subcmd_counts[sub]
+                    else 0,
+                }
+                for sub in sorted(
+                    subcmd_totals, key=lambda s: subcmd_totals[s], reverse=True
+                )
+            },
+        }
+        return report
+
+    def write_report(self, out_path: Path) -> None:
+        """Write the JSON report."""
+        out_path.write_text(
+            json.dumps(self.build_report(), indent=2) + "\n"
+        )
+
+    def print_summary(self) -> None:
+        """Print a human-readable summary to stderr."""
+        if not self.tests:
+            print("\n[docker-profile] No tests profiled.", file=sys.stderr)
+            return
+
+        print("\n" + "=" * 72, file=sys.stderr)
+        print("[docker-profile] Docker + sleep timing breakdown", file=sys.stderr)
+        print("=" * 72, file=sys.stderr)
+
+        subcmd_totals: dict[str, float] = defaultdict(float)
+        subcmd_counts: dict[str, int] = defaultdict(int)
+        for tp in self.tests.values():
+            for sub, calls in tp.by_subcommand().items():
+                subcmd_totals[sub] += sum(c.duration_s for c in calls)
+                subcmd_counts[sub] += len(calls)
+        total_sleep = sum(tp.total_sleep_s for tp in self.tests.values())
+        total_docker = sum(tp.total_docker_s for tp in self.tests.values())
+        total_wall = total_docker + total_sleep
+
+        print(
+            f"\n  Total wall time: {total_wall:.1f}s"
+            f" = {total_docker:.1f}s docker + {total_sleep:.1f}s sleep\n",
+            file=sys.stderr,
+        )
+        print(
+            f"  {'Category':<15} {'Count':>8} {'Total':>10} {'Avg':>8} {'%':>6}",
+            file=sys.stderr,
+        )
+        print(
+            f"  {'─' * 15} {'─' * 8} {'─' * 10} {'─' * 8} {'─' * 6}",
+            file=sys.stderr,
+        )
+        for sub in sorted(
+            subcmd_totals, key=lambda s: subcmd_totals[s], reverse=True
+        ):
+            t = subcmd_totals[sub]
+            n = subcmd_counts[sub]
+            pct = (t / total_wall * 100) if total_wall else 0
+            print(
+                f"  docker {sub:<8} {n:>8} {t:>9.1f}s {t / n:>7.2f}s {pct:>5.1f}%",
+                file=sys.stderr,
+            )
+        # Sleep row
+        sleep_count = sum(len(tp.sleeps) for tp in self.tests.values())
+        pct = (total_sleep / total_wall * 100) if total_wall else 0
+        print(
+            f"  {'time.sleep':<15} {sleep_count:>8} {total_sleep:>9.1f}s"
+            f" {total_sleep / sleep_count if sleep_count else 0:>7.2f}s {pct:>5.1f}%",
+            file=sys.stderr,
+        )
+
+        # Top 10 slowest tests by WALL time
+        print(
+            f"\n  Top 10 slowest tests (by wall time = docker + sleep):\n",
+            file=sys.stderr,
+        )
+        sorted_tests = sorted(
+            self.tests.values(), key=lambda t: t.total_wall_s, reverse=True
+        )
+        for i, tp in enumerate(sorted_tests[:10], 1):
+            print(
+                f"  {i:>2}. {tp.total_wall_s:>6.1f}s "
+                f"({tp.total_docker_s:.1f}s docker + {tp.total_sleep_s:.1f}s sleep) "
+                f"{tp.call_count:>3} calls  ...{tp.name[-50:]}",
+                file=sys.stderr,
+            )
+
+        print("\n" + "=" * 72, file=sys.stderr)
+
+
+# ---------------------------------------------------------------------------
+# Plugin hooks
+# ---------------------------------------------------------------------------
+
+_collector: Optional[ProfileCollector] = None
+
+
+def _get_collector() -> ProfileCollector:
+    global _collector
+    if _collector is None:
+        _collector = ProfileCollector()
+    return _collector
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_call(item):
+    """Wrap each test call to track per-test docker operations."""
+    if not _ACTIVE:
+        yield
+        return
+    collector = _get_collector()
+    collector.start_test(item.nodeid)
+    yield
+    collector.end_test()
+
+
+def pytest_sessionstart(session):
+    """Install the subprocess.run + time.sleep patches at session start."""
+    if not _ACTIVE:
+        return
+    collector = _get_collector()
+    collector.install_patch()
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """Write the report and print the summary at session end."""
+    if not _ACTIVE:
+        return
+    collector = _get_collector()
+    collector.uninstall_patch()
+
+    default_out = str(Path.cwd() / f"docker-test-profile-{os.getpid()}.json")
+    out = os.environ.get("HERMES_DOCKER_PROFILE_OUT", default_out)
+    out_path = Path(out)
+    collector.write_report(out_path)
+    collector.print_summary()
+    print(f"\n[docker-profile] Report written to {out_path}", file=sys.stderr)

tests/docker/test_dashboard.py

@@ -19,11 +19,10 @@ from tests.docker.conftest import docker_exec, docker_exec_sh, start_container,
 
 
 def test_dashboard_not_running_by_default(
-    built_image: str, container_name: str,
+    shared_container: str,
 ) -> None:
     """Without HERMES_DASHBOARD, no dashboard process should be running."""
-    start_container(built_image, container_name, cmd="sleep 60")
-    r = docker_exec(container_name, "pgrep", "-f", "hermes dashboard")
+    r = docker_exec(shared_container, "pgrep", "-f", "hermes dashboard")
     # pgrep exits non-zero when no match found
     assert r.returncode != 0, (
         "Dashboard should not be running without HERMES_DASHBOARD"
@@ -31,7 +30,7 @@ def test_dashboard_not_running_by_default(
 
 
 def test_dashboard_slot_reports_down_when_disabled(
-    built_image: str, container_name: str,
+    shared_container: str,
 ) -> None:
     """Without HERMES_DASHBOARD, s6-svstat should report the dashboard
     slot as DOWN (not up-with-sleep-infinity, which would
@@ -41,11 +40,10 @@ def test_dashboard_slot_reports_down_when_disabled(
     writes a `down` marker file in the live service-dir when
     HERMES_DASHBOARD is unset, so the slot reflects reality.
     """
-    start_container(built_image, container_name, cmd="sleep 60")
     # /command/ isn't on PATH for docker-exec sessions, so call by
     # absolute path.
     r = docker_exec(
-        container_name, "/command/s6-svstat", "/run/service/dashboard",
+        shared_container, "/command/s6-svstat", "/run/service/dashboard",
     )
     assert r.returncode == 0, f"s6-svstat failed: {r.stderr!r} / {r.stdout!r}"
     assert "down" in r.stdout, (

tests/docker/test_dump_build_sha.py

@@ -25,24 +25,21 @@ from __future__ import annotations
 import re
 import subprocess
 
+from tests.docker.conftest import docker_exec_sh, wait_for_container_ready
+
 
 _VERSION_LINE = re.compile(r"^version:\s+(?P<rest>.+)$", re.MULTILINE)
 _SHA_BRACKET = re.compile(r"\[(?P<sha>[^\]]+)\]\s*$")
 
 
-def _run_dump(image: str) -> str:
-    """Return the stdout of ``docker run <image> dump``.
+def _run_dump(container: str) -> str:
+    """Return the stdout of ``hermes dump`` inside the running container.
 
-    Relies on Docker's anonymous VOLUME for ``/opt/data`` (declared by the
-    Dockerfile) so the container's hermes user (UID 10000) can bootstrap
-    its config.  Anonymous volumes are auto-cleaned by ``--rm``, so unlike
-    a host bind-mount we don't have to chown anything to UID 10000 (which
-    would break cleanup on non-root hosts).
+    The container is already booted with ``sleep infinity`` by the
+    ``shared_container`` fixture, so we just ``docker exec`` the command
+    instead of paying the full ``docker run`` startup cost each time.
     """
-    r = subprocess.run(
-        ["docker", "run", "--rm", image, "dump"],
-        capture_output=True, text=True, timeout=120,
-    )
+    r = docker_exec_sh(container, "hermes dump", timeout=60)
     assert r.returncode == 0, (
         f"hermes dump exited {r.returncode}: "
         f"stderr={r.stderr[-1000:]!r}\nstdout={r.stdout[-1000:]!r}"
@@ -50,29 +47,25 @@ def _run_dump(image: str) -> str:
     return r.stdout
 
 
-def _read_baked_sha_from_image(image: str) -> str | None:
+def _read_baked_sha_from_container(container: str) -> str | None:
     """Return the ``/opt/hermes/.hermes_build_sha`` content, or None if absent."""
-    r = subprocess.run(
-        [
-            "docker", "run", "--rm", "--entrypoint", "cat", image,
-            "/opt/hermes/.hermes_build_sha",
-        ],
-        capture_output=True, text=True, timeout=30,
+    r = docker_exec_sh(
+        container, "cat /opt/hermes/.hermes_build_sha 2>/dev/null", timeout=10,
     )
-    if r.returncode != 0:
+    if r.returncode != 0 or not r.stdout.strip():
         return None
     return r.stdout.strip() or None
 
 
-def test_dump_reports_baked_sha_when_present(built_image: str) -> None:
+def test_dump_reports_baked_sha_when_present(shared_container: str) -> None:
     """When the image was built with ``HERMES_GIT_SHA``, dump must surface it.
 
     Together with the smoke-test action (which exercises ``--help``), this
     closes the regression loop for the missing-sha bug: any future change
     that breaks the baked-file -> dump pipeline will fail CI here.
     """
-    baked = _read_baked_sha_from_image(built_image)
-    stdout = _run_dump(built_image)
+    baked = _read_baked_sha_from_container(shared_container)
+    stdout = _run_dump(shared_container)
 
     match = _VERSION_LINE.search(stdout)
     assert match, f"no `version:` line in dump output:\n{stdout[:2000]}"

tests/docker/test_immutable_install.py

@@ -7,6 +7,10 @@ Build the real image and verify at runtime:
   3. /opt/hermes/.install_method contains "docker" (code-scoped stamp)
   4. $HERMES_HOME/.install_method is NOT stamped as "docker" by stage2
   5. A stale "docker" stamp in $HERMES_HOME is healed (removed) on boot
+
+Tests 1–4 are read-only checks against the default container state and
+share the module-scoped ``shared_container`` fixture. Test 5 mutates
+state and triggers a restart, so it uses its own container.
 """
 from __future__ import annotations
 
@@ -19,7 +23,7 @@ from tests.docker.conftest import (
 
 
 def test_install_tree_not_writable_by_hermes(
-    built_image: str, container_name: str,
+    shared_container: str,
 ) -> None:
     """The hermes user must not be able to modify /opt/hermes.
 
@@ -27,10 +31,8 @@ def test_install_tree_not_writable_by_hermes(
     root-owned and non-writable so an agent session cannot self-modify
     the installation and brick the gateway.
     """
-    start_container(built_image, container_name)
-
     r = docker_exec_sh(
-        container_name,
+        shared_container,
         # Try to create a file under /opt/hermes as the hermes user
         "touch /opt/hermes/test_write 2>&1 && "
         "echo WRITE_SUCCEEDED || echo WRITE_FAILED",
@@ -43,7 +45,7 @@ def test_install_tree_not_writable_by_hermes(
 
     # Also check a key subdirectory
     r = docker_exec_sh(
-        container_name,
+        shared_container,
         "touch /opt/hermes/.venv/test_write 2>&1 && "
         "echo WRITE_SUCCEEDED || echo WRITE_FAILED",
         timeout=10,
@@ -54,15 +56,13 @@ def test_install_tree_not_writable_by_hermes(
 
 
 def test_hermes_disable_lazy_installs_and_dont_write_bytecode(
-    built_image: str, container_name: str,
+    shared_container: str,
 ) -> None:
     """The container must set PYTHONDONTWRITEBYTECODE and
     HERMES_DISABLE_LAZY_INSTALLS=1 so no .pyc files are written to the
     immutable install tree and no lazy installs attempt to modify it."""
-    start_container(built_image, container_name)
-
     r = docker_exec_sh(
-        container_name,
+        shared_container,
         'test "$PYTHONDONTWRITEBYTECODE" = "1" && '
         'test "$HERMES_DISABLE_LAZY_INSTALLS" = "1" && '
         'echo ENV_OK || echo ENV_MISSING',
@@ -75,15 +75,13 @@ def test_hermes_disable_lazy_installs_and_dont_write_bytecode(
 
 
 def test_install_method_stamp_is_code_scoped(
-    built_image: str, container_name: str,
+    shared_container: str,
 ) -> None:
     """The 'docker' install-method stamp must be baked at
     /opt/hermes/.install_method (code-scoped), NOT in $HERMES_HOME."""
-    start_container(built_image, container_name)
-
     # Code-scoped stamp must exist and say "docker"
     r = docker_exec_sh(
-        container_name,
+        shared_container,
         "cat /opt/hermes/.install_method",
         timeout=10,
     )
@@ -96,7 +94,7 @@ def test_install_method_stamp_is_code_scoped(
 
     # $HERMES_HOME must NOT have a 'docker' stamp
     r = docker_exec_sh(
-        container_name,
+        shared_container,
         "cat /opt/data/.install_method 2>/dev/null || echo NONE",
         timeout=10,
     )

tests/docker/test_immutable_install_permissions.py

@@ -1,27 +1,25 @@
 """Docker smoke tests for immutable install permissions."""
 from __future__ import annotations
 
-import subprocess
 import textwrap
 
+from tests.docker.conftest import docker_exec_sh
 
-def test_container_sets_hosted_write_policy_env(built_image: str) -> None:
+
+def test_container_sets_hosted_write_policy_env(shared_container: str) -> None:
     script = (
         'test "$HERMES_HOME" = "/opt/data" && '
         'test "$HERMES_WRITE_SAFE_ROOT" = "/opt/data" && '
         'test "$HERMES_DISABLE_LAZY_INSTALLS" = "1" && '
         'test "$PYTHONDONTWRITEBYTECODE" = "1"'
     )
-    result = subprocess.run(
-        ["docker", "run", "--rm", "--entrypoint", "sh", built_image, "-c", script],
-        capture_output=True,
-        text=True,
-        timeout=60,
-    )
-    assert result.returncode == 0, result.stderr[-2000:]
+    r = docker_exec_sh(shared_container, script, timeout=30)
+    assert r.returncode == 0, r.stderr[-2000:]
 
 
-def test_hermes_user_cannot_modify_install_but_can_write_data(built_image: str) -> None:
+def test_hermes_user_cannot_modify_install_but_can_write_data(
+    shared_container: str,
+) -> None:
     script = textwrap.dedent(
         r"""
         set -eu
@@ -46,22 +44,6 @@ def test_hermes_user_cannot_modify_install_but_can_write_data(built_image: str)
         PY
         """
     ).strip()
-    result = subprocess.run(
-        [
-            "docker",
-            "run",
-            "--rm",
-            "--entrypoint",
-            "su",
-            built_image,
-            "hermes",
-            "-s",
-            "/bin/sh",
-            "-c",
-            script,
-        ],
-        capture_output=True,
-        text=True,
-        timeout=120,
-    )
-    assert result.returncode == 0, result.stderr[-2000:]
+    # Run as hermes user via docker_exec_sh's default user context.
+    r = docker_exec_sh(shared_container, script, timeout=60)
+    assert r.returncode == 0, r.stderr[-2000:]

tests/docker/test_license_file_present.py

@@ -6,21 +6,17 @@ Docker image).
 """
 from __future__ import annotations
 
-import subprocess
+from tests.docker.conftest import docker_exec
 
 
-def test_docker_image_contains_license_file(built_image: str) -> None:
+def test_docker_image_contains_license_file(shared_container: str) -> None:
     """The LICENSE file must be present inside the built Docker image.
 
     PEP 639 license-files metadata references LICENSE, and the Docker
     build context must not exclude it.
     """
-    r = subprocess.run(
-        ["docker", "run", "--rm", "--entrypoint", "test",
-         built_image, "-f", "/opt/hermes/LICENSE"],
-        capture_output=True, text=True, timeout=60,
-    )
+    r = docker_exec(shared_container, "test", "-f", "/opt/hermes/LICENSE")
     assert r.returncode == 0, (
         f"LICENSE file not found at /opt/hermes/LICENSE inside the Docker "
         f"image: {r.stderr[-500:]}"
-    )
+    )

tests/docker/test_tini_compat_shim.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 import subprocess
 
 
-def test_tini_compat_symlink_exists(built_image: str) -> None:
+def test_tini_compat_symlink_exists(shared_container: str) -> None:
     """/usr/bin/tini must exist as a symlink to /init.
 
     Regression for #34192: orchestration templates (e.g. Hostinger's
@@ -20,11 +20,10 @@ def test_tini_compat_symlink_exists(built_image: str) -> None:
     PID-1 reaper without behavior change.
     """
     r = subprocess.run(
-        ["docker", "run", "--rm", "--entrypoint", "sh",
-         built_image, "-c",
+        ["docker", "exec", shared_container, "sh", "-c",
          'test -L /usr/bin/tini && '
          'test "$(readlink -f /usr/bin/tini)" = "/init"'],
-        capture_output=True, text=True, timeout=60,
+        capture_output=True, text=True, timeout=30,
     )
     assert r.returncode == 0, (
         f"/usr/bin/tini is not a symlink to /init: {r.stderr[-500:]}"
@@ -51,4 +50,4 @@ def test_entrypoint_is_init_not_tini(built_image: str) -> None:
     # /usr/bin/tini should NOT be in the entrypoint.
     assert "tini" not in entrypoint.lower(), (
         f"ENTRYPOINT references tini instead of /init: {entrypoint!r}"
-    )
+    )

tests/docker/test_tui_prebuilt_bundle.py

@@ -18,33 +18,26 @@ from __future__ import annotations
 
 import json
 import shlex
-import subprocess
+
+from tests.docker.conftest import docker_exec_sh
 
 
-def _exec_py(image: str, py: str) -> str:
-    """Run a Python snippet inside the image as the hermes user, return stdout."""
+def _exec_py(container: str, py: str) -> str:
+    """Run a Python snippet inside the container as the hermes user, return stdout."""
     inner = (
-        "source /opt/hermes/.venv/bin/activate && "
+        ". /opt/hermes/.venv/bin/activate && "
         "cd /opt/hermes && "
         f"python3 -c {shlex.quote(py)}"
     )
-    # Drop to the hermes user (UID 10000) so we exercise the same path the
-    # dashboard PTY child runs as — not root.
-    cmd = [
-        "docker", "run", "--rm", "--entrypoint", "su", image,
-        "hermes", "-s", "/bin/bash", "-c", inner,
-    ]
-    r = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
+    r = docker_exec_sh(container, inner, timeout=60)
     assert r.returncode == 0, f"in-container python failed:\n{r.stderr[-2000:]}"
     return r.stdout.strip()
 
 
-def test_hermes_tui_dir_env_is_set(built_image: str) -> None:
+def test_hermes_tui_dir_env_is_set(shared_container: str) -> None:
     """HERMES_TUI_DIR must point at the prebuilt bundle dir in the image."""
-    r = subprocess.run(
-        ["docker", "run", "--rm", "--entrypoint", "sh", built_image,
-         "-c", 'printf "%s" "$HERMES_TUI_DIR"'],
-        capture_output=True, text=True, timeout=60,
+    r = docker_exec_sh(
+        shared_container, 'printf "%s" "$HERMES_TUI_DIR"', timeout=30,
     )
     assert r.returncode == 0, r.stderr[-2000:]
     assert r.stdout.strip() == "/opt/hermes/ui-tui", (
@@ -52,7 +45,9 @@ def test_hermes_tui_dir_env_is_set(built_image: str) -> None:
     )
 
 
-def test_prebuilt_bundle_present_and_no_runtime_install(built_image: str) -> None:
+def test_prebuilt_bundle_present_and_no_runtime_install(
+    shared_container: str,
+) -> None:
     """The launcher must (a) find the prebuilt bundle and (b) NOT want an
     npm install — i.e. it takes the same path as a nix/packaged release."""
     py = (
@@ -69,7 +64,7 @@ def test_prebuilt_bundle_present_and_no_runtime_install(built_image: str) -> Non
         "}\n"
         "print(json.dumps(out))\n"
     )
-    out = json.loads(_exec_py(built_image, py))
+    out = json.loads(_exec_py(shared_container, py))
     assert out["dist_entry_exists"], "prebuilt ui-tui/dist/entry.js missing from image"
     # With HERMES_TUI_DIR set, _make_tui_argv returns the prebuilt path BEFORE
     # ever reaching the install check — so the resolved argv is what matters.