fix(checkpoints): require positive volume-attachment evidence before orphan classification

Follow-up to the cherry-picked #69063: egilewski's review found that the
_dir_has_any_entry(parent) guard treats ANY entry in the mount point's
parent as proof the volume is attached — but unmounting exposes the
UNDERLAY directory's own files (e.g. a .keep placeholder), so a populated
underlying mount-point dir still classified the project as an orphan and
deleted its ref/index/metadata. Reproduced on both main and the PR head.

Attachment evidence is now positive instead of circumstantial:

* _volume_evidence() records the parent directory's (st_dev, st_ino)
  identity in the project's metadata while the workdir is observably
  live (at _register_project/_touch_project time). A mount point
  resolves to the mounted filesystem's root while attached and to the
  underlay directory after detach — same path, different directory,
  different identity.
* _workdir_is_observably_gone() now requires the parent visible at
  prune time to match that recorded identity before the populated-parent
  check can classify an orphan. A mismatch means a different directory
  (the underlay) is showing through — a detached volume, not an
  observed deletion.
* Metadata without a recorded identity (written by older versions) is
  never orphan-classified — unsure never deletes; the retention/stale
  rule still reclaims genuinely abandoned projects off last_touch.
* The frozen pre-v2 layout has no metadata channel for the identity, so
  it keeps the structural checks only (require_parent_identity=False).
* A failed evidence probe on re-registration preserves the previously
  recorded identity — stale evidence can only make pruning MORE
  conservative.

Windows: st_dev/st_ino of 0 (filesystems without file IDs, some network
shares) is treated as "no evidence recorded", which falls into the
conservative never-orphan path. os.path.ismount and Path.stat are
cross-platform; no POSIX-only calls added.

tests/tools/test_checkpoint_manager.py: adds egilewski's exact
regression (checkpoint history for mnt/volume/project, detach exposes
mnt/volume/.keep, prune with orphan deletion enabled → NOT deleted;
fails on the bare cherry-pick, passes with this fix), plus
no-recorded-identity conservatism and probe-failure identity
preservation. His absent-parent/empty-parent/retention/genuine-deletion/
live-project controls all still pass.

Reported-by: egilewski (review on #69063)

tests/tools/test_checkpoint_manager.py

@@ -4,6 +4,7 @@ import argparse
 import json
 import logging
 import os
+import shutil
 import subprocess
 import time
 import pytest
@@ -1156,3 +1157,262 @@ class TestClearFunctions:
         result = clear_all()
         assert result["deleted"] is False
         assert result["bytes_freed"] == 0
+
+
+# =========================================================================
+# Orphan pruning must not act on an unreachable volume
+# =========================================================================
+
+class TestOrphanPruneRequiresObservableDeletion:
+    """A missing workdir is ambiguous: deleted, or just not mounted right now.
+
+    Orphan pruning deletes a project's whole checkpoint history and runs
+    unattended at startup (``maybe_auto_prune_checkpoints`` from the CLI and
+    the gateway), so it must only fire when the deletion is something we
+    actually observed — the parent directory present, the project gone. An
+    unplugged drive, a share behind a downed VPN, or a bind-mount absent from
+    this container must not cost the user their restore points.
+    """
+
+    def _project_with_history(self, work_dir, checkpoint_base, monkeypatch):
+        monkeypatch.setattr(
+            "tools.checkpoint_manager.CHECKPOINT_BASE", checkpoint_base
+        )
+        m = CheckpointManager(enabled=True, max_snapshots=10)
+        m.ensure_checkpoint(str(work_dir), "initial")
+        return m
+
+    def _history_survives(self, checkpoint_base, work_dir):
+        store = _store_path(checkpoint_base)
+        return _project_meta_path(
+            store, _project_hash(str(work_dir))
+        ).exists()
+
+    def test_unreachable_volume_keeps_its_checkpoints(
+        self, tmp_path, checkpoint_base, monkeypatch,
+    ):
+        """The whole mount is absent (parent missing too) → keep the history."""
+        mount = tmp_path / "mnt" / "nas"
+        work_dir = mount / "work" / "proj"
+        work_dir.mkdir(parents=True)
+        (work_dir / "main.py").write_text("print('x')\n")
+        self._project_with_history(work_dir, checkpoint_base, monkeypatch)
+
+        # The volume goes away — project AND its parents disappear together.
+        shutil.rmtree(tmp_path / "mnt")
+        assert not work_dir.exists()
+
+        prune_checkpoints(
+            retention_days=0, delete_orphans=True, checkpoint_base=checkpoint_base,
+        )
+
+        assert self._history_survives(checkpoint_base, work_dir), (
+            "checkpoint history was deleted for a project whose volume was "
+            "merely unmounted"
+        )
+
+    def test_surviving_empty_mountpoint_keeps_its_checkpoints(
+        self, tmp_path, checkpoint_base, monkeypatch,
+    ):
+        """The mount point outlives the volume as an empty dir → keep history.
+
+        The classic static layout (``/mnt/volume/proj``, an fstab entry, a
+        container bind-mount) does not remove the mount point on unmount: the
+        project disappears but ``/mnt/volume`` stays behind as an empty
+        directory. The parent being present is therefore not evidence on its
+        own — an empty parent looks exactly the same whether the volume was
+        detached or the project was deleted.
+        """
+        mount_point = tmp_path / "mnt" / "volume"
+        work_dir = mount_point / "proj"
+        work_dir.mkdir(parents=True)
+        (work_dir / "main.py").write_text("print('x')\n")
+        self._project_with_history(work_dir, checkpoint_base, monkeypatch)
+
+        # Unmount: contents go, the mount point itself remains.
+        shutil.rmtree(work_dir)
+        assert mount_point.is_dir() and not any(mount_point.iterdir())
+
+        prune_checkpoints(
+            retention_days=0, delete_orphans=True, checkpoint_base=checkpoint_base,
+        )
+
+        assert self._history_survives(checkpoint_base, work_dir), (
+            "checkpoint history was deleted for a project whose mount point "
+            "merely survived the unmount as an empty directory"
+        )
+
+    def test_empty_parent_project_is_still_reclaimed_by_retention(
+        self, tmp_path, checkpoint_base, monkeypatch,
+    ):
+        """Skipping an ambiguous parent defers reclamation, it does not lose it.
+
+        A project deleted out of an otherwise-empty parent is indistinguishable
+        from an unmounted volume, so orphan pruning leaves it alone — but the
+        retention rule, which reads ``last_touch`` instead of probing the
+        filesystem, still reclaims it.
+        """
+        parent = tmp_path / "solo"
+        work_dir = parent / "proj"
+        work_dir.mkdir(parents=True)
+        (work_dir / "main.py").write_text("print('x')\n")
+        self._project_with_history(work_dir, checkpoint_base, monkeypatch)
+
+        shutil.rmtree(work_dir)
+        assert parent.is_dir() and not any(parent.iterdir())
+
+        store = _store_path(checkpoint_base)
+        meta_path = _project_meta_path(store, _project_hash(str(work_dir)))
+        meta = json.loads(meta_path.read_text())
+        meta["last_touch"] = time.time() - 60 * 86400
+        meta_path.write_text(json.dumps(meta))
+
+        result = prune_checkpoints(
+            retention_days=30, delete_orphans=True, checkpoint_base=checkpoint_base,
+        )
+
+        assert result["deleted_stale"] >= 1
+        assert not self._history_survives(checkpoint_base, work_dir)
+
+    def test_genuinely_deleted_project_is_still_pruned(
+        self, tmp_path, checkpoint_base, monkeypatch,
+    ):
+        """Control: a populated parent without the project → a real orphan."""
+        parent = tmp_path / "projects"
+        work_dir = parent / "proj"
+        work_dir.mkdir(parents=True)
+        (work_dir / "main.py").write_text("print('x')\n")
+        (parent / "other-project").mkdir()
+        self._project_with_history(work_dir, checkpoint_base, monkeypatch)
+
+        shutil.rmtree(work_dir)
+        assert parent.exists() and not work_dir.exists()
+
+        result = prune_checkpoints(
+            retention_days=0, delete_orphans=True, checkpoint_base=checkpoint_base,
+        )
+
+        assert result["deleted_orphan"] >= 1
+        assert not self._history_survives(checkpoint_base, work_dir)
+
+    def test_live_project_is_never_pruned_as_orphan(
+        self, work_dir, checkpoint_base, monkeypatch,
+    ):
+        """Control: a present project keeps its history."""
+        self._project_with_history(work_dir, checkpoint_base, monkeypatch)
+
+        result = prune_checkpoints(
+            retention_days=0, delete_orphans=True, checkpoint_base=checkpoint_base,
+        )
+
+        assert result["deleted_orphan"] == 0
+        assert self._history_survives(checkpoint_base, work_dir)
+
+    def test_populated_underlay_mountpoint_keeps_its_checkpoints(
+        self, tmp_path, checkpoint_base, monkeypatch,
+    ):
+        """A detached volume exposing a populated underlay dir → keep history.
+
+        An entry in the parent does not prove the project volume is attached.
+        Unmounting swaps which directory is visible at the mount point's
+        path: the mounted filesystem's root gives way to the *underlying*
+        directory, whose own files (a ``.keep`` placeholder, sibling mount
+        points) become visible. Those entries were never next to the project,
+        so they must not corroborate an orphan classification — yet a naive
+        "parent has any entry" guard treats them as proof of deletion.
+
+        Simulate the detach faithfully: the directory visible at
+        ``mnt/volume`` after the unmount is a *different* directory (new
+        inode — the underlay) carrying a ``.keep`` placeholder.
+        """
+        mount_point = tmp_path / "mnt" / "volume"
+        work_dir = mount_point / "project"
+        work_dir.mkdir(parents=True)
+        (work_dir / "main.py").write_text("print('x')\n")
+        self._project_with_history(work_dir, checkpoint_base, monkeypatch)
+
+        # Detach: the directory that was mounted at the path (with the
+        # project on it) stops being visible there; the underlay directory —
+        # same path, different directory, its own contents — shows through,
+        # exposing a placeholder. Renaming (rather than deleting) the mounted
+        # directory keeps its inode alive, so the underlay directory is
+        # guaranteed a distinct identity, exactly as in a real unmount.
+        mount_point.rename(tmp_path / "detached-volume")
+        mount_point.mkdir()
+        (mount_point / ".keep").write_text("")
+        assert not work_dir.exists()
+        assert any(mount_point.iterdir())
+
+        result = prune_checkpoints(
+            retention_days=0, delete_orphans=True, checkpoint_base=checkpoint_base,
+        )
+
+        assert result["deleted_orphan"] == 0
+        assert self._history_survives(checkpoint_base, work_dir), (
+            "checkpoint history was deleted because underlay entries exposed "
+            "by an unmount were mistaken for attachment evidence"
+        )
+
+    def test_metadata_without_parent_identity_is_not_orphan_classified(
+        self, tmp_path, checkpoint_base, monkeypatch,
+    ):
+        """Metadata from an older version (no recorded identity) → keep.
+
+        Without a recorded parent ``(st_dev, st_ino)`` we cannot tell the
+        project's real parent from an underlay directory exposed by an
+        unmount, so the classification stays conservative: not an orphan.
+        The retention rule still reclaims genuinely abandoned projects.
+        """
+        parent = tmp_path / "projects"
+        work_dir = parent / "proj"
+        work_dir.mkdir(parents=True)
+        (work_dir / "main.py").write_text("print('x')\n")
+        (parent / "other-project").mkdir()
+        self._project_with_history(work_dir, checkpoint_base, monkeypatch)
+
+        # Strip the recorded identity, as metadata written by an older
+        # version would lack it.
+        store = _store_path(checkpoint_base)
+        meta_path = _project_meta_path(store, _project_hash(str(work_dir)))
+        meta = json.loads(meta_path.read_text())
+        meta.pop("workdir_parent_dev", None)
+        meta.pop("workdir_parent_ino", None)
+        meta_path.write_text(json.dumps(meta))
+
+        shutil.rmtree(work_dir)
+
+        result = prune_checkpoints(
+            retention_days=0, delete_orphans=True, checkpoint_base=checkpoint_base,
+        )
+
+        assert result["deleted_orphan"] == 0
+        assert self._history_survives(checkpoint_base, work_dir)
+
+    def test_recorded_parent_identity_survives_probe_failure_on_touch(
+        self, tmp_path, checkpoint_base, monkeypatch,
+    ):
+        """A later failed evidence probe must not erase recorded identity."""
+        from tools.checkpoint_manager import _register_project
+
+        parent = tmp_path / "projects"
+        work_dir = parent / "proj"
+        work_dir.mkdir(parents=True)
+        (work_dir / "main.py").write_text("print('x')\n")
+        self._project_with_history(work_dir, checkpoint_base, monkeypatch)
+
+        store = _store_path(checkpoint_base)
+        meta_path = _project_meta_path(store, _project_hash(str(work_dir)))
+        before = json.loads(meta_path.read_text())
+        assert "workdir_parent_dev" in before
+        assert "workdir_parent_ino" in before
+
+        # Re-register while the evidence probe fails (e.g. transient I/O
+        # error): the previously recorded identity must be preserved.
+        monkeypatch.setattr(
+            "tools.checkpoint_manager._volume_evidence", lambda _wd: {},
+        )
+        _register_project(store, str(work_dir))
+
+        after = json.loads(meta_path.read_text())
+        assert after["workdir_parent_dev"] == before["workdir_parent_dev"]
+        assert after["workdir_parent_ino"] == before["workdir_parent_ino"]

tools/checkpoint_manager.py

@@ -483,6 +483,39 @@ def _init_store(store: Path, working_dir: str) -> Optional[str]:
     return None
 
 
+def _volume_evidence(workdir: Path) -> Dict:
+    """Record the identity of ``workdir``'s parent while the project is live.
+
+    ``(st_dev, st_ino)`` of the parent directory, captured at a moment when
+    the workdir itself is reachable, identifies the *directory* — not just
+    the path.  A mount point resolves to the mounted filesystem's root while
+    the volume is attached and to the underlying (underlay) directory after
+    unmount: same path, different directory, different ``(st_dev, st_ino)``.
+    Orphan pruning uses this to distinguish "the project was deleted out of
+    the directory we knew" from "a different directory is now visible at
+    that path because the volume is detached".
+
+    Returns ``{}`` when the workdir is not currently reachable, when the
+    filesystem does not provide a usable directory identity (a zero
+    ``st_dev`` or ``st_ino`` — e.g. Windows filesystems without file IDs and
+    some network shares), or when the probe fails — callers treat all of
+    these as "no evidence recorded" and orphan pruning stays conservative
+    for the project (never classified as orphan; retention still applies).
+    """
+    try:
+        if not workdir.exists():
+            return {}
+        st = workdir.parent.stat()
+        if not st.st_dev or not st.st_ino:
+            return {}
+        return {
+            "workdir_parent_dev": st.st_dev,
+            "workdir_parent_ino": st.st_ino,
+        }
+    except OSError:
+        return {}
+
+
 def _register_project(store: Path, working_dir: str) -> None:
     """Create or update ``projects/<hash>.json`` with workdir + timestamps."""
     dir_hash = _project_hash(working_dir)
@@ -490,11 +523,22 @@ def _register_project(store: Path, working_dir: str) -> None:
     now = time.time()
     meta: Dict = {"workdir": str(_normalize_path(working_dir)),
                   "created_at": now, "last_touch": now}
+    evidence = _volume_evidence(_normalize_path(working_dir))
+    if evidence:
+        meta.update(evidence)
     if meta_path.exists():
         try:
             existing = json.loads(meta_path.read_text(encoding="utf-8"))
             if isinstance(existing, dict):
                 meta["created_at"] = existing.get("created_at", now)
+                if not evidence:
+                    # Fresh probe failed — keep the previously recorded
+                    # parent identity rather than dropping it. Stale evidence
+                    # only makes pruning MORE conservative (mismatch => not
+                    # an orphan).
+                    for key in ("workdir_parent_dev", "workdir_parent_ino"):
+                        if key in existing:
+                            meta[key] = existing[key]
         except (OSError, ValueError):
             pass
     try:
@@ -520,6 +564,13 @@ def _touch_project(store: Path, working_dir: str) -> None:
     meta["workdir"] = str(_normalize_path(working_dir))
     meta["last_touch"] = time.time()
     meta.setdefault("created_at", meta["last_touch"])
+    # Refresh the parent-directory identity while the project is observably
+    # live — a remount can legitimately change it (new device, new inode).
+    # On probe failure the previous evidence is kept: stale evidence can only
+    # make pruning MORE conservative (mismatch => not an orphan).
+    evidence = _volume_evidence(_normalize_path(working_dir))
+    if evidence:
+        meta.update(evidence)
     try:
         meta_path.write_text(json.dumps(meta), encoding="utf-8")
     except OSError as exc:
@@ -564,16 +615,21 @@ def _pre_v2_shadow_repos(base: Path) -> List[Dict]:
         if not (child / "HEAD").exists():
             continue
         workdir: Optional[str] = None
+        marker_unreadable = False
         wd_marker = child / "HERMES_WORKDIR"
         if wd_marker.exists():
             try:
                 workdir = wd_marker.read_text(encoding="utf-8").strip()
             except (OSError, UnicodeDecodeError):
+                # The marker is there, we just could not read it. That is
+                # not evidence the project is gone — never delete on it.
                 workdir = None
+                marker_unreadable = True
         out.append({
             "path": child,
             "workdir": workdir,
             "exists": bool(workdir) and Path(workdir).exists(),
+            "marker_unreadable": marker_unreadable,
         })
     return out
 
@@ -1288,6 +1344,106 @@ def _delete_ref(store: Path, ref: str) -> bool:
     return ok
 
 
+def _workdir_is_observably_gone(
+    workdir: str,
+    parent_dev: Optional[int] = None,
+    parent_ino: Optional[int] = None,
+    require_parent_identity: bool = True,
+) -> bool:
+    """True only when we can positively observe that ``workdir`` was removed.
+
+    ``Path.exists()`` returns False for a deleted directory AND for one whose
+    storage simply is not attached right now — an unplugged external drive, a
+    network share behind a downed VPN, a bind-mount absent from this
+    container, an offline Windows mapped drive. Orphan pruning deletes the
+    project's entire checkpoint history, so treating that ambiguity as
+    "deleted" throws away the user's restore points over a transient mount
+    state, unattended, at startup.
+
+    Require corroboration, in three steps.
+
+    First, the parent directory must be present, so the absence of the project
+    inside it is something we actually observed. When the parent is missing
+    too, the volume is not there and we know nothing.
+
+    Second, the present parent must be the directory we knew — not merely a
+    directory at the same path. Unmounting swaps the directory visible at a
+    mount point: while the volume is attached the path resolves to the
+    mounted filesystem's root; after detach it resolves to the *underlying*
+    (underlay) directory, which may carry entries of its own (a ``.keep``
+    placeholder, sibling mount points). Those entries were never next to the
+    project and prove nothing about the volume being attached. So the
+    parent's ``(st_dev, st_ino)`` must match the identity recorded in the
+    project's metadata while the project was observably live
+    (``parent_dev``/``parent_ino``). A mismatch means a different directory
+    is visible at that path — a detached volume, not an observed deletion.
+    When no identity was ever recorded (metadata written by an older
+    version) and ``require_parent_identity`` is True, stay conservative and
+    do not classify as orphan. Callers that have no identity channel at all
+    (the frozen pre-v2 layout) pass ``require_parent_identity=False`` to
+    keep the structural checks only.
+
+    Third, the (identity-confirmed) parent must actually carry information.
+    Unmounting leaves classic static mount points (``/mnt/volume/proj``, an
+    fstab entry, a container bind-mount) behind as *empty* directories, so an
+    empty parent is the signature of a detached volume just as much as of a
+    deleted project. Prune only when the parent holds something else (we
+    observed a populated directory that does not contain the project) or is
+    itself a live mount point (the volume is demonstrably attached and the
+    project is demonstrably not on it).
+
+    Genuinely abandoned projects are still reclaimed by the retention/stale
+    rule, which runs off ``last_touch`` rather than a filesystem probe.
+    """
+    if not workdir:
+        return False
+    path = Path(workdir)
+    try:
+        if path.exists():
+            return False
+        parent = path.parent
+        # A path whose parent is itself (a filesystem root) gives us nothing
+        # to corroborate against.
+        if parent == path:
+            return False
+        if not parent.is_dir():
+            return False
+        if parent_dev is not None and parent_ino is not None:
+            st = parent.stat()
+            if (st.st_dev, st.st_ino) != (parent_dev, parent_ino):
+                # A different directory is visible at the parent's path than
+                # the one the project lived in — the volume is detached (its
+                # underlay showing through) or was swapped. Not a deletion.
+                return False
+        elif require_parent_identity:
+            # No recorded identity to check against — we cannot tell the
+            # project's real parent from an underlay directory exposed by an
+            # unmount. Unsure never deletes; retention still reclaims.
+            return False
+        if _dir_has_any_entry(parent):
+            return True
+        # Empty parent: only evidence if that directory is a mount point, i.e.
+        # the volume is attached right now and simply does not hold the
+        # project. An empty plain directory is an unmounted mount point as
+        # readily as an emptied project root.
+        return os.path.ismount(parent)
+    except OSError:
+        # Probe failed (permission, I/O error) — not evidence of deletion.
+        return False
+
+
+def _dir_has_any_entry(directory: Path) -> bool:
+    """True when ``directory`` contains at least one entry.
+
+    Stops after the first entry rather than materializing the listing; a
+    project root can hold a large tree.
+    """
+    with os.scandir(directory) as entries:
+        for _ in entries:
+            return True
+    return False
+
+
 def prune_checkpoints(
     retention_days: int = 7,
     delete_orphans: bool = True,
@@ -1380,7 +1536,16 @@ def prune_checkpoints(
         reason: Optional[str] = None
         if (
             delete_orphans
-            and not repo["exists"]
+            and not repo["marker_unreadable"]
+            and (
+                repo["workdir"] is None
+                # The frozen pre-v2 layout has no metadata channel to carry a
+                # recorded parent identity, so only the structural checks
+                # (parent present + populated / live mount point) apply here.
+                or _workdir_is_observably_gone(
+                    repo["workdir"], require_parent_identity=False,
+                )
+            )
             and (orphan_allowlist is None or str(child) in orphan_allowlist)
         ):
             reason = "orphan"
@@ -1421,9 +1586,22 @@ def prune_checkpoints(
                 continue
             result["scanned"] += 1
             reason = None
+            parent_dev = meta.get("workdir_parent_dev")
+            parent_ino = meta.get("workdir_parent_ino")
+            if not isinstance(parent_dev, int) or isinstance(parent_dev, bool):
+                parent_dev = None
+            if not isinstance(parent_ino, int) or isinstance(parent_ino, bool):
+                parent_ino = None
             if (
                 delete_orphans
-                and (not workdir or not Path(workdir).exists())
+                and (
+                    not workdir
+                    or _workdir_is_observably_gone(
+                        workdir,
+                        parent_dev=parent_dev,
+                        parent_ino=parent_ino,
+                    )
+                )
                 and (orphan_allowlist is None or dir_hash in orphan_allowlist)
             ):
                 reason = "orphan"