Merge lab after verified process lifecycle

tests/README.md

@@ -93,6 +93,39 @@ fast lane; the test stays runnable directly, e.g.:
 ./venv/bin/python -m pytest -m slow
 ```
 
+## Parallel shards (`--shard N/M`)
+
+CI no longer runs the whole suite as one workload. The `python-tests` job is a
+four-way matrix, and each job runs one section:
+
+```bash
+./venv/bin/python -m pytest -q --shard 1/4
+```
+
+`tests/_shards.py` owns the partition and `tests/conftest.py` applies it. The
+unit of a shard is a **test file**, so tests that share module state stay
+together, and assignment is a total function of the file path - every file
+lands in exactly one shard, and the four shards together run every test exactly
+once. The partition is deliberately *not* built on the `area_*` markers: those
+do not partition the suite, because a file may carry a hand-applied `area_*`
+mark on top of the one derived from its filename.
+
+Sharding deselects; it does not narrow collection. Every test module is still
+imported, in the same order, in every shard, so the import-time stubbing in
+`conftest.py` behaves identically whether the suite runs whole or in sections.
+Only the deselected tests' call phase is skipped.
+
+Balance comes from the `slow` marker: a `slow` item is weighted far above an
+ordinary one, and files are packed heaviest-first into the lightest shard. The
+plan depends only on the collected file set, so every parallel job computes the
+same one from the same commit. As more tests earn a `slow` mark from duration
+evidence, the sections even out further - no duration table to keep current.
+
+`--shard 1/1` is a no-op, and a selector that is malformed or out of range ends
+the run with a usage error rather than quietly testing a subset. If you change
+the shard count, change `DEFAULT_SHARD_COUNT` and the `ci.yml` matrix together;
+`tests/test_shards.py` fails when they drift apart.
+
 ## Order-sensitivity reporting (report-only)
 
 `tests/run_order_report.py` runs pytest with the collected test items shuffled

tests/_shards.py

@@ -0,0 +1,156 @@
+"""Deterministic, balanced partition of the test suite into parallel shards.
+
+The suite runs ~7.8k tests in a single pytest workload. This module splits that
+workload into N sections that CI runs as parallel jobs, so wall-clock time is
+bounded by the slowest section rather than by the whole suite.
+
+Two properties matter more than speed, and both are structural here rather than
+checked after the fact:
+
+* **Exhaustive and disjoint.** Shard assignment is a total function of the test
+  *file*, so every test file lands in exactly one shard. No test can be dropped
+  by a marker typo, and none runs twice. This is deliberately not built on the
+  ``area_*`` taxonomy markers: those are not a partition in practice, because a
+  test may also carry a hand-applied ``area_*`` mark on top of the one
+  ``tests/conftest.py`` derives from its filename (``test_hwfit_container_
+  visibility_warning.py`` carries three).
+* **Whole files stay together.** Tests in one file share module state and are
+  written to run in file order, so a file is the smallest unit a shard can hold.
+
+Balance uses the existing ``slow`` marker as its weight signal rather than a
+committed duration table that would go stale silently. Packing is greedy
+longest-processing-time-first, which is deterministic for a given file set -
+every parallel job computes the identical plan from the same commit.
+
+This module imports nothing from the application or from pytest - only the
+standard library - so the planner is directly unit-testable. The pytest wiring
+lives in ``tests/conftest.py``. See ``tests/README.md``.
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+# Number of sections CI runs in parallel. Kept here as documentation of the
+# intended default; the shard count actually used comes from the --shard value.
+DEFAULT_SHARD_COUNT = 4
+
+# Relative cost of one test item. Measured on dev at 2026-10-02 over a full
+# `pytest -q --durations=25` run: the five `slow`-marked items average 9.0 s
+# each and the remaining 7789 items average 0.018 s, a ratio of roughly 500.
+# Exact values do not matter - only that a `slow` item outweighs a whole
+# ordinary file, so the packer spreads the slow ones across sections first.
+DEFAULT_ITEM_WEIGHT = 1.0
+SLOW_ITEM_WEIGHT = 500.0
+
+_SHARD_SPEC_PATTERN = re.compile(r"\A(\d+)/(\d+)\Z")
+
+
+class ShardSpecError(ValueError):
+    """Raised when a ``--shard`` value is not a usable ``N/M`` selector."""
+
+
+@dataclass(frozen=True)
+class ShardSpec:
+    """A one-based shard selector: shard ``index`` of ``count``."""
+
+    index: int
+    count: int
+
+    @property
+    def selects_everything(self) -> bool:
+        """True when the selector is a no-op (``1/1``) and nothing is deselected."""
+        return self.count == 1
+
+    def __str__(self) -> str:
+        return f"{self.index}/{self.count}"
+
+
+def parse_shard_spec(value: str) -> ShardSpec:
+    """Parse ``"N/M"`` into a :class:`ShardSpec`.
+
+    Rejects anything that would silently run the wrong subset: a malformed
+    value, a zero or negative part, or an index past the shard count. Surrounding
+    whitespace is tolerated because CI passes the value through a shell variable.
+    """
+    match = _SHARD_SPEC_PATTERN.match(value.strip())
+    if match is None:
+        raise ShardSpecError(
+            f"invalid shard {value!r}: expected N/M, e.g. 1/{DEFAULT_SHARD_COUNT}"
+        )
+    index, count = int(match.group(1)), int(match.group(2))
+    if count < 1:
+        raise ShardSpecError(f"invalid shard {value!r}: shard count must be >= 1")
+    if not 1 <= index <= count:
+        raise ShardSpecError(
+            f"invalid shard {value!r}: shard index must be between 1 and {count}"
+        )
+    return ShardSpec(index=index, count=count)
+
+
+def item_weight(is_slow: bool) -> float:
+    """Weight of a single test item, by whether it carries the ``slow`` marker."""
+    return SLOW_ITEM_WEIGHT if is_slow else DEFAULT_ITEM_WEIGHT
+
+
+def accumulate_file_weights(entries: Iterable[tuple[str, bool]]) -> dict[str, float]:
+    """Sum per-item weights into a per-file total.
+
+    ``entries`` yields ``(file_key, is_slow)`` for each collected test item, so a
+    file's weight reflects both how many tests it holds and how slow they are.
+    """
+    weights: dict[str, float] = {}
+    for file_key, is_slow in entries:
+        weights[file_key] = weights.get(file_key, 0.0) + item_weight(is_slow)
+    return weights
+
+
+def plan_shards(
+    file_weights: Mapping[str, float], count: int
+) -> tuple[frozenset[str], ...]:
+    """Partition the files of ``file_weights`` into ``count`` balanced shards.
+
+    Greedy longest-processing-time-first: heaviest file first, each one placed in
+    the lightest shard so far. Ties break on the file key and then on the lowest
+    shard index, so the plan depends only on the input and is identical in every
+    parallel job. Returns one frozenset per shard, in shard order; shards may be
+    empty when there are fewer files than shards.
+    """
+    if count < 1:
+        raise ShardSpecError(f"shard count must be >= 1, got {count}")
+    buckets: list[set[str]] = [set() for _ in range(count)]
+    loads = [0.0] * count
+    # Heaviest first, with the file key as a deterministic tie-break.
+    ordered = sorted(file_weights.items(), key=lambda item: (-item[1], item[0]))
+    for file_key, weight in ordered:
+        target = min(range(count), key=lambda index: (loads[index], index))
+        buckets[target].add(file_key)
+        loads[target] += weight
+    return tuple(frozenset(bucket) for bucket in buckets)
+
+
+def shard_loads(
+    file_weights: Mapping[str, float], plan: tuple[frozenset[str], ...]
+) -> tuple[float, ...]:
+    """Total weight of each shard in ``plan`` - the balance the packer achieved."""
+    return tuple(
+        sum(file_weights[file_key] for file_key in bucket) for bucket in plan
+    )
+
+
+def relative_file_key(path: str | Path, root: str | Path | None = None) -> str:
+    """Stable per-file key: the posix path relative to ``root`` when possible.
+
+    Falls back to the absolute posix path when ``path`` lies outside ``root`` or
+    the relationship cannot be resolved, which keeps the key defined for every
+    collected item rather than dropping one from the plan.
+    """
+    resolved = Path(path)
+    if root is not None:
+        try:
+            return resolved.resolve().relative_to(Path(root).resolve()).as_posix()
+        except (OSError, ValueError):
+            pass
+    return resolved.as_posix()

tests/conftest.py

@@ -62,6 +62,37 @@ if "src.database" not in sys.modules:
 # collection, which breaks session import in subsequent tests).
 import core.models  # noqa: E402
 
+def pytest_addoption(parser):
+    """Add ``--shard N/M`` so CI can run the suite as parallel sections."""
+    group = parser.getgroup("sharding", "parallel test sharding")
+    group.addoption(
+        "--shard",
+        action="store",
+        default=None,
+        metavar="N/M",
+        help=(
+            "run only shard N of M (1-based), e.g. --shard 1/4. Shards partition "
+            "the suite by test file, so together they run every test exactly "
+            "once. See tests/_shards.py."
+        ),
+    )
+
+
+def _shard_spec(config):
+    """Parse the ``--shard`` option into a ShardSpec, or None when unset."""
+    from tests._shards import ShardSpecError, parse_shard_spec
+
+    value = config.getoption("shard")
+    if value is None:
+        return None
+    try:
+        return parse_shard_spec(value)
+    except ShardSpecError as error:
+        # UsageError fails the run immediately rather than silently running a
+        # subset nobody asked for - a dropped shard is invisible in a green CI.
+        raise pytest.UsageError(str(error)) from error
+
+
 def pytest_configure(config):
     """Register the dynamic taxonomy ``sub_*`` markers before collection.
 
@@ -79,15 +110,24 @@ def pytest_configure(config):
         if marker_name.startswith("sub_"):
             config.addinivalue_line("markers", f"{marker_name}: taxonomy sub-area marker")
 
+    # Validate --shard before collection so a bad selector fails the run up
+    # front instead of after a few minutes of collecting.
+    _shard_spec(config)
+
 
 def pytest_collection_modifyitems(config, items):
-    """Tag each collected test with its taxonomy ``area_*`` and ``sub_*`` markers.
+    """Tag each collected test with its taxonomy markers, then apply ``--shard``.
 
-    Collection-time only: this adds markers and nothing else. It does not skip,
-    reorder, or deselect tests, mutate fixtures or the environment, or import any
+    Tagging is collection-time only: it adds markers and nothing else. It does
+    not skip, reorder, mutate fixtures or the environment, or import any
     production module. See ``tests/_taxonomy.py`` for the classification rules.
+
+    Sharding deselects the test files that belong to another shard. It runs
+    after collection, so every test module is still imported, in the same order,
+    in every shard - the import-time stubbing above behaves identically whether
+    the suite runs whole or as one section of it. Only the deselected tests'
+    call phase is skipped. See ``tests/_shards.py`` for the partition.
     """
-    import pytest
     from tests._taxonomy import markers_for_path
 
     for item in items:
@@ -95,6 +135,30 @@ def pytest_collection_modifyitems(config, items):
         for marker_name in markers_for_path(path):
             item.add_marker(getattr(pytest.mark, marker_name))
 
+    spec = _shard_spec(config)
+    if spec is None or spec.selects_everything:
+        return
+
+    from tests._shards import accumulate_file_weights, plan_shards, relative_file_key
+
+    root = getattr(config, "rootpath", None)
+    keys = [
+        relative_file_key(getattr(item, "path", None) or item.fspath, root)
+        for item in items
+    ]
+    weights = accumulate_file_weights(
+        (key, item.get_closest_marker("slow") is not None)
+        for key, item in zip(keys, items)
+    )
+    selected_files = plan_shards(weights, spec.count)[spec.index - 1]
+
+    selected, deselected = [], []
+    for key, item in zip(keys, items):
+        (selected if key in selected_files else deselected).append(item)
+    if deselected:
+        config.hook.pytest_deselected(items=deselected)
+    items[:] = selected
+
 
 @pytest.fixture(scope="session", autouse=True)
 def _serve_test_static():

tests/test_agent_tmux_retirement.py

@@ -132,3 +132,34 @@ def test_legacy_cleanup_against_a_private_real_tmux_server(tmp_path, monkeypatch
         assert containment.active_grants() == []
     finally:
         subprocess.run([real_tmux, "-S", socket, "kill-server"], capture_output=True)
+
+
+def test_a_descendant_reissued_after_the_table_read_is_never_released(legacy, monkeypatch):
+    reads = {"n": 0}
+
+    def table():
+        reads["n"] += 1
+        rows = {4200: process_ownership.ProcessInfo(4200, 4100, "/bin/bash --noprofile --norc")}
+        if reads["n"] <= 2:
+            rows[4201] = process_ownership.ProcessInfo(4201, 4200, "sleep 60")
+        else:
+            # The child exited and its pid now names an unrelated process.
+            rows[4201] = process_ownership.ProcessInfo(4201, 1, "sshd: stranger")
+        return rows
+
+    monkeypatch.setattr(process_ownership, "process_table", table)
+    process_reaper.reap_legacy_agent_tmux()
+    assert 4201 not in [pid for pid, _ in legacy["released"]]
+
+
+def test_an_unidentifiable_descendant_keeps_the_session_unsignalled(legacy, monkeypatch):
+    def start_token(pid):
+        if pid == 4201:
+            raise process_ownership.InspectionUnavailable("/proc/4201/stat")
+        return f"token:{pid}"
+
+    monkeypatch.setattr(process_ownership, "start_token", start_token)
+    report = process_reaper.reap_legacy_agent_tmux()
+    assert report["unverifiable"] == 1
+    assert legacy["released"] == []
+    assert all(call[1] != "kill-session" for call in legacy["calls"])

tests/test_browser_lifecycle.py

@@ -13,14 +13,18 @@ import pytest
 
 from core import platform_compat
 import src.agent_tools.web_tools as web_tools
-from src import browser_lifecycle
+from src import browser_lifecycle, process_ownership
 from src.agent_tools.web_tools import PrivateBrowserTool
 
 
-def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline: str, state: str = "S") -> None:
+def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline: str,
+               state: str = "S", starttime: int | None = None) -> None:
     entry = root / str(pid)
     entry.mkdir(parents=True)
-    (entry / "stat").write_text(f"{pid} (x y) {state} {ppid} {pgid} {sid} 0 0 0")
+    # A full stat line: fields after the comm up to starttime (field 22), which
+    # is what process identity is read from.
+    tail = " ".join(["0"] * 15 + [str(starttime if starttime is not None else 1000 + pid)])
+    (entry / "stat").write_text(f"{pid} (x y) {state} {ppid} {pgid} {sid} {tail}")
     (entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
 
 
@@ -28,7 +32,12 @@ def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline:
 def fake_procfs(monkeypatch, tmp_path):
     proc = tmp_path / "proc"
     proc.mkdir()
+    boot = proc / "sys/kernel/random/boot_id"
+    boot.parent.mkdir(parents=True)
+    boot.write_text("fake-boot\n")
     monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
+    # Identity reads its own binding of the proc root.
+    monkeypatch.setattr(process_ownership, "PROC_ROOT", proc)
     killed: list[tuple[int, int]] = []
 
     def _kill(pid, sig):
@@ -101,6 +110,69 @@ def test_forced_cleanup_kills_tree_and_removes_owned_resources(fake_procfs, tmp_
     assert (proc / "900").exists()
 
 
+def test_member_recycled_between_membership_and_identity_is_never_signalled(
+    fake_procfs, tmp_path, monkeypatch,
+) -> None:
+    """Membership sees the old renderer; its pid is reused before any capture or signal.
+
+    The replacement is an unrelated process occupying the same pid slot. Its
+    identity must never be the one teardown verifies, so it is never hit.
+    """
+    proc, killed = fake_procfs
+    _browser_tree(proc, tmp_path / "agent-browser-chrome-a")
+    lifecycle = browser_lifecycle.process_lifecycle
+    recycled = []
+
+    def recycle_once():
+        if not recycled:
+            recycled.append(True)
+            shutil.rmtree(proc / "502")
+            _fake_proc(proc, 502, ppid=1, pgid=502, sid=502, cmdline="sshd", starttime=99999)
+
+    # Whichever comes first after membership is decided — an identity capture
+    # or the signalling sweep — the old renderer is gone and its pid reissued.
+    real_capture = lifecycle.ProcessIdentity.capture
+    real_terminate = lifecycle.terminate_identities
+
+    def capture(pid, **kwargs):
+        recycle_once()
+        return real_capture(pid, **kwargs)
+
+    def terminate(identities, **kwargs):
+        identities = list(identities)
+        recycle_once()
+        return real_terminate(identities, **kwargs)
+
+    monkeypatch.setattr(lifecycle.ProcessIdentity, "capture", staticmethod(capture))
+    monkeypatch.setattr(lifecycle, "terminate_identities", terminate)
+
+    killed_pids, survivors, _ = browser_lifecycle.kill_browser_tree(500, settle_s=0.1)
+
+    assert recycled, "the race was never staged"
+    assert 502 not in [pid for pid, _ in killed], "the replacement process was signalled"
+    assert (proc / "502").exists()
+    assert killed_pids == [501, 500] and survivors == []
+
+
+def test_member_without_identity_is_a_survivor_and_keeps_its_profile(fake_procfs, tmp_path) -> None:
+    proc, killed = fake_procfs
+    root = tmp_path / "rt"
+    root.mkdir()
+    (root / "ody-k.pid").write_text("500")
+    profile = tmp_path / "agent-browser-chrome-a"
+    profile.mkdir()
+    _browser_tree(proc, profile)
+    # The renderer's stat is unreadable as identity (no start time): this host
+    # cannot say which process holds the pid, so it must not be signalled.
+    (proc / "502" / "stat").write_text("502 (x y) S 501 501 500")
+
+    receipt = browser_lifecycle.force_cleanup(root, "ody-k")
+
+    assert 502 not in [pid for pid, _ in killed]
+    assert receipt.survivors == [502] and not receipt.verified
+    assert profile.exists() and (root / "ody-k.pid").exists()
+
+
 def test_forced_cleanup_without_procfs_never_kills_unverified_processes(monkeypatch, tmp_path) -> None:
     monkeypatch.setattr(platform_compat, "PROC_ROOT", tmp_path / "missing")
     monkeypatch.setattr(browser_lifecycle.os, "kill", lambda *a: pytest.fail("killed"))

tests/test_cookbook_stop_without_procfs.py

@@ -320,3 +320,69 @@ def test_model_process_scan_returns_empty_without_procfs(monkeypatch, tmp_path):
     monkeypatch.setattr(os, "listdir", _unexpected_listdir)
 
     assert tools._scan_running_model_processes() == []
+
+
+@pytest.mark.asyncio
+async def test_stop_reports_a_survivor_it_can_no_longer_identify(monkeypatch, tmp_path):
+    """Captured as ours, unverifiable at sweep time: not signalled, and said so."""
+    from src import process_ownership
+
+    tracked_cmd = "python -m vllm.entrypoints.openai.api_server --model org/model"
+    state = _tracked_state(cmd=tracked_cmd)
+    posts = _install_httpx_client(monkeypatch, state)
+    _install_successful_tmux_kill(monkeypatch, panes="serve-abc123 900\n")
+    table = _fake_table(monkeypatch, {900: (1, "bash"), 101: (900, tracked_cmd)})
+    asked = {"n": 0}
+
+    def _token(pid):
+        if int(pid) == 101:
+            asked["n"] += 1
+            if asked["n"] > 1:  # the capture succeeded; every later look fails
+                raise process_ownership.InspectionUnavailable("/proc/101/stat")
+            return "token:101"
+        return f"token:{pid}" if int(pid or 0) in table else None
+
+    monkeypatch.setattr(process_ownership, "start_token", _token)
+    signalled = _install_effective_kill(monkeypatch, table)
+
+    result = await tools.do_stop_served_model(json.dumps({"session_id": "serve-abc123"}))
+
+    assert result["exit_code"] == 0
+    assert not any(pid == 101 for pid, _sig in signalled)
+    assert "could not be re-identified and were not signalled (pid 101)" in result["output"]
+    assert _stopped_statuses(posts, "serve-abc123") == ["stopped"]
+
+
+@pytest.mark.asyncio
+async def test_stop_never_signals_a_pid_reissued_between_the_table_and_its_capture(
+    monkeypatch, tmp_path
+):
+    """The table places 101 under the pane; 101 is then reissued to a stranger.
+
+    The stranger's token must never be the one recorded for the session.
+    """
+    from src import process_ownership
+
+    tracked_cmd = "python -m vllm.entrypoints.openai.api_server --model org/model"
+    state = _tracked_state(cmd=tracked_cmd)
+    posts = _install_httpx_client(monkeypatch, state)
+    _install_successful_tmux_kill(monkeypatch, panes="serve-abc123 900\n")
+    table = _fake_table(monkeypatch, {900: (1, "bash"), 101: (900, tracked_cmd)})
+    reads = {"n": 0}
+
+    def _table():
+        reads["n"] += 1
+        if reads["n"] > 1:
+            # After the first read: the server exited and its pid now belongs
+            # to an unrelated process with a fresh identity.
+            table[101] = process_ownership.ProcessInfo(101, 1, "sshd: stranger")
+        return dict(table)
+
+    monkeypatch.setattr(process_ownership, "process_table", _table)
+    signalled = _install_effective_kill(monkeypatch, table)
+
+    result = await tools.do_stop_served_model(json.dumps({"session_id": "serve-abc123"}))
+
+    assert result["exit_code"] == 0
+    assert not any(pid == 101 for pid, _sig in signalled)
+    assert _stopped_statuses(posts, "serve-abc123") == ["stopped"]

tests/test_private_browser_tool.py

@@ -2,6 +2,7 @@ import asyncio
 import pytest
 import base64
 import json
+import shutil
 from pathlib import Path
 
 from core import platform_compat
@@ -1436,27 +1437,77 @@ def test_private_browser_screenshot_without_path_returns_image_payload(monkeypat
     }]
 
 
-def test_private_browser_timeout_terminates_the_process_group(monkeypatch) -> None:
-    calls = []
-
+def _cli_proc(calls, identity=None):
     class _Proc:
         pid = 1234
+        returncode = None
+        _ody_identity = identity
 
         def kill(self):
             calls.append("fallback-kill")
 
+    return _Proc()
+
+
+def test_private_browser_timeout_terminates_the_process_group(monkeypatch) -> None:
+    from src import process_lifecycle, process_ownership
+
+    calls = []
     monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
+    monkeypatch.setattr(process_ownership, "verify", lambda pid, token: process_ownership.OWNED)
     monkeypatch.setattr(
         web_tools.os,
         "killpg",
         lambda pgid, signum: calls.append((pgid, signum)),
     )
 
-    PrivateBrowserTool._terminate_subprocess(_Proc())
+    PrivateBrowserTool._terminate_subprocess(
+        _cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
 
     assert calls == [(1234, web_tools.signal.SIGKILL), "fallback-kill"]
 
 
+@pytest.mark.parametrize("verdict", ["foreign", "unverifiable", "gone"])
+def test_cli_group_is_not_signalled_once_its_identity_is_lost(monkeypatch, verdict) -> None:
+    """A reaped CLI's pid may be reissued; its group is then not ours to kill."""
+    from src import process_lifecycle, process_ownership
+
+    calls = []
+    monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
+    monkeypatch.setattr(process_ownership, "verify", lambda pid, token: verdict)
+    monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled an unowned group"))
+
+    PrivateBrowserTool._terminate_subprocess(
+        _cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
+
+    assert calls == ["fallback-kill"]
+
+
+def test_cli_without_a_spawn_identity_is_never_group_signalled(monkeypatch) -> None:
+    calls = []
+    monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
+    monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled a bare pid's group"))
+
+    PrivateBrowserTool._terminate_subprocess(_cli_proc(calls))
+
+    assert calls == ["fallback-kill"]
+
+
+def test_cli_group_that_moved_is_not_signalled(monkeypatch) -> None:
+    """The identity verifies but no longer leads the recorded group."""
+    from src import process_lifecycle, process_ownership
+
+    calls = []
+    monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: 999)
+    monkeypatch.setattr(process_ownership, "verify", lambda pid, token: process_ownership.OWNED)
+    monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled a group it does not lead"))
+
+    PrivateBrowserTool._terminate_subprocess(
+        _cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
+
+    assert calls == ["fallback-kill"]
+
+
 def test_private_browser_retries_one_timed_out_local_open(monkeypatch, tmp_path) -> None:
     page = tmp_path / "output.html"
     page.write_text("<html><title>retry</title></html>")
@@ -1900,29 +1951,142 @@ def test_terminate_owned_chrome_kills_only_this_runtimes_profile(
 ) -> None:
     """With procfs present, match on the runtime-owned profile prefix alone."""
 
-    proc = tmp_path / "proc"
+    proc = _fake_procfs(monkeypatch, tmp_path)
     tmpdir = tmp_path / "runtime-tmp"
     tmpdir.mkdir()
     profile_prefix = str(tmpdir.resolve() / "agent-browser-chrome-")
 
-    def _write_pid(pid: str, cmdline: str) -> None:
-        entry = proc / pid
-        entry.mkdir(parents=True)
-        (entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
-
-    _write_pid("101", f"chrome --user-data-dir={profile_prefix}abc")
-    _write_pid("202", "chrome --user-data-dir=/Users/someone/Library/Chrome")
+    _fake_process(proc, 101, f"chrome --user-data-dir={profile_prefix}abc")
+    _fake_process(proc, 202, "chrome --user-data-dir=/Users/someone/Library/Chrome")
     (proc / "self").mkdir()
 
-    monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
-    killed: list[int] = []
-    monkeypatch.setattr(web_tools.os, "kill", lambda pid, sig: killed.append(pid))
+    killed = _install_lethal_kill(monkeypatch, proc)
 
     PrivateBrowserTool._terminate_owned_chrome({"TMPDIR": str(tmpdir)})
 
     assert killed == [101]
 
 
+def _fake_procfs(monkeypatch, tmp_path):
+    from src import process_ownership
+
+    proc = tmp_path / "proc"
+    boot = proc / "sys/kernel/random/boot_id"
+    boot.parent.mkdir(parents=True)
+    boot.write_text("fake-boot\n")
+    monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
+    monkeypatch.setattr(process_ownership, "PROC_ROOT", proc)
+    return proc
+
+
+def _fake_process(proc, pid: int, cmdline: str, *, starttime: int | None = None) -> None:
+    entry = proc / str(pid)
+    entry.mkdir(parents=True)
+    tail = " ".join(["0"] * 15 + [str(starttime if starttime is not None else 1000 + pid)])
+    (entry / "stat").write_text(f"{pid} (x) S 1 {pid} {pid} {tail}")
+    (entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
+
+
+def _install_lethal_kill(monkeypatch, proc, *, before_kill=None):
+    killed: list[int] = []
+
+    def _kill(pid, sig):
+        if before_kill is not None:
+            before_kill(pid)
+        entry = proc / str(pid)
+        if not entry.exists():
+            raise ProcessLookupError(pid)
+        killed.append(pid)
+        shutil.rmtree(entry)
+
+    monkeypatch.setattr(web_tools.os, "kill", _kill)
+    return killed
+
+
+def test_owned_chrome_sweep_never_signals_a_reused_pid(monkeypatch, tmp_path) -> None:
+    """Matched by profile, then reissued to a stranger before the signal."""
+    from src import process_lifecycle
+
+    proc = _fake_procfs(monkeypatch, tmp_path)
+    tmpdir = tmp_path / "runtime-tmp"
+    tmpdir.mkdir()
+    _fake_process(proc, 101, f"chrome --user-data-dir={tmpdir.resolve()}/agent-browser-chrome-x")
+    killed = _install_lethal_kill(monkeypatch, proc)
+    real_terminate = process_lifecycle.terminate_identities
+
+    def recycle_then_terminate(identities, **kwargs):
+        identities = list(identities)
+        shutil.rmtree(proc / "101")
+        _fake_process(proc, 101, "postgres", starttime=99999)
+        return real_terminate(identities, **kwargs)
+
+    monkeypatch.setattr(process_lifecycle, "terminate_identities", recycle_then_terminate)
+
+    PrivateBrowserTool._terminate_owned_chrome({"TMPDIR": str(tmpdir)})
+
+    assert killed == [] and (proc / "101").exists()
+
+
+def _legacy_pid_file_for(tmp_path, monkeypatch, namespace, session, pid):
+    """A pid file in the legacy namespace layout, which only the fallback loop reads."""
+    monkeypatch.setenv("XDG_RUNTIME_DIR", str(tmp_path))
+    monkeypatch.setenv("ODYSSEUS_BROWSER_NAMESPACE", namespace)
+    legacy_run = (tmp_path / "agent-browser" / "namespaces"
+                  / web_tools._bounded_browser_identity(namespace) / "run")
+    legacy_run.mkdir(parents=True, exist_ok=True)
+    target = legacy_run / f"ody-{web_tools._bounded_browser_identity(session)}.pid"
+    assert target in web_tools._browser_pid_file_candidates(tmp_path, namespace, session)
+    target.write_text(str(pid))
+    return target
+
+
+def test_legacy_daemon_pid_file_kills_only_the_verified_daemon(monkeypatch, tmp_path) -> None:
+    proc = _fake_procfs(monkeypatch, tmp_path)
+    _fake_process(proc, 4401, "node agent-browser --serve")
+    killed = _install_lethal_kill(monkeypatch, proc)
+    pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-7", 4401)
+
+    PrivateBrowserTool._terminate_owned_daemon({}, "session-7")
+
+    assert killed == [4401] and not pid_file.exists()
+
+
+def test_legacy_daemon_pid_reused_before_the_signal_is_spared(monkeypatch, tmp_path) -> None:
+    """The pid file still names the slot; the daemon in it was replaced."""
+    from src import process_lifecycle
+
+    proc = _fake_procfs(monkeypatch, tmp_path)
+    _fake_process(proc, 4402, "node agent-browser --serve")
+    killed = _install_lethal_kill(monkeypatch, proc)
+    pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-8", 4402)
+    real_terminate = process_lifecycle.terminate_identities
+
+    def recycle_then_terminate(identities, **kwargs):
+        identities = list(identities)
+        shutil.rmtree(proc / "4402")
+        _fake_process(proc, 4402, "node agent-browser --serve", starttime=99999)
+        return real_terminate(identities, **kwargs)
+
+    monkeypatch.setattr(process_lifecycle, "terminate_identities", recycle_then_terminate)
+
+    PrivateBrowserTool._terminate_owned_daemon({}, "session-8")
+
+    # Even a lookalike command line is not the process the match was made on.
+    assert killed == [] and (proc / "4402").exists()
+
+
+def test_legacy_daemon_without_identity_keeps_its_pid_file(monkeypatch, tmp_path) -> None:
+    proc = _fake_procfs(monkeypatch, tmp_path)
+    _fake_process(proc, 4403, "node agent-browser --serve")
+    (proc / "4403" / "stat").write_text("4403 (x) S 1")  # no start time: unidentifiable
+    monkeypatch.setattr(web_tools.os, "kill", lambda *a: pytest.fail("signalled an unidentified pid"))
+    pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-9", 4403)
+
+    PrivateBrowserTool._terminate_owned_daemon({}, "session-9")
+
+    assert pid_file.exists()
+
+
 def _pid_file_for(tmp_path, monkeypatch, namespace, session, pid):
     """Write a pid file where the daemon helpers will look for it."""
     monkeypatch.setenv("XDG_RUNTIME_DIR", str(tmp_path))

tests/test_process_lifecycle.py

@@ -0,0 +1,365 @@
+"""The generic process lifecycle: identity, probes, escalation, verified death.
+
+These pin the rules every consumer — containment, the PTY shell, the Cookbook
+sweep, the browser lifecycle and kill_process_tree — inherits from one place.
+"""
+
+import asyncio
+import errno
+import os
+import signal
+import subprocess
+import sys
+import time
+
+import pytest
+
+from core import platform_compat
+from src import process_lifecycle, process_ownership
+
+posix_only = pytest.mark.skipif(os.name == "nt", reason="POSIX process groups and signals")
+
+_IGNORE_TERM = (
+    "import signal, sys, time\n"
+    "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n"
+    "print('ready', flush=True)\n"
+    "time.sleep(60)\n"
+)
+
+
+def _spawn(code: str = _IGNORE_TERM) -> subprocess.Popen:
+    proc = subprocess.Popen([sys.executable, "-c", code], stdout=subprocess.PIPE,
+                            stderr=subprocess.DEVNULL, start_new_session=True)
+    assert proc.stdout.readline().strip() == b"ready"
+    return proc
+
+
+def _cleanup(proc: subprocess.Popen) -> None:
+    if proc.poll() is None:
+        proc.kill()
+    proc.wait(timeout=5)
+    proc.stdout.close()
+
+
+# ── Groups ──────────────────────────────────────────────────────────────────
+@posix_only
+def test_our_own_group_is_never_present_and_never_signalled(monkeypatch):
+    own = os.getpgid(0)
+    sent = []
+    monkeypatch.setattr(os, "killpg", lambda pgid, sig: sent.append(("group", pgid, sig)))
+    monkeypatch.setattr(os, "kill", lambda pid, sig: sent.append(("pid", pid, sig)))
+
+    assert process_lifecycle.group_present(own) is False
+    assert process_lifecycle.signal_group(4242, own, signal.SIGTERM) is True
+    assert sent == [("pid", 4242, signal.SIGTERM)]
+
+
+@posix_only
+def test_a_refused_group_probe_is_a_live_group(monkeypatch):
+    def refuse(*_args):
+        raise PermissionError(errno.EPERM, "Operation not permitted")
+
+    monkeypatch.setattr(os, "killpg", refuse)
+    assert process_lifecycle.group_present(987654, own=1) is True
+
+    def gone(*_args):
+        raise ProcessLookupError(errno.ESRCH, "No such process")
+
+    monkeypatch.setattr(os, "killpg", gone)
+    assert process_lifecycle.group_present(987654, own=1) is False
+
+
+@posix_only
+def test_signal_group_reports_when_nothing_was_left_to_signal(monkeypatch):
+    def gone(*_args):
+        raise ProcessLookupError(errno.ESRCH, "No such process")
+
+    monkeypatch.setattr(os, "killpg", gone)
+    monkeypatch.setattr(os, "kill", gone)
+    assert process_lifecycle.signal_group(4242, 4242, signal.SIGTERM, own=1) is False
+
+
+# ── Escalation ──────────────────────────────────────────────────────────────
+def _ladder():
+    return ((signal.SIGTERM, 0.01), (getattr(signal, "SIGKILL", signal.SIGTERM), 0.01))
+
+
+def test_escalate_does_not_signal_a_target_already_gone():
+    sent = []
+    result = process_lifecycle.escalate(lambda: True, sent.append, steps=_ladder())
+    assert result.dead is True and result.escalated is False and sent == []
+
+
+def test_escalate_terms_then_kills_and_reports_the_observed_outcome():
+    sent = []
+    result = process_lifecycle.escalate(lambda: False, sent.append, steps=_ladder(), poll_s=0.001)
+    assert sent == [step[0] for step in _ladder()]
+    assert result.dead is False and result.escalated is True
+
+
+def test_escalate_stops_when_term_is_enough():
+    sent = []
+    result = process_lifecycle.escalate(lambda: bool(sent), sent.append, steps=_ladder())
+    assert sent == [signal.SIGTERM]
+    assert result.dead is True and result.escalated is False
+
+
+def test_escalate_reverifies_before_kill_and_refuses_on_a_lost_identity():
+    sent = []
+    result = process_lifecycle.escalate(
+        lambda: False, sent.append, steps=_ladder(), poll_s=0.001,
+        before_step=lambda sig: "foreign",
+    )
+    assert sent == [signal.SIGTERM]
+    assert result.refusal == "foreign" and result.dead is False
+
+
+def test_escalate_stops_the_ladder_when_nothing_is_left_to_signal():
+    sent = []
+
+    def send(sig):
+        sent.append(sig)
+        return False
+
+    result = process_lifecycle.escalate(lambda: False, send, steps=_ladder())
+    assert sent == [signal.SIGTERM] and result.dead is False
+
+
+async def test_escalate_async_signals_first_and_reaps_inside_the_window():
+    sent, waited = [], []
+    state = {"gone": False}
+
+    async def wait():
+        waited.append(True)
+        state["gone"] = True
+
+    result = await process_lifecycle.escalate_async(
+        lambda: state["gone"], sent.append, steps=_ladder(), wait=wait,
+    )
+    # No precheck: an awaited leader is reaped only after the first signal.
+    assert sent == [signal.SIGTERM] and waited == [True]
+    assert result.dead is True and result.escalated is False
+
+
+async def test_escalate_async_gives_the_reap_a_floor_even_at_zero_grace():
+    seen = {}
+
+    async def wait():
+        seen["ran"] = True
+
+    await process_lifecycle.escalate_async(
+        lambda: False, lambda sig: None, steps=((signal.SIGTERM, 0.0),), wait=wait,
+    )
+    assert seen == {"ran": True}
+
+
+# ── Identity ────────────────────────────────────────────────────────────────
+def test_identity_from_record_needs_a_pid():
+    assert process_lifecycle.ProcessIdentity.from_record({}) is None
+    assert process_lifecycle.ProcessIdentity.from_record({"pid": "x"}) is None
+    ident = process_lifecycle.ProcessIdentity.from_record(
+        {"supervisor_pid": 42, "supervisor_token": "t", "pgid": "7"},
+        pid_key="supervisor_pid", token_key="supervisor_token",
+    )
+    assert ident == process_lifecycle.ProcessIdentity(pid=42, start_token="t", pgid=7)
+
+
+def test_a_pid_without_a_token_is_never_owned_and_never_exited(monkeypatch):
+    monkeypatch.setattr(process_ownership, "start_token", lambda pid: "live")
+    ident = process_lifecycle.ProcessIdentity(pid=4242, start_token=None)
+    assert ident.verdict() == process_ownership.UNVERIFIABLE
+    assert ident.owned() is False and ident.exited() is False
+
+
+@pytest.mark.parametrize("current,exited", [(None, True), ("other", True), ("mine", False)])
+def test_exited_means_gone_or_recycled(monkeypatch, current, exited):
+    monkeypatch.setattr(process_ownership, "start_token", lambda pid: current)
+    monkeypatch.setattr(process_lifecycle, "is_zombie", lambda pid: False)
+    ident = process_lifecycle.ProcessIdentity(pid=4242, start_token="mine")
+    assert ident.exited() is exited
+
+
+@pytest.mark.skipif(not platform_compat.has_procfs(), reason="zombie state needs procfs")
+def test_an_unreaped_zombie_has_exited():
+    proc = subprocess.Popen([sys.executable, "-c", "pass"])
+    ident = process_lifecycle.ProcessIdentity.capture(proc.pid)
+    try:
+        for _ in range(250):
+            if process_lifecycle.is_zombie(proc.pid):
+                break
+            time.sleep(0.02)
+        assert ident.verdict() == process_ownership.OWNED  # still in its slot
+        assert ident.exited() is True
+    finally:
+        proc.wait(timeout=5)
+
+
+# ── Snapshot identities ─────────────────────────────────────────────────────
+def test_terminate_identities_never_signals_foreign_or_unverifiable(monkeypatch):
+    tokens = {1: "now-someone-else", 2: None}
+
+    def start_token(pid):
+        if pid == 3:
+            raise process_ownership.InspectionUnavailable("stat")
+        return tokens.get(pid)
+
+    monkeypatch.setattr(process_ownership, "start_token", start_token)
+    monkeypatch.setattr(os, "kill", lambda *a: pytest.fail("signalled an unowned pid"))
+    sweep = process_lifecycle.terminate_identities(
+        [process_lifecycle.ProcessIdentity(1, "mine"),
+         process_lifecycle.ProcessIdentity(2, "mine"),
+         process_lifecycle.ProcessIdentity(3, "mine")],
+        steps=_ladder(),
+    )
+    assert sweep.killed == () and sweep.survivors == ()
+    assert sweep.unverified == (3,) and sweep.dead is False
+
+
+def test_terminate_identities_kills_in_order_and_reverifies_each_signal(monkeypatch):
+    alive = {10: "a", 11: "b", 12: "c"}
+    sent = []
+    monkeypatch.setattr(process_ownership, "start_token", lambda pid: alive.get(pid))
+    monkeypatch.setattr(process_lifecycle, "is_zombie", lambda pid: False)
+
+    def kill(pid, sig):
+        sent.append((pid, sig))
+        alive.pop(pid, None)
+
+    monkeypatch.setattr(os, "kill", kill)
+    sweep = process_lifecycle.terminate_identities(
+        [process_lifecycle.ProcessIdentity(pid, token) for pid, token in list(alive.items())],
+        steps=((signal.SIGTERM, 0.05),),
+    )
+    assert [pid for pid, _ in sent] == [10, 11, 12]
+    assert sweep.killed == (10, 11, 12) and sweep.dead
+
+
+@posix_only
+@pytest.mark.skipif(not platform_compat.has_procfs(), reason="real identity needs procfs")
+def test_terminate_identities_escalates_past_an_ignored_sigterm():
+    proc = _spawn()
+    try:
+        ident = process_lifecycle.ProcessIdentity.capture(proc.pid)
+        sweep = process_lifecycle.terminate_identities(
+            [ident], steps=process_lifecycle.term_kill_steps(0.2))
+        assert sweep.killed == (proc.pid,) and sweep.survivors == ()
+        assert proc.wait(timeout=5) == -signal.SIGKILL
+    finally:
+        _cleanup(proc)
+
+
+# ── terminate_tree / kill_process_tree ──────────────────────────────────────
+@posix_only
+@pytest.mark.skipif(not platform_compat.has_procfs(), reason="real identity needs procfs")
+def test_terminate_tree_requires_a_matching_identity():
+    proc = _spawn()
+    try:
+        refused = process_lifecycle.terminate_tree(
+            proc.pid, pgid=proc.pid, start_token="procfs:not-this-process",
+            require_identity=True, grace_s=0.1)
+        assert refused.dead is False and refused.ownership == process_ownership.FOREIGN
+        assert refused.survivors == ()
+        assert proc.poll() is None
+
+        token = process_ownership.start_token(proc.pid)
+        outcome = process_lifecycle.terminate_tree(
+            proc.pid, pgid=proc.pid, start_token=token, require_identity=True, grace_s=0.2)
+        assert outcome.dead is True and outcome.escalated is True
+    finally:
+        _cleanup(proc)
+
+
+@posix_only
+def test_terminate_tree_keeps_a_leaderless_group_visible(monkeypatch):
+    monkeypatch.setattr(process_ownership, "start_token", lambda pid: None)
+    monkeypatch.setattr(process_lifecycle, "group_present", lambda pgid, **kw: True)
+    monkeypatch.setattr(process_lifecycle, "signal_group",
+                        lambda *a, **kw: pytest.fail("signalled an unproven group"))
+    outcome = process_lifecycle.terminate_tree(4242, pgid=4242, start_token="t",
+                                               require_identity=True)
+    assert outcome.dead is False and outcome.survivors == (4242,)
+    assert outcome.ownership == process_ownership.GONE
+
+
+@posix_only
+def test_kill_process_tree_writes_no_containment_record(monkeypatch):
+    from src import containment
+
+    monkeypatch.setattr(containment, "_update_record", lambda *a, **k: pytest.fail("grant store touched"))
+    proc = _spawn("import time\nprint('ready', flush=True)\ntime.sleep(60)\n")
+    try:
+        outcome = platform_compat.kill_process_tree(proc.pid)
+        assert outcome.dead is True
+        assert isinstance(outcome, containment.ReleaseOutcome)
+    finally:
+        _cleanup(proc)
+
+
+def test_termination_outcome_shape_is_the_teardown_block():
+    outcome = process_lifecycle.TerminationOutcome(dead=False, escalated=True, survivors=(1,),
+                                                   mechanism="process_group", ownership="owned")
+    assert outcome.to_dict() == {"dead": False, "escalated": True, "survivors": [1],
+                                 "mechanism": "process_group", "ownership": "owned"}
+
+
+# ── Identity-bound observation ──────────────────────────────────────────────
+def _tokens(monkeypatch, sequence):
+    calls = iter(sequence)
+
+    def start_token(pid):
+        value = next(calls)
+        if isinstance(value, Exception):
+            raise value
+        return value
+
+    monkeypatch.setattr(process_ownership, "start_token", start_token)
+
+
+def test_observe_binds_facts_to_the_token_read_around_them(monkeypatch):
+    _tokens(monkeypatch, ["t1", "t1"])
+    seen = process_lifecycle.observe(42, lambda pid: "facts")
+    assert seen.identity == process_lifecycle.ProcessIdentity(42, "t1") and seen.facts == "facts"
+
+
+def test_observe_drops_facts_read_across_a_pid_reuse(monkeypatch):
+    _tokens(monkeypatch, ["old", "new"])
+    assert process_lifecycle.observe(42, lambda pid: "whose facts?") is None
+
+
+def test_observe_of_a_gone_pid_reads_nothing(monkeypatch):
+    _tokens(monkeypatch, [None])
+    assert process_lifecycle.observe(42, lambda pid: pytest.fail("read a gone pid")) is None
+
+
+@pytest.mark.parametrize("sequence", [
+    [process_ownership.InspectionUnavailable("stat")],
+    ["t1", process_ownership.InspectionUnavailable("stat")],
+])
+def test_observe_keeps_unidentifiable_facts_without_a_token(monkeypatch, sequence):
+    _tokens(monkeypatch, sequence)
+    seen = process_lifecycle.observe(42, lambda pid: "facts")
+    assert seen.identity.start_token is None and seen.facts == "facts"
+    assert seen.identity.verdict() == process_ownership.UNVERIFIABLE
+
+
+def test_bind_descendants_drops_a_pid_reparented_between_snapshots(monkeypatch):
+    Info = process_ownership.ProcessInfo
+    tables = iter([
+        {10: Info(10, 1, "shell"), 11: Info(11, 10, "server")},
+        {10: Info(10, 1, "shell"), 11: Info(11, 1, "stranger")},
+    ])
+    monkeypatch.setattr(process_ownership, "process_table", lambda: next(tables))
+    monkeypatch.setattr(process_ownership, "start_token", lambda pid: f"t{pid}")
+    bound = process_lifecycle.bind_descendants([10])
+    assert [seen.identity.pid for seen in bound] == [10]
+
+
+def test_bind_descendants_drops_a_pid_whose_token_changed(monkeypatch):
+    Info = process_ownership.ProcessInfo
+    table = {10: Info(10, 1, "shell"), 11: Info(11, 10, "server")}
+    monkeypatch.setattr(process_ownership, "process_table", lambda: dict(table))
+    monkeypatch.setattr(process_ownership, "start_token", lambda pid: f"t{pid}")
+    monkeypatch.setattr(process_ownership, "verify",
+                        lambda pid, token: process_ownership.FOREIGN if pid == 11 else process_ownership.OWNED)
+    bound = process_lifecycle.bind_descendants([10], exclude={99})
+    assert [(seen.identity.pid, seen.facts.command) for seen in bound] == [(10, "shell")]

tests/test_runtime_behavior_regressions.py

@@ -243,6 +243,9 @@ def test_native_terminal_runtime_adds_offered_tools_deliberately():
 # belonging to the user, or to another worktree, must survive it.
 
 def test_chrome_sweep_kills_only_this_runtimes_profile(monkeypatch, tmp_path):
+    import shutil
+
+    from src import process_ownership
     from src.agent_tools.web_tools import PrivateBrowserTool
 
     proc = tmp_path / "proc"
@@ -250,9 +253,19 @@ def test_chrome_sweep_kills_only_this_runtimes_profile(monkeypatch, tmp_path):
     tmpdir.mkdir()
     ours = str(tmpdir.resolve() / "agent-browser-chrome-")
 
+    # A fake process needs an identity, not only a command line: the sweep
+    # signals a process only after verifying the start token it matched on.
+    # Without a stat here the token would be read from the host's real /proc,
+    # so the outcome would depend on whether this pid happens to exist.
+    boot = proc / "sys/kernel/random/boot_id"
+    boot.parent.mkdir(parents=True)
+    boot.write_text("fake-boot\n")
+
     def _pid(pid, cmdline):
         entry = proc / pid
         entry.mkdir(parents=True)
+        starttime = " ".join(["0"] * 15 + [str(1000 + int(pid))])
+        (entry / "stat").write_text(f"{pid} (chrome) S 1 {pid} {pid} {starttime}")
         (entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
 
     _pid("101", f"chrome --user-data-dir={ours}session-a")
@@ -261,8 +274,14 @@ def test_chrome_sweep_kills_only_this_runtimes_profile(monkeypatch, tmp_path):
     (proc / "self").mkdir()
 
     monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
+    monkeypatch.setattr(process_ownership, "PROC_ROOT", proc)
     killed = []
-    monkeypatch.setattr(al_web.os, "kill", lambda pid, sig: killed.append(pid))
+
+    def _kill(pid, sig):
+        killed.append(pid)
+        shutil.rmtree(proc / str(pid), ignore_errors=True)
+
+    monkeypatch.setattr(al_web.os, "kill", _kill)
 
     PrivateBrowserTool._terminate_owned_chrome({"TMPDIR": str(tmpdir)})
 

tests/test_shards.py

@@ -0,0 +1,214 @@
+"""Unit tests for tests/_shards.py - the parallel shard planner.
+
+These pin the partition guarantees directly, without running pytest collection:
+every test file lands in exactly one shard, the plan is identical in every
+parallel job, and a bad ``--shard`` value is rejected rather than quietly
+running a subset. They import only the module under test (a test-support
+module, not production code) and touch no filesystem.
+"""
+from pathlib import Path
+
+import pytest
+
+from tests._shards import (
+    DEFAULT_ITEM_WEIGHT,
+    DEFAULT_SHARD_COUNT,
+    SLOW_ITEM_WEIGHT,
+    ShardSpec,
+    ShardSpecError,
+    accumulate_file_weights,
+    item_weight,
+    parse_shard_spec,
+    plan_shards,
+    relative_file_key,
+    shard_loads,
+)
+
+
+def even_weights(count, weight=1.0):
+    """``count`` file keys of equal weight, named so sort order is stable."""
+    return {f"tests/test_{index:03d}.py": weight for index in range(count)}
+
+
+# --- parse_shard_spec --------------------------------------------------------
+
+def test_parse_accepts_a_simple_selector():
+    assert parse_shard_spec("2/4") == ShardSpec(index=2, count=4)
+
+
+def test_parse_tolerates_surrounding_whitespace_from_a_shell_variable():
+    assert parse_shard_spec("  3/4\n") == ShardSpec(index=3, count=4)
+
+
+@pytest.mark.parametrize("value", ["", "abc", "1", "1/", "/4", "1/4/4", "1-4", "1 / 4"])
+def test_parse_rejects_malformed_selectors(value):
+    with pytest.raises(ShardSpecError):
+        parse_shard_spec(value)
+
+
+@pytest.mark.parametrize("value", ["0/4", "5/4", "-1/4", "1/0"])
+def test_parse_rejects_out_of_range_selectors(value):
+    with pytest.raises(ShardSpecError):
+        parse_shard_spec(value)
+
+
+def test_parse_error_names_the_offending_value():
+    with pytest.raises(ShardSpecError, match="9/4"):
+        parse_shard_spec("9/4")
+
+
+def test_single_shard_selects_everything_and_larger_counts_do_not():
+    assert parse_shard_spec("1/1").selects_everything is True
+    assert parse_shard_spec("1/2").selects_everything is False
+
+
+def test_spec_renders_as_the_selector_it_came_from():
+    assert str(parse_shard_spec("3/4")) == "3/4"
+
+
+# --- weights -----------------------------------------------------------------
+
+def test_a_slow_item_outweighs_an_ordinary_one():
+    assert item_weight(is_slow=True) == SLOW_ITEM_WEIGHT
+    assert item_weight(is_slow=False) == DEFAULT_ITEM_WEIGHT
+    assert SLOW_ITEM_WEIGHT > DEFAULT_ITEM_WEIGHT
+
+
+def test_file_weight_sums_the_items_in_that_file():
+    weights = accumulate_file_weights([
+        ("tests/test_a.py", False),
+        ("tests/test_a.py", False),
+        ("tests/test_b.py", True),
+    ])
+    assert weights == {
+        "tests/test_a.py": 2 * DEFAULT_ITEM_WEIGHT,
+        "tests/test_b.py": SLOW_ITEM_WEIGHT,
+    }
+
+
+def test_file_weights_of_an_empty_collection_are_empty():
+    assert accumulate_file_weights([]) == {}
+
+
+# --- plan_shards: the partition guarantees -----------------------------------
+
+@pytest.mark.parametrize("count", [1, 2, 3, 4, 5, 8])
+def test_every_file_lands_in_exactly_one_shard(count):
+    weights = even_weights(37)
+    plan = plan_shards(weights, count)
+
+    assert len(plan) == count
+    placements = [key for bucket in plan for key in bucket]
+    assert sorted(placements) == sorted(weights)
+    assert len(placements) == len(set(placements))
+
+
+def test_a_single_shard_holds_the_whole_suite():
+    weights = even_weights(10)
+    assert plan_shards(weights, 1) == (frozenset(weights),)
+
+
+def test_the_plan_is_identical_for_the_same_input():
+    weights = even_weights(50)
+    assert plan_shards(weights, 4) == plan_shards(weights, 4)
+
+
+def test_the_plan_does_not_depend_on_file_insertion_order():
+    keys = list(even_weights(20))
+    forward = plan_shards({key: 1.0 for key in keys}, 4)
+    reversed_order = plan_shards({key: 1.0 for key in reversed(keys)}, 4)
+    assert forward == reversed_order
+
+
+def test_equal_weights_are_spread_evenly():
+    weights = even_weights(40)
+    plan = plan_shards(weights, 4)
+    assert [len(bucket) for bucket in plan] == [10, 10, 10, 10]
+
+
+def test_shards_may_be_empty_when_files_are_scarcer_than_shards():
+    plan = plan_shards(even_weights(2), 4)
+    assert sorted(len(bucket) for bucket in plan) == [0, 0, 1, 1]
+
+
+def test_an_empty_suite_still_yields_the_requested_number_of_shards():
+    assert plan_shards({}, 3) == (frozenset(), frozenset(), frozenset())
+
+
+@pytest.mark.parametrize("count", [0, -1])
+def test_plan_rejects_a_nonsensical_shard_count(count):
+    with pytest.raises(ShardSpecError):
+        plan_shards(even_weights(4), count)
+
+
+# --- plan_shards: balance ----------------------------------------------------
+
+def test_a_heavy_file_is_offset_by_giving_its_shard_fewer_others():
+    # The real shape of the suite: one file of `slow` tests worth about a
+    # quarter of the total, and a long tail of ordinary files.
+    weights = {"tests/test_heavy.py": 100.0, **even_weights(300)}
+    plan = plan_shards(weights, 4)
+
+    assert shard_loads(weights, plan) == (100.0, 100.0, 100.0, 100.0)
+    heavy_shard = next(i for i, b in enumerate(plan) if "tests/test_heavy.py" in b)
+    assert len(plan[heavy_shard]) == 1
+
+
+def test_a_file_heavier_than_an_even_share_sets_the_floor_alone():
+    # A shard cannot be lighter than its heaviest file, so the packer stops
+    # adding to that shard rather than balancing the others against it.
+    weights = {"tests/test_heavy.py": 300.0, **even_weights(300)}
+    plan = plan_shards(weights, 4)
+    loads = shard_loads(weights, plan)
+
+    assert max(loads) == 300.0
+    heavy_shard = next(i for i, b in enumerate(plan) if "tests/test_heavy.py" in b)
+    assert plan[heavy_shard] == frozenset({"tests/test_heavy.py"})
+    others = [load for i, load in enumerate(loads) if i != heavy_shard]
+    assert max(others) - min(others) <= 1.0
+
+
+def test_the_heaviest_files_are_placed_in_different_shards():
+    weights = {f"tests/test_slow_{index}.py": 500.0 for index in range(4)}
+    weights.update(even_weights(100))
+    plan = plan_shards(weights, 4)
+
+    for index in range(4):
+        holders = [bucket for bucket in plan if f"tests/test_slow_{index}.py" in bucket]
+        assert len(holders) == 1
+    assert all(
+        sum(1 for key in bucket if key.startswith("tests/test_slow_")) == 1
+        for bucket in plan
+    )
+
+
+def test_shard_loads_account_for_every_file():
+    weights = {"tests/test_a.py": 2.0, "tests/test_b.py": 3.0, "tests/test_c.py": 5.0}
+    assert sum(shard_loads(weights, plan_shards(weights, 2))) == 10.0
+
+
+# --- relative_file_key -------------------------------------------------------
+
+def test_key_is_relative_to_the_repository_root():
+    assert relative_file_key("/repo/tests/test_a.py", "/repo") == "tests/test_a.py"
+
+
+def test_key_falls_back_to_the_full_path_outside_the_root():
+    assert relative_file_key("/elsewhere/test_a.py", "/repo") == "/elsewhere/test_a.py"
+
+
+def test_key_without_a_root_is_the_path_as_given():
+    assert relative_file_key("tests/test_a.py") == "tests/test_a.py"
+
+
+# --- the default the CI matrix is written against ----------------------------
+
+def test_default_shard_count_matches_the_ci_matrix():
+    """A matrix that drifts from the default silently stops running a shard."""
+    workflow = Path(__file__).resolve().parents[1] / ".github" / "workflows" / "ci.yml"
+    text = workflow.read_text(encoding="utf-8")
+    for index in range(1, DEFAULT_SHARD_COUNT + 1):
+        assert f'"{index}/{DEFAULT_SHARD_COUNT}"' in text, (
+            f"ci.yml does not run shard {index}/{DEFAULT_SHARD_COUNT}"
+        )
+    assert f'"{DEFAULT_SHARD_COUNT + 1}/' not in text

tests/test_shell_routes.py

@@ -119,13 +119,31 @@ pty_session = pytest.mark.skipif(
 
 
 async def _spawn_pty_style_session(script: str):
-    """Spawn `script` the way _generate_pty does: its own session via setsid."""
-    return await asyncio.create_subprocess_shell(
+    """Spawn `script` the way _generate_pty does: its own session via setsid,
+    with the leader's identity and group bound at spawn."""
+    import routes.shell_routes as shell_routes
+
+    proc = await asyncio.create_subprocess_shell(
         script,
         stdout=asyncio.subprocess.DEVNULL,
         stderr=asyncio.subprocess.DEVNULL,
         preexec_fn=os.setsid,
     )
+    shell_routes._bind_pty_spawn_identity(proc)
+    return proc
+
+
+def _bound_fake_leader(monkeypatch, pid=4242):
+    """A fake PTY leader whose spawn identity verifies and still leads its group."""
+    from src import process_lifecycle, process_ownership
+
+    real_getpgid = os.getpgid
+    monkeypatch.setattr(process_ownership, "verify", lambda p, token: process_ownership.OWNED)
+    monkeypatch.setattr(os, "getpgid", lambda p: pid if p == pid else real_getpgid(p))
+    return SimpleNamespace(
+        pid=pid, returncode=0, wait=None,
+        _ody_pty_identity=process_lifecycle.ProcessIdentity(pid, "spawn-token", pgid=pid),
+    )
 
 
 def _stubborn_child(pid_file: Path, ignore: tuple[str, ...]) -> str:
@@ -281,9 +299,8 @@ async def test_terminate_pty_session_reports_a_session_it_could_not_kill(
     monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
     monkeypatch.setattr(shell_routes, "_signal_session", lambda *_: True)
     monkeypatch.setattr(shell_routes, "_session_alive", lambda *_: True)
-    monkeypatch.setattr(shell_routes, "_session_pgid", lambda _: 4242)
 
-    proc = SimpleNamespace(pid=4242, returncode=0, wait=None)
+    proc = _bound_fake_leader(monkeypatch)
     assert await shell_routes._terminate_pty_session(proc) is False
 
 
@@ -293,7 +310,6 @@ async def test_terminate_pty_session_escalates_before_giving_up(monkeypatch):
 
     sent = []
     monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
-    monkeypatch.setattr(shell_routes, "_session_pgid", lambda _: 4242)
     monkeypatch.setattr(shell_routes, "_session_alive", lambda *_: True)
     monkeypatch.setattr(
         shell_routes,
@@ -301,7 +317,7 @@ async def test_terminate_pty_session_escalates_before_giving_up(monkeypatch):
         lambda pgid, pid, sig: sent.append(sig) or True,
     )
 
-    proc = SimpleNamespace(pid=4242, returncode=0, wait=None)
+    proc = _bound_fake_leader(monkeypatch)
     await shell_routes._terminate_pty_session(proc)
 
     assert sent == [signal.SIGTERM, signal.SIGKILL]
@@ -338,6 +354,65 @@ async def test_generate_pty_timeout_says_so_when_the_session_survives(
     )
 
 
+@pytest.mark.skipif(os.name == "nt", reason="POSIX process groups")
+async def test_terminate_pty_session_never_signals_the_servers_own_group(monkeypatch):
+    """If setsid did not apply, the child's group is ours: reach the child alone."""
+    import routes.shell_routes as shell_routes
+
+    from src import process_ownership
+
+    own = os.getpgid(0)
+    sent = []
+    monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
+    monkeypatch.setattr(shell_routes.process_lifecycle, "pgid_of", lambda _pid: own)
+    monkeypatch.setattr(process_ownership, "start_token", lambda pid: "the-leader")
+
+    proc = SimpleNamespace(pid=987654, returncode=0, wait=None)
+    assert shell_routes._session_pgid(proc.pid) is None
+    shell_routes._bind_pty_spawn_identity(proc)
+    assert proc._ody_pty_identity.pgid is None  # no safe session group recorded
+
+    monkeypatch.setattr(os, "killpg", lambda pgid, sig: sent.append(("group", pgid, sig)))
+    monkeypatch.setattr(os, "kill", lambda pid, sig: sent.append(("pid", pid, sig)))
+    await shell_routes._terminate_pty_session(proc)
+
+    assert sent and all(kind == "pid" and target == 987654 for kind, target, _ in sent), sent
+
+
+@pytest.mark.skipif(os.name == "nt", reason="POSIX process groups")
+async def test_terminate_pty_session_never_signals_a_reused_leader_pid(monkeypatch):
+    """Leader spawned and bound → reaped → pid reissued → teardown signals nothing.
+
+    The replacement is the worst case: an unrelated session leader, so both
+    its pid and its process group carry the number our leader had.
+    """
+    import routes.shell_routes as shell_routes
+    from src import process_ownership
+
+    pid = 987650
+    real_getpgid = os.getpgid
+    occupant = {"token": "leader-token"}
+    monkeypatch.setattr(process_ownership, "start_token",
+                        lambda p: occupant["token"] if int(p) == pid else None)
+    monkeypatch.setattr(os, "getpgid", lambda p: pid if p == pid else real_getpgid(p))
+
+    proc = SimpleNamespace(pid=pid, returncode=None, wait=None)
+    shell_routes._bind_pty_spawn_identity(proc)  # spawn time: the leader we just created
+    assert proc._ody_pty_identity.pgid == pid
+
+    # The leader exits and is reaped; the kernel reissues its pid to a stranger.
+    proc.returncode = 0
+    occupant["token"] = "replacement-token"
+    signalled = []
+    monkeypatch.setattr(os, "killpg", lambda g, sig: sig and signalled.append(("group", g, sig)))
+    monkeypatch.setattr(os, "kill", lambda p, sig: sig and signalled.append(("pid", p, sig)))
+    monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
+
+    await shell_routes._terminate_pty_session(proc)
+
+    assert signalled == [], f"teardown signalled the replacement: {signalled}"
+
+
 def test_session_alive_treats_a_refused_probe_as_alive(monkeypatch):
     """EPERM says the group exists but we may not signal it, not that it died.
 
@@ -372,11 +447,10 @@ async def test_terminate_pty_session_reports_a_group_it_may_not_signal(monkeypat
         raise PermissionError(errno.EPERM, "Operation not permitted")
 
     monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
-    monkeypatch.setattr(shell_routes, "_session_pgid", lambda _: 4242)
+    proc = _bound_fake_leader(monkeypatch)
     monkeypatch.setattr(shell_routes.os, "killpg", refuse)
     monkeypatch.setattr(shell_routes.os, "kill", refuse)
 
-    proc = SimpleNamespace(pid=4242, returncode=0, wait=None)
     assert await shell_routes._terminate_pty_session(proc) is False
 
 

tests/test_workspace_artifact_tool_floor.py

@@ -1,5 +1,7 @@
 from pathlib import Path
 
+import pytest
+
 
 def test_unoffered_artifact_recovery_is_bounded():
     from src.agent_loop import _artifact_unoffered_recovery_exhausted