refactor(runtime): centralize verified process lifecycle

Extract the generic process lifecycle layer (src/process_lifecycle.py)
shared by runtime-owned subprocesses: process identity (pid + boot-bound
start token), identity-bound observation, group and pidfd probes, the
TERM -> verify -> KILL -> verify escalation with re-gating before
escalation, identity-scoped sweeps, and the termination receipt.

Containment, the PTY shell, the Cookbook survivor sweep, the browser
lifecycle, web_tools browser cleanup, kill_process_tree and the startup
reaper consume it while keeping their own ownership semantics.

Safety corrections:
- browser membership and identity are bound in one snapshot; no identity
  is recaptured after membership is decided
- web_tools legacy pid-file and profile-match kills signal only verified
  identities; browser CLI groups only while their spawn identity verifies
- Cookbook and legacy-tmux descendant capture bind membership to identity
- PTY teardown never signals the server's own process group
- unverifiable processes are reported, never signalled
This commit is contained in:
Alexandre Teixeira
2026-10-02 01:27:08 +01:00
parent 7aa891e5e6
commit f9aa2818c4
14 changed files with 1692 additions and 408 deletions
+31
View File
@@ -132,3 +132,34 @@ def test_legacy_cleanup_against_a_private_real_tmux_server(tmp_path, monkeypatch
assert containment.active_grants() == []
finally:
subprocess.run([real_tmux, "-S", socket, "kill-server"], capture_output=True)
def test_a_descendant_reissued_after_the_table_read_is_never_released(legacy, monkeypatch):
reads = {"n": 0}
def table():
reads["n"] += 1
rows = {4200: process_ownership.ProcessInfo(4200, 4100, "/bin/bash --noprofile --norc")}
if reads["n"] <= 2:
rows[4201] = process_ownership.ProcessInfo(4201, 4200, "sleep 60")
else:
# The child exited and its pid now names an unrelated process.
rows[4201] = process_ownership.ProcessInfo(4201, 1, "sshd: stranger")
return rows
monkeypatch.setattr(process_ownership, "process_table", table)
process_reaper.reap_legacy_agent_tmux()
assert 4201 not in [pid for pid, _ in legacy["released"]]
def test_an_unidentifiable_descendant_keeps_the_session_unsignalled(legacy, monkeypatch):
def start_token(pid):
if pid == 4201:
raise process_ownership.InspectionUnavailable("/proc/4201/stat")
return f"token:{pid}"
monkeypatch.setattr(process_ownership, "start_token", start_token)
report = process_reaper.reap_legacy_agent_tmux()
assert report["unverifiable"] == 1
assert legacy["released"] == []
assert all(call[1] != "kill-session" for call in legacy["calls"])
+75 -3
View File
@@ -13,14 +13,18 @@ import pytest
from core import platform_compat
import src.agent_tools.web_tools as web_tools
from src import browser_lifecycle
from src import browser_lifecycle, process_ownership
from src.agent_tools.web_tools import PrivateBrowserTool
def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline: str, state: str = "S") -> None:
def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline: str,
state: str = "S", starttime: int | None = None) -> None:
entry = root / str(pid)
entry.mkdir(parents=True)
(entry / "stat").write_text(f"{pid} (x y) {state} {ppid} {pgid} {sid} 0 0 0")
# A full stat line: fields after the comm up to starttime (field 22), which
# is what process identity is read from.
tail = " ".join(["0"] * 15 + [str(starttime if starttime is not None else 1000 + pid)])
(entry / "stat").write_text(f"{pid} (x y) {state} {ppid} {pgid} {sid} {tail}")
(entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
@@ -28,7 +32,12 @@ def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline:
def fake_procfs(monkeypatch, tmp_path):
proc = tmp_path / "proc"
proc.mkdir()
boot = proc / "sys/kernel/random/boot_id"
boot.parent.mkdir(parents=True)
boot.write_text("fake-boot\n")
monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
# Identity reads its own binding of the proc root.
monkeypatch.setattr(process_ownership, "PROC_ROOT", proc)
killed: list[tuple[int, int]] = []
def _kill(pid, sig):
@@ -101,6 +110,69 @@ def test_forced_cleanup_kills_tree_and_removes_owned_resources(fake_procfs, tmp_
assert (proc / "900").exists()
def test_member_recycled_between_membership_and_identity_is_never_signalled(
fake_procfs, tmp_path, monkeypatch,
) -> None:
"""Membership sees the old renderer; its pid is reused before any capture or signal.
The replacement is an unrelated process occupying the same pid slot. Its
identity must never be the one teardown verifies, so it is never hit.
"""
proc, killed = fake_procfs
_browser_tree(proc, tmp_path / "agent-browser-chrome-a")
lifecycle = browser_lifecycle.process_lifecycle
recycled = []
def recycle_once():
if not recycled:
recycled.append(True)
shutil.rmtree(proc / "502")
_fake_proc(proc, 502, ppid=1, pgid=502, sid=502, cmdline="sshd", starttime=99999)
# Whichever comes first after membership is decided — an identity capture
# or the signalling sweep — the old renderer is gone and its pid reissued.
real_capture = lifecycle.ProcessIdentity.capture
real_terminate = lifecycle.terminate_identities
def capture(pid, **kwargs):
recycle_once()
return real_capture(pid, **kwargs)
def terminate(identities, **kwargs):
identities = list(identities)
recycle_once()
return real_terminate(identities, **kwargs)
monkeypatch.setattr(lifecycle.ProcessIdentity, "capture", staticmethod(capture))
monkeypatch.setattr(lifecycle, "terminate_identities", terminate)
killed_pids, survivors, _ = browser_lifecycle.kill_browser_tree(500, settle_s=0.1)
assert recycled, "the race was never staged"
assert 502 not in [pid for pid, _ in killed], "the replacement process was signalled"
assert (proc / "502").exists()
assert killed_pids == [501, 500] and survivors == []
def test_member_without_identity_is_a_survivor_and_keeps_its_profile(fake_procfs, tmp_path) -> None:
proc, killed = fake_procfs
root = tmp_path / "rt"
root.mkdir()
(root / "ody-k.pid").write_text("500")
profile = tmp_path / "agent-browser-chrome-a"
profile.mkdir()
_browser_tree(proc, profile)
# The renderer's stat is unreadable as identity (no start time): this host
# cannot say which process holds the pid, so it must not be signalled.
(proc / "502" / "stat").write_text("502 (x y) S 501 501 500")
receipt = browser_lifecycle.force_cleanup(root, "ody-k")
assert 502 not in [pid for pid, _ in killed]
assert receipt.survivors == [502] and not receipt.verified
assert profile.exists() and (root / "ody-k.pid").exists()
def test_forced_cleanup_without_procfs_never_kills_unverified_processes(monkeypatch, tmp_path) -> None:
monkeypatch.setattr(platform_compat, "PROC_ROOT", tmp_path / "missing")
monkeypatch.setattr(browser_lifecycle.os, "kill", lambda *a: pytest.fail("killed"))
@@ -320,3 +320,69 @@ def test_model_process_scan_returns_empty_without_procfs(monkeypatch, tmp_path):
monkeypatch.setattr(os, "listdir", _unexpected_listdir)
assert tools._scan_running_model_processes() == []
@pytest.mark.asyncio
async def test_stop_reports_a_survivor_it_can_no_longer_identify(monkeypatch, tmp_path):
"""Captured as ours, unverifiable at sweep time: not signalled, and said so."""
from src import process_ownership
tracked_cmd = "python -m vllm.entrypoints.openai.api_server --model org/model"
state = _tracked_state(cmd=tracked_cmd)
posts = _install_httpx_client(monkeypatch, state)
_install_successful_tmux_kill(monkeypatch, panes="serve-abc123 900\n")
table = _fake_table(monkeypatch, {900: (1, "bash"), 101: (900, tracked_cmd)})
asked = {"n": 0}
def _token(pid):
if int(pid) == 101:
asked["n"] += 1
if asked["n"] > 1: # the capture succeeded; every later look fails
raise process_ownership.InspectionUnavailable("/proc/101/stat")
return "token:101"
return f"token:{pid}" if int(pid or 0) in table else None
monkeypatch.setattr(process_ownership, "start_token", _token)
signalled = _install_effective_kill(monkeypatch, table)
result = await tools.do_stop_served_model(json.dumps({"session_id": "serve-abc123"}))
assert result["exit_code"] == 0
assert not any(pid == 101 for pid, _sig in signalled)
assert "could not be re-identified and were not signalled (pid 101)" in result["output"]
assert _stopped_statuses(posts, "serve-abc123") == ["stopped"]
@pytest.mark.asyncio
async def test_stop_never_signals_a_pid_reissued_between_the_table_and_its_capture(
monkeypatch, tmp_path
):
"""The table places 101 under the pane; 101 is then reissued to a stranger.
The stranger's token must never be the one recorded for the session.
"""
from src import process_ownership
tracked_cmd = "python -m vllm.entrypoints.openai.api_server --model org/model"
state = _tracked_state(cmd=tracked_cmd)
posts = _install_httpx_client(monkeypatch, state)
_install_successful_tmux_kill(monkeypatch, panes="serve-abc123 900\n")
table = _fake_table(monkeypatch, {900: (1, "bash"), 101: (900, tracked_cmd)})
reads = {"n": 0}
def _table():
reads["n"] += 1
if reads["n"] > 1:
# After the first read: the server exited and its pid now belongs
# to an unrelated process with a fresh identity.
table[101] = process_ownership.ProcessInfo(101, 1, "sshd: stranger")
return dict(table)
monkeypatch.setattr(process_ownership, "process_table", _table)
signalled = _install_effective_kill(monkeypatch, table)
result = await tools.do_stop_served_model(json.dumps({"session_id": "serve-abc123"}))
assert result["exit_code"] == 0
assert not any(pid == 101 for pid, _sig in signalled)
assert _stopped_statuses(posts, "serve-abc123") == ["stopped"]
+179 -15
View File
@@ -2,6 +2,7 @@ import asyncio
import pytest
import base64
import json
import shutil
from pathlib import Path
from core import platform_compat
@@ -1436,27 +1437,77 @@ def test_private_browser_screenshot_without_path_returns_image_payload(monkeypat
}]
def test_private_browser_timeout_terminates_the_process_group(monkeypatch) -> None:
calls = []
def _cli_proc(calls, identity=None):
class _Proc:
pid = 1234
returncode = None
_ody_identity = identity
def kill(self):
calls.append("fallback-kill")
return _Proc()
def test_private_browser_timeout_terminates_the_process_group(monkeypatch) -> None:
from src import process_lifecycle, process_ownership
calls = []
monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
monkeypatch.setattr(process_ownership, "verify", lambda pid, token: process_ownership.OWNED)
monkeypatch.setattr(
web_tools.os,
"killpg",
lambda pgid, signum: calls.append((pgid, signum)),
)
PrivateBrowserTool._terminate_subprocess(_Proc())
PrivateBrowserTool._terminate_subprocess(
_cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
assert calls == [(1234, web_tools.signal.SIGKILL), "fallback-kill"]
@pytest.mark.parametrize("verdict", ["foreign", "unverifiable", "gone"])
def test_cli_group_is_not_signalled_once_its_identity_is_lost(monkeypatch, verdict) -> None:
"""A reaped CLI's pid may be reissued; its group is then not ours to kill."""
from src import process_lifecycle, process_ownership
calls = []
monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
monkeypatch.setattr(process_ownership, "verify", lambda pid, token: verdict)
monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled an unowned group"))
PrivateBrowserTool._terminate_subprocess(
_cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
assert calls == ["fallback-kill"]
def test_cli_without_a_spawn_identity_is_never_group_signalled(monkeypatch) -> None:
calls = []
monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled a bare pid's group"))
PrivateBrowserTool._terminate_subprocess(_cli_proc(calls))
assert calls == ["fallback-kill"]
def test_cli_group_that_moved_is_not_signalled(monkeypatch) -> None:
"""The identity verifies but no longer leads the recorded group."""
from src import process_lifecycle, process_ownership
calls = []
monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: 999)
monkeypatch.setattr(process_ownership, "verify", lambda pid, token: process_ownership.OWNED)
monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled a group it does not lead"))
PrivateBrowserTool._terminate_subprocess(
_cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
assert calls == ["fallback-kill"]
def test_private_browser_retries_one_timed_out_local_open(monkeypatch, tmp_path) -> None:
page = tmp_path / "output.html"
page.write_text("<html><title>retry</title></html>")
@@ -1900,29 +1951,142 @@ def test_terminate_owned_chrome_kills_only_this_runtimes_profile(
) -> None:
"""With procfs present, match on the runtime-owned profile prefix alone."""
proc = tmp_path / "proc"
proc = _fake_procfs(monkeypatch, tmp_path)
tmpdir = tmp_path / "runtime-tmp"
tmpdir.mkdir()
profile_prefix = str(tmpdir.resolve() / "agent-browser-chrome-")
def _write_pid(pid: str, cmdline: str) -> None:
entry = proc / pid
entry.mkdir(parents=True)
(entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
_write_pid("101", f"chrome --user-data-dir={profile_prefix}abc")
_write_pid("202", "chrome --user-data-dir=/Users/someone/Library/Chrome")
_fake_process(proc, 101, f"chrome --user-data-dir={profile_prefix}abc")
_fake_process(proc, 202, "chrome --user-data-dir=/Users/someone/Library/Chrome")
(proc / "self").mkdir()
monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
killed: list[int] = []
monkeypatch.setattr(web_tools.os, "kill", lambda pid, sig: killed.append(pid))
killed = _install_lethal_kill(monkeypatch, proc)
PrivateBrowserTool._terminate_owned_chrome({"TMPDIR": str(tmpdir)})
assert killed == [101]
def _fake_procfs(monkeypatch, tmp_path):
from src import process_ownership
proc = tmp_path / "proc"
boot = proc / "sys/kernel/random/boot_id"
boot.parent.mkdir(parents=True)
boot.write_text("fake-boot\n")
monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
monkeypatch.setattr(process_ownership, "PROC_ROOT", proc)
return proc
def _fake_process(proc, pid: int, cmdline: str, *, starttime: int | None = None) -> None:
entry = proc / str(pid)
entry.mkdir(parents=True)
tail = " ".join(["0"] * 15 + [str(starttime if starttime is not None else 1000 + pid)])
(entry / "stat").write_text(f"{pid} (x) S 1 {pid} {pid} {tail}")
(entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
def _install_lethal_kill(monkeypatch, proc, *, before_kill=None):
killed: list[int] = []
def _kill(pid, sig):
if before_kill is not None:
before_kill(pid)
entry = proc / str(pid)
if not entry.exists():
raise ProcessLookupError(pid)
killed.append(pid)
shutil.rmtree(entry)
monkeypatch.setattr(web_tools.os, "kill", _kill)
return killed
def test_owned_chrome_sweep_never_signals_a_reused_pid(monkeypatch, tmp_path) -> None:
"""Matched by profile, then reissued to a stranger before the signal."""
from src import process_lifecycle
proc = _fake_procfs(monkeypatch, tmp_path)
tmpdir = tmp_path / "runtime-tmp"
tmpdir.mkdir()
_fake_process(proc, 101, f"chrome --user-data-dir={tmpdir.resolve()}/agent-browser-chrome-x")
killed = _install_lethal_kill(monkeypatch, proc)
real_terminate = process_lifecycle.terminate_identities
def recycle_then_terminate(identities, **kwargs):
identities = list(identities)
shutil.rmtree(proc / "101")
_fake_process(proc, 101, "postgres", starttime=99999)
return real_terminate(identities, **kwargs)
monkeypatch.setattr(process_lifecycle, "terminate_identities", recycle_then_terminate)
PrivateBrowserTool._terminate_owned_chrome({"TMPDIR": str(tmpdir)})
assert killed == [] and (proc / "101").exists()
def _legacy_pid_file_for(tmp_path, monkeypatch, namespace, session, pid):
"""A pid file in the legacy namespace layout, which only the fallback loop reads."""
monkeypatch.setenv("XDG_RUNTIME_DIR", str(tmp_path))
monkeypatch.setenv("ODYSSEUS_BROWSER_NAMESPACE", namespace)
legacy_run = (tmp_path / "agent-browser" / "namespaces"
/ web_tools._bounded_browser_identity(namespace) / "run")
legacy_run.mkdir(parents=True, exist_ok=True)
target = legacy_run / f"ody-{web_tools._bounded_browser_identity(session)}.pid"
assert target in web_tools._browser_pid_file_candidates(tmp_path, namespace, session)
target.write_text(str(pid))
return target
def test_legacy_daemon_pid_file_kills_only_the_verified_daemon(monkeypatch, tmp_path) -> None:
proc = _fake_procfs(monkeypatch, tmp_path)
_fake_process(proc, 4401, "node agent-browser --serve")
killed = _install_lethal_kill(monkeypatch, proc)
pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-7", 4401)
PrivateBrowserTool._terminate_owned_daemon({}, "session-7")
assert killed == [4401] and not pid_file.exists()
def test_legacy_daemon_pid_reused_before_the_signal_is_spared(monkeypatch, tmp_path) -> None:
"""The pid file still names the slot; the daemon in it was replaced."""
from src import process_lifecycle
proc = _fake_procfs(monkeypatch, tmp_path)
_fake_process(proc, 4402, "node agent-browser --serve")
killed = _install_lethal_kill(monkeypatch, proc)
pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-8", 4402)
real_terminate = process_lifecycle.terminate_identities
def recycle_then_terminate(identities, **kwargs):
identities = list(identities)
shutil.rmtree(proc / "4402")
_fake_process(proc, 4402, "node agent-browser --serve", starttime=99999)
return real_terminate(identities, **kwargs)
monkeypatch.setattr(process_lifecycle, "terminate_identities", recycle_then_terminate)
PrivateBrowserTool._terminate_owned_daemon({}, "session-8")
# Even a lookalike command line is not the process the match was made on.
assert killed == [] and (proc / "4402").exists()
def test_legacy_daemon_without_identity_keeps_its_pid_file(monkeypatch, tmp_path) -> None:
proc = _fake_procfs(monkeypatch, tmp_path)
_fake_process(proc, 4403, "node agent-browser --serve")
(proc / "4403" / "stat").write_text("4403 (x) S 1") # no start time: unidentifiable
monkeypatch.setattr(web_tools.os, "kill", lambda *a: pytest.fail("signalled an unidentified pid"))
pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-9", 4403)
PrivateBrowserTool._terminate_owned_daemon({}, "session-9")
assert pid_file.exists()
def _pid_file_for(tmp_path, monkeypatch, namespace, session, pid):
"""Write a pid file where the daemon helpers will look for it."""
monkeypatch.setenv("XDG_RUNTIME_DIR", str(tmp_path))
+365
View File
@@ -0,0 +1,365 @@
"""The generic process lifecycle: identity, probes, escalation, verified death.
These pin the rules every consumer — containment, the PTY shell, the Cookbook
sweep, the browser lifecycle and kill_process_tree — inherits from one place.
"""
import asyncio
import errno
import os
import signal
import subprocess
import sys
import time
import pytest
from core import platform_compat
from src import process_lifecycle, process_ownership
posix_only = pytest.mark.skipif(os.name == "nt", reason="POSIX process groups and signals")
_IGNORE_TERM = (
"import signal, sys, time\n"
"signal.signal(signal.SIGTERM, signal.SIG_IGN)\n"
"print('ready', flush=True)\n"
"time.sleep(60)\n"
)
def _spawn(code: str = _IGNORE_TERM) -> subprocess.Popen:
proc = subprocess.Popen([sys.executable, "-c", code], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, start_new_session=True)
assert proc.stdout.readline().strip() == b"ready"
return proc
def _cleanup(proc: subprocess.Popen) -> None:
if proc.poll() is None:
proc.kill()
proc.wait(timeout=5)
proc.stdout.close()
# ── Groups ──────────────────────────────────────────────────────────────────
@posix_only
def test_our_own_group_is_never_present_and_never_signalled(monkeypatch):
own = os.getpgid(0)
sent = []
monkeypatch.setattr(os, "killpg", lambda pgid, sig: sent.append(("group", pgid, sig)))
monkeypatch.setattr(os, "kill", lambda pid, sig: sent.append(("pid", pid, sig)))
assert process_lifecycle.group_present(own) is False
assert process_lifecycle.signal_group(4242, own, signal.SIGTERM) is True
assert sent == [("pid", 4242, signal.SIGTERM)]
@posix_only
def test_a_refused_group_probe_is_a_live_group(monkeypatch):
def refuse(*_args):
raise PermissionError(errno.EPERM, "Operation not permitted")
monkeypatch.setattr(os, "killpg", refuse)
assert process_lifecycle.group_present(987654, own=1) is True
def gone(*_args):
raise ProcessLookupError(errno.ESRCH, "No such process")
monkeypatch.setattr(os, "killpg", gone)
assert process_lifecycle.group_present(987654, own=1) is False
@posix_only
def test_signal_group_reports_when_nothing_was_left_to_signal(monkeypatch):
def gone(*_args):
raise ProcessLookupError(errno.ESRCH, "No such process")
monkeypatch.setattr(os, "killpg", gone)
monkeypatch.setattr(os, "kill", gone)
assert process_lifecycle.signal_group(4242, 4242, signal.SIGTERM, own=1) is False
# ── Escalation ──────────────────────────────────────────────────────────────
def _ladder():
return ((signal.SIGTERM, 0.01), (getattr(signal, "SIGKILL", signal.SIGTERM), 0.01))
def test_escalate_does_not_signal_a_target_already_gone():
sent = []
result = process_lifecycle.escalate(lambda: True, sent.append, steps=_ladder())
assert result.dead is True and result.escalated is False and sent == []
def test_escalate_terms_then_kills_and_reports_the_observed_outcome():
sent = []
result = process_lifecycle.escalate(lambda: False, sent.append, steps=_ladder(), poll_s=0.001)
assert sent == [step[0] for step in _ladder()]
assert result.dead is False and result.escalated is True
def test_escalate_stops_when_term_is_enough():
sent = []
result = process_lifecycle.escalate(lambda: bool(sent), sent.append, steps=_ladder())
assert sent == [signal.SIGTERM]
assert result.dead is True and result.escalated is False
def test_escalate_reverifies_before_kill_and_refuses_on_a_lost_identity():
sent = []
result = process_lifecycle.escalate(
lambda: False, sent.append, steps=_ladder(), poll_s=0.001,
before_step=lambda sig: "foreign",
)
assert sent == [signal.SIGTERM]
assert result.refusal == "foreign" and result.dead is False
def test_escalate_stops_the_ladder_when_nothing_is_left_to_signal():
sent = []
def send(sig):
sent.append(sig)
return False
result = process_lifecycle.escalate(lambda: False, send, steps=_ladder())
assert sent == [signal.SIGTERM] and result.dead is False
async def test_escalate_async_signals_first_and_reaps_inside_the_window():
sent, waited = [], []
state = {"gone": False}
async def wait():
waited.append(True)
state["gone"] = True
result = await process_lifecycle.escalate_async(
lambda: state["gone"], sent.append, steps=_ladder(), wait=wait,
)
# No precheck: an awaited leader is reaped only after the first signal.
assert sent == [signal.SIGTERM] and waited == [True]
assert result.dead is True and result.escalated is False
async def test_escalate_async_gives_the_reap_a_floor_even_at_zero_grace():
seen = {}
async def wait():
seen["ran"] = True
await process_lifecycle.escalate_async(
lambda: False, lambda sig: None, steps=((signal.SIGTERM, 0.0),), wait=wait,
)
assert seen == {"ran": True}
# ── Identity ────────────────────────────────────────────────────────────────
def test_identity_from_record_needs_a_pid():
assert process_lifecycle.ProcessIdentity.from_record({}) is None
assert process_lifecycle.ProcessIdentity.from_record({"pid": "x"}) is None
ident = process_lifecycle.ProcessIdentity.from_record(
{"supervisor_pid": 42, "supervisor_token": "t", "pgid": "7"},
pid_key="supervisor_pid", token_key="supervisor_token",
)
assert ident == process_lifecycle.ProcessIdentity(pid=42, start_token="t", pgid=7)
def test_a_pid_without_a_token_is_never_owned_and_never_exited(monkeypatch):
monkeypatch.setattr(process_ownership, "start_token", lambda pid: "live")
ident = process_lifecycle.ProcessIdentity(pid=4242, start_token=None)
assert ident.verdict() == process_ownership.UNVERIFIABLE
assert ident.owned() is False and ident.exited() is False
@pytest.mark.parametrize("current,exited", [(None, True), ("other", True), ("mine", False)])
def test_exited_means_gone_or_recycled(monkeypatch, current, exited):
monkeypatch.setattr(process_ownership, "start_token", lambda pid: current)
monkeypatch.setattr(process_lifecycle, "is_zombie", lambda pid: False)
ident = process_lifecycle.ProcessIdentity(pid=4242, start_token="mine")
assert ident.exited() is exited
@pytest.mark.skipif(not platform_compat.has_procfs(), reason="zombie state needs procfs")
def test_an_unreaped_zombie_has_exited():
proc = subprocess.Popen([sys.executable, "-c", "pass"])
ident = process_lifecycle.ProcessIdentity.capture(proc.pid)
try:
for _ in range(250):
if process_lifecycle.is_zombie(proc.pid):
break
time.sleep(0.02)
assert ident.verdict() == process_ownership.OWNED # still in its slot
assert ident.exited() is True
finally:
proc.wait(timeout=5)
# ── Snapshot identities ─────────────────────────────────────────────────────
def test_terminate_identities_never_signals_foreign_or_unverifiable(monkeypatch):
tokens = {1: "now-someone-else", 2: None}
def start_token(pid):
if pid == 3:
raise process_ownership.InspectionUnavailable("stat")
return tokens.get(pid)
monkeypatch.setattr(process_ownership, "start_token", start_token)
monkeypatch.setattr(os, "kill", lambda *a: pytest.fail("signalled an unowned pid"))
sweep = process_lifecycle.terminate_identities(
[process_lifecycle.ProcessIdentity(1, "mine"),
process_lifecycle.ProcessIdentity(2, "mine"),
process_lifecycle.ProcessIdentity(3, "mine")],
steps=_ladder(),
)
assert sweep.killed == () and sweep.survivors == ()
assert sweep.unverified == (3,) and sweep.dead is False
def test_terminate_identities_kills_in_order_and_reverifies_each_signal(monkeypatch):
alive = {10: "a", 11: "b", 12: "c"}
sent = []
monkeypatch.setattr(process_ownership, "start_token", lambda pid: alive.get(pid))
monkeypatch.setattr(process_lifecycle, "is_zombie", lambda pid: False)
def kill(pid, sig):
sent.append((pid, sig))
alive.pop(pid, None)
monkeypatch.setattr(os, "kill", kill)
sweep = process_lifecycle.terminate_identities(
[process_lifecycle.ProcessIdentity(pid, token) for pid, token in list(alive.items())],
steps=((signal.SIGTERM, 0.05),),
)
assert [pid for pid, _ in sent] == [10, 11, 12]
assert sweep.killed == (10, 11, 12) and sweep.dead
@posix_only
@pytest.mark.skipif(not platform_compat.has_procfs(), reason="real identity needs procfs")
def test_terminate_identities_escalates_past_an_ignored_sigterm():
proc = _spawn()
try:
ident = process_lifecycle.ProcessIdentity.capture(proc.pid)
sweep = process_lifecycle.terminate_identities(
[ident], steps=process_lifecycle.term_kill_steps(0.2))
assert sweep.killed == (proc.pid,) and sweep.survivors == ()
assert proc.wait(timeout=5) == -signal.SIGKILL
finally:
_cleanup(proc)
# ── terminate_tree / kill_process_tree ──────────────────────────────────────
@posix_only
@pytest.mark.skipif(not platform_compat.has_procfs(), reason="real identity needs procfs")
def test_terminate_tree_requires_a_matching_identity():
proc = _spawn()
try:
refused = process_lifecycle.terminate_tree(
proc.pid, pgid=proc.pid, start_token="procfs:not-this-process",
require_identity=True, grace_s=0.1)
assert refused.dead is False and refused.ownership == process_ownership.FOREIGN
assert refused.survivors == ()
assert proc.poll() is None
token = process_ownership.start_token(proc.pid)
outcome = process_lifecycle.terminate_tree(
proc.pid, pgid=proc.pid, start_token=token, require_identity=True, grace_s=0.2)
assert outcome.dead is True and outcome.escalated is True
finally:
_cleanup(proc)
@posix_only
def test_terminate_tree_keeps_a_leaderless_group_visible(monkeypatch):
monkeypatch.setattr(process_ownership, "start_token", lambda pid: None)
monkeypatch.setattr(process_lifecycle, "group_present", lambda pgid, **kw: True)
monkeypatch.setattr(process_lifecycle, "signal_group",
lambda *a, **kw: pytest.fail("signalled an unproven group"))
outcome = process_lifecycle.terminate_tree(4242, pgid=4242, start_token="t",
require_identity=True)
assert outcome.dead is False and outcome.survivors == (4242,)
assert outcome.ownership == process_ownership.GONE
@posix_only
def test_kill_process_tree_writes_no_containment_record(monkeypatch):
from src import containment
monkeypatch.setattr(containment, "_update_record", lambda *a, **k: pytest.fail("grant store touched"))
proc = _spawn("import time\nprint('ready', flush=True)\ntime.sleep(60)\n")
try:
outcome = platform_compat.kill_process_tree(proc.pid)
assert outcome.dead is True
assert isinstance(outcome, containment.ReleaseOutcome)
finally:
_cleanup(proc)
def test_termination_outcome_shape_is_the_teardown_block():
outcome = process_lifecycle.TerminationOutcome(dead=False, escalated=True, survivors=(1,),
mechanism="process_group", ownership="owned")
assert outcome.to_dict() == {"dead": False, "escalated": True, "survivors": [1],
"mechanism": "process_group", "ownership": "owned"}
# ── Identity-bound observation ──────────────────────────────────────────────
def _tokens(monkeypatch, sequence):
calls = iter(sequence)
def start_token(pid):
value = next(calls)
if isinstance(value, Exception):
raise value
return value
monkeypatch.setattr(process_ownership, "start_token", start_token)
def test_observe_binds_facts_to_the_token_read_around_them(monkeypatch):
_tokens(monkeypatch, ["t1", "t1"])
seen = process_lifecycle.observe(42, lambda pid: "facts")
assert seen.identity == process_lifecycle.ProcessIdentity(42, "t1") and seen.facts == "facts"
def test_observe_drops_facts_read_across_a_pid_reuse(monkeypatch):
_tokens(monkeypatch, ["old", "new"])
assert process_lifecycle.observe(42, lambda pid: "whose facts?") is None
def test_observe_of_a_gone_pid_reads_nothing(monkeypatch):
_tokens(monkeypatch, [None])
assert process_lifecycle.observe(42, lambda pid: pytest.fail("read a gone pid")) is None
@pytest.mark.parametrize("sequence", [
[process_ownership.InspectionUnavailable("stat")],
["t1", process_ownership.InspectionUnavailable("stat")],
])
def test_observe_keeps_unidentifiable_facts_without_a_token(monkeypatch, sequence):
_tokens(monkeypatch, sequence)
seen = process_lifecycle.observe(42, lambda pid: "facts")
assert seen.identity.start_token is None and seen.facts == "facts"
assert seen.identity.verdict() == process_ownership.UNVERIFIABLE
def test_bind_descendants_drops_a_pid_reparented_between_snapshots(monkeypatch):
Info = process_ownership.ProcessInfo
tables = iter([
{10: Info(10, 1, "shell"), 11: Info(11, 10, "server")},
{10: Info(10, 1, "shell"), 11: Info(11, 1, "stranger")},
])
monkeypatch.setattr(process_ownership, "process_table", lambda: next(tables))
monkeypatch.setattr(process_ownership, "start_token", lambda pid: f"t{pid}")
bound = process_lifecycle.bind_descendants([10])
assert [seen.identity.pid for seen in bound] == [10]
def test_bind_descendants_drops_a_pid_whose_token_changed(monkeypatch):
Info = process_ownership.ProcessInfo
table = {10: Info(10, 1, "shell"), 11: Info(11, 10, "server")}
monkeypatch.setattr(process_ownership, "process_table", lambda: dict(table))
monkeypatch.setattr(process_ownership, "start_token", lambda pid: f"t{pid}")
monkeypatch.setattr(process_ownership, "verify",
lambda pid, token: process_ownership.FOREIGN if pid == 11 else process_ownership.OWNED)
bound = process_lifecycle.bind_descendants([10], exclude={99})
assert [(seen.identity.pid, seen.facts.command) for seen in bound] == [(10, "shell")]
+19
View File
@@ -338,6 +338,25 @@ async def test_generate_pty_timeout_says_so_when_the_session_survives(
)
@pytest.mark.skipif(os.name == "nt", reason="POSIX process groups")
async def test_terminate_pty_session_never_signals_the_servers_own_group(monkeypatch):
"""If setsid did not apply, the child's group is ours: reach the child alone."""
import routes.shell_routes as shell_routes
own = os.getpgid(0)
sent = []
monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
monkeypatch.setattr(shell_routes.process_lifecycle, "pgid_of", lambda _pid: own)
monkeypatch.setattr(os, "killpg", lambda pgid, sig: sent.append(("group", pgid, sig)))
monkeypatch.setattr(os, "kill", lambda pid, sig: sent.append(("pid", pid, sig)))
proc = SimpleNamespace(pid=987654, returncode=0, wait=None)
assert shell_routes._session_pgid(proc.pid) is None
await shell_routes._terminate_pty_session(proc)
assert sent and all(kind == "pid" and target == 987654 for kind, target, _ in sent), sent
def test_session_alive_treats_a_refused_probe_as_alive(monkeypatch):
"""EPERM says the group exists but we may not signal it, not that it died.