mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-09 16:32:21 +02:00
refactor(runtime): centralize verified process lifecycle
Extract the generic process lifecycle layer (src/process_lifecycle.py) shared by runtime-owned subprocesses: process identity (pid + boot-bound start token), identity-bound observation, group and pidfd probes, the TERM -> verify -> KILL -> verify escalation with re-gating before escalation, identity-scoped sweeps, and the termination receipt. Containment, the PTY shell, the Cookbook survivor sweep, the browser lifecycle, web_tools browser cleanup, kill_process_tree and the startup reaper consume it while keeping their own ownership semantics. Safety corrections: - browser membership and identity are bound in one snapshot; no identity is recaptured after membership is decided - web_tools legacy pid-file and profile-match kills signal only verified identities; browser CLI groups only while their spawn identity verifies - Cookbook and legacy-tmux descendant capture bind membership to identity - PTY teardown never signals the server's own process group - unverifiable processes are reported, never signalled
This commit is contained in:
@@ -132,3 +132,34 @@ def test_legacy_cleanup_against_a_private_real_tmux_server(tmp_path, monkeypatch
|
||||
assert containment.active_grants() == []
|
||||
finally:
|
||||
subprocess.run([real_tmux, "-S", socket, "kill-server"], capture_output=True)
|
||||
|
||||
|
||||
def test_a_descendant_reissued_after_the_table_read_is_never_released(legacy, monkeypatch):
|
||||
reads = {"n": 0}
|
||||
|
||||
def table():
|
||||
reads["n"] += 1
|
||||
rows = {4200: process_ownership.ProcessInfo(4200, 4100, "/bin/bash --noprofile --norc")}
|
||||
if reads["n"] <= 2:
|
||||
rows[4201] = process_ownership.ProcessInfo(4201, 4200, "sleep 60")
|
||||
else:
|
||||
# The child exited and its pid now names an unrelated process.
|
||||
rows[4201] = process_ownership.ProcessInfo(4201, 1, "sshd: stranger")
|
||||
return rows
|
||||
|
||||
monkeypatch.setattr(process_ownership, "process_table", table)
|
||||
process_reaper.reap_legacy_agent_tmux()
|
||||
assert 4201 not in [pid for pid, _ in legacy["released"]]
|
||||
|
||||
|
||||
def test_an_unidentifiable_descendant_keeps_the_session_unsignalled(legacy, monkeypatch):
|
||||
def start_token(pid):
|
||||
if pid == 4201:
|
||||
raise process_ownership.InspectionUnavailable("/proc/4201/stat")
|
||||
return f"token:{pid}"
|
||||
|
||||
monkeypatch.setattr(process_ownership, "start_token", start_token)
|
||||
report = process_reaper.reap_legacy_agent_tmux()
|
||||
assert report["unverifiable"] == 1
|
||||
assert legacy["released"] == []
|
||||
assert all(call[1] != "kill-session" for call in legacy["calls"])
|
||||
|
||||
@@ -13,14 +13,18 @@ import pytest
|
||||
|
||||
from core import platform_compat
|
||||
import src.agent_tools.web_tools as web_tools
|
||||
from src import browser_lifecycle
|
||||
from src import browser_lifecycle, process_ownership
|
||||
from src.agent_tools.web_tools import PrivateBrowserTool
|
||||
|
||||
|
||||
def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline: str, state: str = "S") -> None:
|
||||
def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline: str,
|
||||
state: str = "S", starttime: int | None = None) -> None:
|
||||
entry = root / str(pid)
|
||||
entry.mkdir(parents=True)
|
||||
(entry / "stat").write_text(f"{pid} (x y) {state} {ppid} {pgid} {sid} 0 0 0")
|
||||
# A full stat line: fields after the comm up to starttime (field 22), which
|
||||
# is what process identity is read from.
|
||||
tail = " ".join(["0"] * 15 + [str(starttime if starttime is not None else 1000 + pid)])
|
||||
(entry / "stat").write_text(f"{pid} (x y) {state} {ppid} {pgid} {sid} {tail}")
|
||||
(entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
|
||||
|
||||
|
||||
@@ -28,7 +32,12 @@ def _fake_proc(root: Path, pid: int, *, ppid: int, pgid: int, sid: int, cmdline:
|
||||
def fake_procfs(monkeypatch, tmp_path):
|
||||
proc = tmp_path / "proc"
|
||||
proc.mkdir()
|
||||
boot = proc / "sys/kernel/random/boot_id"
|
||||
boot.parent.mkdir(parents=True)
|
||||
boot.write_text("fake-boot\n")
|
||||
monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
|
||||
# Identity reads its own binding of the proc root.
|
||||
monkeypatch.setattr(process_ownership, "PROC_ROOT", proc)
|
||||
killed: list[tuple[int, int]] = []
|
||||
|
||||
def _kill(pid, sig):
|
||||
@@ -101,6 +110,69 @@ def test_forced_cleanup_kills_tree_and_removes_owned_resources(fake_procfs, tmp_
|
||||
assert (proc / "900").exists()
|
||||
|
||||
|
||||
def test_member_recycled_between_membership_and_identity_is_never_signalled(
|
||||
fake_procfs, tmp_path, monkeypatch,
|
||||
) -> None:
|
||||
"""Membership sees the old renderer; its pid is reused before any capture or signal.
|
||||
|
||||
The replacement is an unrelated process occupying the same pid slot. Its
|
||||
identity must never be the one teardown verifies, so it is never hit.
|
||||
"""
|
||||
proc, killed = fake_procfs
|
||||
_browser_tree(proc, tmp_path / "agent-browser-chrome-a")
|
||||
lifecycle = browser_lifecycle.process_lifecycle
|
||||
recycled = []
|
||||
|
||||
def recycle_once():
|
||||
if not recycled:
|
||||
recycled.append(True)
|
||||
shutil.rmtree(proc / "502")
|
||||
_fake_proc(proc, 502, ppid=1, pgid=502, sid=502, cmdline="sshd", starttime=99999)
|
||||
|
||||
# Whichever comes first after membership is decided — an identity capture
|
||||
# or the signalling sweep — the old renderer is gone and its pid reissued.
|
||||
real_capture = lifecycle.ProcessIdentity.capture
|
||||
real_terminate = lifecycle.terminate_identities
|
||||
|
||||
def capture(pid, **kwargs):
|
||||
recycle_once()
|
||||
return real_capture(pid, **kwargs)
|
||||
|
||||
def terminate(identities, **kwargs):
|
||||
identities = list(identities)
|
||||
recycle_once()
|
||||
return real_terminate(identities, **kwargs)
|
||||
|
||||
monkeypatch.setattr(lifecycle.ProcessIdentity, "capture", staticmethod(capture))
|
||||
monkeypatch.setattr(lifecycle, "terminate_identities", terminate)
|
||||
|
||||
killed_pids, survivors, _ = browser_lifecycle.kill_browser_tree(500, settle_s=0.1)
|
||||
|
||||
assert recycled, "the race was never staged"
|
||||
assert 502 not in [pid for pid, _ in killed], "the replacement process was signalled"
|
||||
assert (proc / "502").exists()
|
||||
assert killed_pids == [501, 500] and survivors == []
|
||||
|
||||
|
||||
def test_member_without_identity_is_a_survivor_and_keeps_its_profile(fake_procfs, tmp_path) -> None:
|
||||
proc, killed = fake_procfs
|
||||
root = tmp_path / "rt"
|
||||
root.mkdir()
|
||||
(root / "ody-k.pid").write_text("500")
|
||||
profile = tmp_path / "agent-browser-chrome-a"
|
||||
profile.mkdir()
|
||||
_browser_tree(proc, profile)
|
||||
# The renderer's stat is unreadable as identity (no start time): this host
|
||||
# cannot say which process holds the pid, so it must not be signalled.
|
||||
(proc / "502" / "stat").write_text("502 (x y) S 501 501 500")
|
||||
|
||||
receipt = browser_lifecycle.force_cleanup(root, "ody-k")
|
||||
|
||||
assert 502 not in [pid for pid, _ in killed]
|
||||
assert receipt.survivors == [502] and not receipt.verified
|
||||
assert profile.exists() and (root / "ody-k.pid").exists()
|
||||
|
||||
|
||||
def test_forced_cleanup_without_procfs_never_kills_unverified_processes(monkeypatch, tmp_path) -> None:
|
||||
monkeypatch.setattr(platform_compat, "PROC_ROOT", tmp_path / "missing")
|
||||
monkeypatch.setattr(browser_lifecycle.os, "kill", lambda *a: pytest.fail("killed"))
|
||||
|
||||
@@ -320,3 +320,69 @@ def test_model_process_scan_returns_empty_without_procfs(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(os, "listdir", _unexpected_listdir)
|
||||
|
||||
assert tools._scan_running_model_processes() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_reports_a_survivor_it_can_no_longer_identify(monkeypatch, tmp_path):
|
||||
"""Captured as ours, unverifiable at sweep time: not signalled, and said so."""
|
||||
from src import process_ownership
|
||||
|
||||
tracked_cmd = "python -m vllm.entrypoints.openai.api_server --model org/model"
|
||||
state = _tracked_state(cmd=tracked_cmd)
|
||||
posts = _install_httpx_client(monkeypatch, state)
|
||||
_install_successful_tmux_kill(monkeypatch, panes="serve-abc123 900\n")
|
||||
table = _fake_table(monkeypatch, {900: (1, "bash"), 101: (900, tracked_cmd)})
|
||||
asked = {"n": 0}
|
||||
|
||||
def _token(pid):
|
||||
if int(pid) == 101:
|
||||
asked["n"] += 1
|
||||
if asked["n"] > 1: # the capture succeeded; every later look fails
|
||||
raise process_ownership.InspectionUnavailable("/proc/101/stat")
|
||||
return "token:101"
|
||||
return f"token:{pid}" if int(pid or 0) in table else None
|
||||
|
||||
monkeypatch.setattr(process_ownership, "start_token", _token)
|
||||
signalled = _install_effective_kill(monkeypatch, table)
|
||||
|
||||
result = await tools.do_stop_served_model(json.dumps({"session_id": "serve-abc123"}))
|
||||
|
||||
assert result["exit_code"] == 0
|
||||
assert not any(pid == 101 for pid, _sig in signalled)
|
||||
assert "could not be re-identified and were not signalled (pid 101)" in result["output"]
|
||||
assert _stopped_statuses(posts, "serve-abc123") == ["stopped"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_never_signals_a_pid_reissued_between_the_table_and_its_capture(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
"""The table places 101 under the pane; 101 is then reissued to a stranger.
|
||||
|
||||
The stranger's token must never be the one recorded for the session.
|
||||
"""
|
||||
from src import process_ownership
|
||||
|
||||
tracked_cmd = "python -m vllm.entrypoints.openai.api_server --model org/model"
|
||||
state = _tracked_state(cmd=tracked_cmd)
|
||||
posts = _install_httpx_client(monkeypatch, state)
|
||||
_install_successful_tmux_kill(monkeypatch, panes="serve-abc123 900\n")
|
||||
table = _fake_table(monkeypatch, {900: (1, "bash"), 101: (900, tracked_cmd)})
|
||||
reads = {"n": 0}
|
||||
|
||||
def _table():
|
||||
reads["n"] += 1
|
||||
if reads["n"] > 1:
|
||||
# After the first read: the server exited and its pid now belongs
|
||||
# to an unrelated process with a fresh identity.
|
||||
table[101] = process_ownership.ProcessInfo(101, 1, "sshd: stranger")
|
||||
return dict(table)
|
||||
|
||||
monkeypatch.setattr(process_ownership, "process_table", _table)
|
||||
signalled = _install_effective_kill(monkeypatch, table)
|
||||
|
||||
result = await tools.do_stop_served_model(json.dumps({"session_id": "serve-abc123"}))
|
||||
|
||||
assert result["exit_code"] == 0
|
||||
assert not any(pid == 101 for pid, _sig in signalled)
|
||||
assert _stopped_statuses(posts, "serve-abc123") == ["stopped"]
|
||||
|
||||
@@ -2,6 +2,7 @@ import asyncio
|
||||
import pytest
|
||||
import base64
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from core import platform_compat
|
||||
@@ -1436,27 +1437,77 @@ def test_private_browser_screenshot_without_path_returns_image_payload(monkeypat
|
||||
}]
|
||||
|
||||
|
||||
def test_private_browser_timeout_terminates_the_process_group(monkeypatch) -> None:
|
||||
calls = []
|
||||
|
||||
def _cli_proc(calls, identity=None):
|
||||
class _Proc:
|
||||
pid = 1234
|
||||
returncode = None
|
||||
_ody_identity = identity
|
||||
|
||||
def kill(self):
|
||||
calls.append("fallback-kill")
|
||||
|
||||
return _Proc()
|
||||
|
||||
|
||||
def test_private_browser_timeout_terminates_the_process_group(monkeypatch) -> None:
|
||||
from src import process_lifecycle, process_ownership
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
|
||||
monkeypatch.setattr(process_ownership, "verify", lambda pid, token: process_ownership.OWNED)
|
||||
monkeypatch.setattr(
|
||||
web_tools.os,
|
||||
"killpg",
|
||||
lambda pgid, signum: calls.append((pgid, signum)),
|
||||
)
|
||||
|
||||
PrivateBrowserTool._terminate_subprocess(_Proc())
|
||||
PrivateBrowserTool._terminate_subprocess(
|
||||
_cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
|
||||
|
||||
assert calls == [(1234, web_tools.signal.SIGKILL), "fallback-kill"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verdict", ["foreign", "unverifiable", "gone"])
|
||||
def test_cli_group_is_not_signalled_once_its_identity_is_lost(monkeypatch, verdict) -> None:
|
||||
"""A reaped CLI's pid may be reissued; its group is then not ours to kill."""
|
||||
from src import process_lifecycle, process_ownership
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
|
||||
monkeypatch.setattr(process_ownership, "verify", lambda pid, token: verdict)
|
||||
monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled an unowned group"))
|
||||
|
||||
PrivateBrowserTool._terminate_subprocess(
|
||||
_cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
|
||||
|
||||
assert calls == ["fallback-kill"]
|
||||
|
||||
|
||||
def test_cli_without_a_spawn_identity_is_never_group_signalled(monkeypatch) -> None:
|
||||
calls = []
|
||||
monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: pid)
|
||||
monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled a bare pid's group"))
|
||||
|
||||
PrivateBrowserTool._terminate_subprocess(_cli_proc(calls))
|
||||
|
||||
assert calls == ["fallback-kill"]
|
||||
|
||||
|
||||
def test_cli_group_that_moved_is_not_signalled(monkeypatch) -> None:
|
||||
"""The identity verifies but no longer leads the recorded group."""
|
||||
from src import process_lifecycle, process_ownership
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(web_tools.os, "getpgid", lambda pid: 999)
|
||||
monkeypatch.setattr(process_ownership, "verify", lambda pid, token: process_ownership.OWNED)
|
||||
monkeypatch.setattr(web_tools.os, "killpg", lambda *a: pytest.fail("signalled a group it does not lead"))
|
||||
|
||||
PrivateBrowserTool._terminate_subprocess(
|
||||
_cli_proc(calls, process_lifecycle.ProcessIdentity(1234, "spawned", pgid=1234)))
|
||||
|
||||
assert calls == ["fallback-kill"]
|
||||
|
||||
|
||||
def test_private_browser_retries_one_timed_out_local_open(monkeypatch, tmp_path) -> None:
|
||||
page = tmp_path / "output.html"
|
||||
page.write_text("<html><title>retry</title></html>")
|
||||
@@ -1900,29 +1951,142 @@ def test_terminate_owned_chrome_kills_only_this_runtimes_profile(
|
||||
) -> None:
|
||||
"""With procfs present, match on the runtime-owned profile prefix alone."""
|
||||
|
||||
proc = tmp_path / "proc"
|
||||
proc = _fake_procfs(monkeypatch, tmp_path)
|
||||
tmpdir = tmp_path / "runtime-tmp"
|
||||
tmpdir.mkdir()
|
||||
profile_prefix = str(tmpdir.resolve() / "agent-browser-chrome-")
|
||||
|
||||
def _write_pid(pid: str, cmdline: str) -> None:
|
||||
entry = proc / pid
|
||||
entry.mkdir(parents=True)
|
||||
(entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
|
||||
|
||||
_write_pid("101", f"chrome --user-data-dir={profile_prefix}abc")
|
||||
_write_pid("202", "chrome --user-data-dir=/Users/someone/Library/Chrome")
|
||||
_fake_process(proc, 101, f"chrome --user-data-dir={profile_prefix}abc")
|
||||
_fake_process(proc, 202, "chrome --user-data-dir=/Users/someone/Library/Chrome")
|
||||
(proc / "self").mkdir()
|
||||
|
||||
monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
|
||||
killed: list[int] = []
|
||||
monkeypatch.setattr(web_tools.os, "kill", lambda pid, sig: killed.append(pid))
|
||||
killed = _install_lethal_kill(monkeypatch, proc)
|
||||
|
||||
PrivateBrowserTool._terminate_owned_chrome({"TMPDIR": str(tmpdir)})
|
||||
|
||||
assert killed == [101]
|
||||
|
||||
|
||||
def _fake_procfs(monkeypatch, tmp_path):
|
||||
from src import process_ownership
|
||||
|
||||
proc = tmp_path / "proc"
|
||||
boot = proc / "sys/kernel/random/boot_id"
|
||||
boot.parent.mkdir(parents=True)
|
||||
boot.write_text("fake-boot\n")
|
||||
monkeypatch.setattr(platform_compat, "PROC_ROOT", proc)
|
||||
monkeypatch.setattr(process_ownership, "PROC_ROOT", proc)
|
||||
return proc
|
||||
|
||||
|
||||
def _fake_process(proc, pid: int, cmdline: str, *, starttime: int | None = None) -> None:
|
||||
entry = proc / str(pid)
|
||||
entry.mkdir(parents=True)
|
||||
tail = " ".join(["0"] * 15 + [str(starttime if starttime is not None else 1000 + pid)])
|
||||
(entry / "stat").write_text(f"{pid} (x) S 1 {pid} {pid} {tail}")
|
||||
(entry / "cmdline").write_bytes(cmdline.replace(" ", "\0").encode())
|
||||
|
||||
|
||||
def _install_lethal_kill(monkeypatch, proc, *, before_kill=None):
|
||||
killed: list[int] = []
|
||||
|
||||
def _kill(pid, sig):
|
||||
if before_kill is not None:
|
||||
before_kill(pid)
|
||||
entry = proc / str(pid)
|
||||
if not entry.exists():
|
||||
raise ProcessLookupError(pid)
|
||||
killed.append(pid)
|
||||
shutil.rmtree(entry)
|
||||
|
||||
monkeypatch.setattr(web_tools.os, "kill", _kill)
|
||||
return killed
|
||||
|
||||
|
||||
def test_owned_chrome_sweep_never_signals_a_reused_pid(monkeypatch, tmp_path) -> None:
|
||||
"""Matched by profile, then reissued to a stranger before the signal."""
|
||||
from src import process_lifecycle
|
||||
|
||||
proc = _fake_procfs(monkeypatch, tmp_path)
|
||||
tmpdir = tmp_path / "runtime-tmp"
|
||||
tmpdir.mkdir()
|
||||
_fake_process(proc, 101, f"chrome --user-data-dir={tmpdir.resolve()}/agent-browser-chrome-x")
|
||||
killed = _install_lethal_kill(monkeypatch, proc)
|
||||
real_terminate = process_lifecycle.terminate_identities
|
||||
|
||||
def recycle_then_terminate(identities, **kwargs):
|
||||
identities = list(identities)
|
||||
shutil.rmtree(proc / "101")
|
||||
_fake_process(proc, 101, "postgres", starttime=99999)
|
||||
return real_terminate(identities, **kwargs)
|
||||
|
||||
monkeypatch.setattr(process_lifecycle, "terminate_identities", recycle_then_terminate)
|
||||
|
||||
PrivateBrowserTool._terminate_owned_chrome({"TMPDIR": str(tmpdir)})
|
||||
|
||||
assert killed == [] and (proc / "101").exists()
|
||||
|
||||
|
||||
def _legacy_pid_file_for(tmp_path, monkeypatch, namespace, session, pid):
|
||||
"""A pid file in the legacy namespace layout, which only the fallback loop reads."""
|
||||
monkeypatch.setenv("XDG_RUNTIME_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("ODYSSEUS_BROWSER_NAMESPACE", namespace)
|
||||
legacy_run = (tmp_path / "agent-browser" / "namespaces"
|
||||
/ web_tools._bounded_browser_identity(namespace) / "run")
|
||||
legacy_run.mkdir(parents=True, exist_ok=True)
|
||||
target = legacy_run / f"ody-{web_tools._bounded_browser_identity(session)}.pid"
|
||||
assert target in web_tools._browser_pid_file_candidates(tmp_path, namespace, session)
|
||||
target.write_text(str(pid))
|
||||
return target
|
||||
|
||||
|
||||
def test_legacy_daemon_pid_file_kills_only_the_verified_daemon(monkeypatch, tmp_path) -> None:
|
||||
proc = _fake_procfs(monkeypatch, tmp_path)
|
||||
_fake_process(proc, 4401, "node agent-browser --serve")
|
||||
killed = _install_lethal_kill(monkeypatch, proc)
|
||||
pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-7", 4401)
|
||||
|
||||
PrivateBrowserTool._terminate_owned_daemon({}, "session-7")
|
||||
|
||||
assert killed == [4401] and not pid_file.exists()
|
||||
|
||||
|
||||
def test_legacy_daemon_pid_reused_before_the_signal_is_spared(monkeypatch, tmp_path) -> None:
|
||||
"""The pid file still names the slot; the daemon in it was replaced."""
|
||||
from src import process_lifecycle
|
||||
|
||||
proc = _fake_procfs(monkeypatch, tmp_path)
|
||||
_fake_process(proc, 4402, "node agent-browser --serve")
|
||||
killed = _install_lethal_kill(monkeypatch, proc)
|
||||
pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-8", 4402)
|
||||
real_terminate = process_lifecycle.terminate_identities
|
||||
|
||||
def recycle_then_terminate(identities, **kwargs):
|
||||
identities = list(identities)
|
||||
shutil.rmtree(proc / "4402")
|
||||
_fake_process(proc, 4402, "node agent-browser --serve", starttime=99999)
|
||||
return real_terminate(identities, **kwargs)
|
||||
|
||||
monkeypatch.setattr(process_lifecycle, "terminate_identities", recycle_then_terminate)
|
||||
|
||||
PrivateBrowserTool._terminate_owned_daemon({}, "session-8")
|
||||
|
||||
# Even a lookalike command line is not the process the match was made on.
|
||||
assert killed == [] and (proc / "4402").exists()
|
||||
|
||||
|
||||
def test_legacy_daemon_without_identity_keeps_its_pid_file(monkeypatch, tmp_path) -> None:
|
||||
proc = _fake_procfs(monkeypatch, tmp_path)
|
||||
_fake_process(proc, 4403, "node agent-browser --serve")
|
||||
(proc / "4403" / "stat").write_text("4403 (x) S 1") # no start time: unidentifiable
|
||||
monkeypatch.setattr(web_tools.os, "kill", lambda *a: pytest.fail("signalled an unidentified pid"))
|
||||
pid_file = _legacy_pid_file_for(tmp_path, monkeypatch, "clawmm-test", "session-9", 4403)
|
||||
|
||||
PrivateBrowserTool._terminate_owned_daemon({}, "session-9")
|
||||
|
||||
assert pid_file.exists()
|
||||
|
||||
|
||||
def _pid_file_for(tmp_path, monkeypatch, namespace, session, pid):
|
||||
"""Write a pid file where the daemon helpers will look for it."""
|
||||
monkeypatch.setenv("XDG_RUNTIME_DIR", str(tmp_path))
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
"""The generic process lifecycle: identity, probes, escalation, verified death.
|
||||
|
||||
These pin the rules every consumer — containment, the PTY shell, the Cookbook
|
||||
sweep, the browser lifecycle and kill_process_tree — inherits from one place.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import errno
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from core import platform_compat
|
||||
from src import process_lifecycle, process_ownership
|
||||
|
||||
posix_only = pytest.mark.skipif(os.name == "nt", reason="POSIX process groups and signals")
|
||||
|
||||
_IGNORE_TERM = (
|
||||
"import signal, sys, time\n"
|
||||
"signal.signal(signal.SIGTERM, signal.SIG_IGN)\n"
|
||||
"print('ready', flush=True)\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
|
||||
|
||||
def _spawn(code: str = _IGNORE_TERM) -> subprocess.Popen:
|
||||
proc = subprocess.Popen([sys.executable, "-c", code], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, start_new_session=True)
|
||||
assert proc.stdout.readline().strip() == b"ready"
|
||||
return proc
|
||||
|
||||
|
||||
def _cleanup(proc: subprocess.Popen) -> None:
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
proc.stdout.close()
|
||||
|
||||
|
||||
# ── Groups ──────────────────────────────────────────────────────────────────
|
||||
@posix_only
|
||||
def test_our_own_group_is_never_present_and_never_signalled(monkeypatch):
|
||||
own = os.getpgid(0)
|
||||
sent = []
|
||||
monkeypatch.setattr(os, "killpg", lambda pgid, sig: sent.append(("group", pgid, sig)))
|
||||
monkeypatch.setattr(os, "kill", lambda pid, sig: sent.append(("pid", pid, sig)))
|
||||
|
||||
assert process_lifecycle.group_present(own) is False
|
||||
assert process_lifecycle.signal_group(4242, own, signal.SIGTERM) is True
|
||||
assert sent == [("pid", 4242, signal.SIGTERM)]
|
||||
|
||||
|
||||
@posix_only
|
||||
def test_a_refused_group_probe_is_a_live_group(monkeypatch):
|
||||
def refuse(*_args):
|
||||
raise PermissionError(errno.EPERM, "Operation not permitted")
|
||||
|
||||
monkeypatch.setattr(os, "killpg", refuse)
|
||||
assert process_lifecycle.group_present(987654, own=1) is True
|
||||
|
||||
def gone(*_args):
|
||||
raise ProcessLookupError(errno.ESRCH, "No such process")
|
||||
|
||||
monkeypatch.setattr(os, "killpg", gone)
|
||||
assert process_lifecycle.group_present(987654, own=1) is False
|
||||
|
||||
|
||||
@posix_only
|
||||
def test_signal_group_reports_when_nothing_was_left_to_signal(monkeypatch):
|
||||
def gone(*_args):
|
||||
raise ProcessLookupError(errno.ESRCH, "No such process")
|
||||
|
||||
monkeypatch.setattr(os, "killpg", gone)
|
||||
monkeypatch.setattr(os, "kill", gone)
|
||||
assert process_lifecycle.signal_group(4242, 4242, signal.SIGTERM, own=1) is False
|
||||
|
||||
|
||||
# ── Escalation ──────────────────────────────────────────────────────────────
|
||||
def _ladder():
|
||||
return ((signal.SIGTERM, 0.01), (getattr(signal, "SIGKILL", signal.SIGTERM), 0.01))
|
||||
|
||||
|
||||
def test_escalate_does_not_signal_a_target_already_gone():
|
||||
sent = []
|
||||
result = process_lifecycle.escalate(lambda: True, sent.append, steps=_ladder())
|
||||
assert result.dead is True and result.escalated is False and sent == []
|
||||
|
||||
|
||||
def test_escalate_terms_then_kills_and_reports_the_observed_outcome():
|
||||
sent = []
|
||||
result = process_lifecycle.escalate(lambda: False, sent.append, steps=_ladder(), poll_s=0.001)
|
||||
assert sent == [step[0] for step in _ladder()]
|
||||
assert result.dead is False and result.escalated is True
|
||||
|
||||
|
||||
def test_escalate_stops_when_term_is_enough():
|
||||
sent = []
|
||||
result = process_lifecycle.escalate(lambda: bool(sent), sent.append, steps=_ladder())
|
||||
assert sent == [signal.SIGTERM]
|
||||
assert result.dead is True and result.escalated is False
|
||||
|
||||
|
||||
def test_escalate_reverifies_before_kill_and_refuses_on_a_lost_identity():
|
||||
sent = []
|
||||
result = process_lifecycle.escalate(
|
||||
lambda: False, sent.append, steps=_ladder(), poll_s=0.001,
|
||||
before_step=lambda sig: "foreign",
|
||||
)
|
||||
assert sent == [signal.SIGTERM]
|
||||
assert result.refusal == "foreign" and result.dead is False
|
||||
|
||||
|
||||
def test_escalate_stops_the_ladder_when_nothing_is_left_to_signal():
|
||||
sent = []
|
||||
|
||||
def send(sig):
|
||||
sent.append(sig)
|
||||
return False
|
||||
|
||||
result = process_lifecycle.escalate(lambda: False, send, steps=_ladder())
|
||||
assert sent == [signal.SIGTERM] and result.dead is False
|
||||
|
||||
|
||||
async def test_escalate_async_signals_first_and_reaps_inside_the_window():
|
||||
sent, waited = [], []
|
||||
state = {"gone": False}
|
||||
|
||||
async def wait():
|
||||
waited.append(True)
|
||||
state["gone"] = True
|
||||
|
||||
result = await process_lifecycle.escalate_async(
|
||||
lambda: state["gone"], sent.append, steps=_ladder(), wait=wait,
|
||||
)
|
||||
# No precheck: an awaited leader is reaped only after the first signal.
|
||||
assert sent == [signal.SIGTERM] and waited == [True]
|
||||
assert result.dead is True and result.escalated is False
|
||||
|
||||
|
||||
async def test_escalate_async_gives_the_reap_a_floor_even_at_zero_grace():
|
||||
seen = {}
|
||||
|
||||
async def wait():
|
||||
seen["ran"] = True
|
||||
|
||||
await process_lifecycle.escalate_async(
|
||||
lambda: False, lambda sig: None, steps=((signal.SIGTERM, 0.0),), wait=wait,
|
||||
)
|
||||
assert seen == {"ran": True}
|
||||
|
||||
|
||||
# ── Identity ────────────────────────────────────────────────────────────────
|
||||
def test_identity_from_record_needs_a_pid():
|
||||
assert process_lifecycle.ProcessIdentity.from_record({}) is None
|
||||
assert process_lifecycle.ProcessIdentity.from_record({"pid": "x"}) is None
|
||||
ident = process_lifecycle.ProcessIdentity.from_record(
|
||||
{"supervisor_pid": 42, "supervisor_token": "t", "pgid": "7"},
|
||||
pid_key="supervisor_pid", token_key="supervisor_token",
|
||||
)
|
||||
assert ident == process_lifecycle.ProcessIdentity(pid=42, start_token="t", pgid=7)
|
||||
|
||||
|
||||
def test_a_pid_without_a_token_is_never_owned_and_never_exited(monkeypatch):
|
||||
monkeypatch.setattr(process_ownership, "start_token", lambda pid: "live")
|
||||
ident = process_lifecycle.ProcessIdentity(pid=4242, start_token=None)
|
||||
assert ident.verdict() == process_ownership.UNVERIFIABLE
|
||||
assert ident.owned() is False and ident.exited() is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("current,exited", [(None, True), ("other", True), ("mine", False)])
|
||||
def test_exited_means_gone_or_recycled(monkeypatch, current, exited):
|
||||
monkeypatch.setattr(process_ownership, "start_token", lambda pid: current)
|
||||
monkeypatch.setattr(process_lifecycle, "is_zombie", lambda pid: False)
|
||||
ident = process_lifecycle.ProcessIdentity(pid=4242, start_token="mine")
|
||||
assert ident.exited() is exited
|
||||
|
||||
|
||||
@pytest.mark.skipif(not platform_compat.has_procfs(), reason="zombie state needs procfs")
|
||||
def test_an_unreaped_zombie_has_exited():
|
||||
proc = subprocess.Popen([sys.executable, "-c", "pass"])
|
||||
ident = process_lifecycle.ProcessIdentity.capture(proc.pid)
|
||||
try:
|
||||
for _ in range(250):
|
||||
if process_lifecycle.is_zombie(proc.pid):
|
||||
break
|
||||
time.sleep(0.02)
|
||||
assert ident.verdict() == process_ownership.OWNED # still in its slot
|
||||
assert ident.exited() is True
|
||||
finally:
|
||||
proc.wait(timeout=5)
|
||||
|
||||
|
||||
# ── Snapshot identities ─────────────────────────────────────────────────────
|
||||
def test_terminate_identities_never_signals_foreign_or_unverifiable(monkeypatch):
|
||||
tokens = {1: "now-someone-else", 2: None}
|
||||
|
||||
def start_token(pid):
|
||||
if pid == 3:
|
||||
raise process_ownership.InspectionUnavailable("stat")
|
||||
return tokens.get(pid)
|
||||
|
||||
monkeypatch.setattr(process_ownership, "start_token", start_token)
|
||||
monkeypatch.setattr(os, "kill", lambda *a: pytest.fail("signalled an unowned pid"))
|
||||
sweep = process_lifecycle.terminate_identities(
|
||||
[process_lifecycle.ProcessIdentity(1, "mine"),
|
||||
process_lifecycle.ProcessIdentity(2, "mine"),
|
||||
process_lifecycle.ProcessIdentity(3, "mine")],
|
||||
steps=_ladder(),
|
||||
)
|
||||
assert sweep.killed == () and sweep.survivors == ()
|
||||
assert sweep.unverified == (3,) and sweep.dead is False
|
||||
|
||||
|
||||
def test_terminate_identities_kills_in_order_and_reverifies_each_signal(monkeypatch):
|
||||
alive = {10: "a", 11: "b", 12: "c"}
|
||||
sent = []
|
||||
monkeypatch.setattr(process_ownership, "start_token", lambda pid: alive.get(pid))
|
||||
monkeypatch.setattr(process_lifecycle, "is_zombie", lambda pid: False)
|
||||
|
||||
def kill(pid, sig):
|
||||
sent.append((pid, sig))
|
||||
alive.pop(pid, None)
|
||||
|
||||
monkeypatch.setattr(os, "kill", kill)
|
||||
sweep = process_lifecycle.terminate_identities(
|
||||
[process_lifecycle.ProcessIdentity(pid, token) for pid, token in list(alive.items())],
|
||||
steps=((signal.SIGTERM, 0.05),),
|
||||
)
|
||||
assert [pid for pid, _ in sent] == [10, 11, 12]
|
||||
assert sweep.killed == (10, 11, 12) and sweep.dead
|
||||
|
||||
|
||||
@posix_only
|
||||
@pytest.mark.skipif(not platform_compat.has_procfs(), reason="real identity needs procfs")
|
||||
def test_terminate_identities_escalates_past_an_ignored_sigterm():
|
||||
proc = _spawn()
|
||||
try:
|
||||
ident = process_lifecycle.ProcessIdentity.capture(proc.pid)
|
||||
sweep = process_lifecycle.terminate_identities(
|
||||
[ident], steps=process_lifecycle.term_kill_steps(0.2))
|
||||
assert sweep.killed == (proc.pid,) and sweep.survivors == ()
|
||||
assert proc.wait(timeout=5) == -signal.SIGKILL
|
||||
finally:
|
||||
_cleanup(proc)
|
||||
|
||||
|
||||
# ── terminate_tree / kill_process_tree ──────────────────────────────────────
|
||||
@posix_only
|
||||
@pytest.mark.skipif(not platform_compat.has_procfs(), reason="real identity needs procfs")
|
||||
def test_terminate_tree_requires_a_matching_identity():
|
||||
proc = _spawn()
|
||||
try:
|
||||
refused = process_lifecycle.terminate_tree(
|
||||
proc.pid, pgid=proc.pid, start_token="procfs:not-this-process",
|
||||
require_identity=True, grace_s=0.1)
|
||||
assert refused.dead is False and refused.ownership == process_ownership.FOREIGN
|
||||
assert refused.survivors == ()
|
||||
assert proc.poll() is None
|
||||
|
||||
token = process_ownership.start_token(proc.pid)
|
||||
outcome = process_lifecycle.terminate_tree(
|
||||
proc.pid, pgid=proc.pid, start_token=token, require_identity=True, grace_s=0.2)
|
||||
assert outcome.dead is True and outcome.escalated is True
|
||||
finally:
|
||||
_cleanup(proc)
|
||||
|
||||
|
||||
@posix_only
|
||||
def test_terminate_tree_keeps_a_leaderless_group_visible(monkeypatch):
|
||||
monkeypatch.setattr(process_ownership, "start_token", lambda pid: None)
|
||||
monkeypatch.setattr(process_lifecycle, "group_present", lambda pgid, **kw: True)
|
||||
monkeypatch.setattr(process_lifecycle, "signal_group",
|
||||
lambda *a, **kw: pytest.fail("signalled an unproven group"))
|
||||
outcome = process_lifecycle.terminate_tree(4242, pgid=4242, start_token="t",
|
||||
require_identity=True)
|
||||
assert outcome.dead is False and outcome.survivors == (4242,)
|
||||
assert outcome.ownership == process_ownership.GONE
|
||||
|
||||
|
||||
@posix_only
|
||||
def test_kill_process_tree_writes_no_containment_record(monkeypatch):
|
||||
from src import containment
|
||||
|
||||
monkeypatch.setattr(containment, "_update_record", lambda *a, **k: pytest.fail("grant store touched"))
|
||||
proc = _spawn("import time\nprint('ready', flush=True)\ntime.sleep(60)\n")
|
||||
try:
|
||||
outcome = platform_compat.kill_process_tree(proc.pid)
|
||||
assert outcome.dead is True
|
||||
assert isinstance(outcome, containment.ReleaseOutcome)
|
||||
finally:
|
||||
_cleanup(proc)
|
||||
|
||||
|
||||
def test_termination_outcome_shape_is_the_teardown_block():
|
||||
outcome = process_lifecycle.TerminationOutcome(dead=False, escalated=True, survivors=(1,),
|
||||
mechanism="process_group", ownership="owned")
|
||||
assert outcome.to_dict() == {"dead": False, "escalated": True, "survivors": [1],
|
||||
"mechanism": "process_group", "ownership": "owned"}
|
||||
|
||||
|
||||
# ── Identity-bound observation ──────────────────────────────────────────────
|
||||
def _tokens(monkeypatch, sequence):
|
||||
calls = iter(sequence)
|
||||
|
||||
def start_token(pid):
|
||||
value = next(calls)
|
||||
if isinstance(value, Exception):
|
||||
raise value
|
||||
return value
|
||||
|
||||
monkeypatch.setattr(process_ownership, "start_token", start_token)
|
||||
|
||||
|
||||
def test_observe_binds_facts_to_the_token_read_around_them(monkeypatch):
|
||||
_tokens(monkeypatch, ["t1", "t1"])
|
||||
seen = process_lifecycle.observe(42, lambda pid: "facts")
|
||||
assert seen.identity == process_lifecycle.ProcessIdentity(42, "t1") and seen.facts == "facts"
|
||||
|
||||
|
||||
def test_observe_drops_facts_read_across_a_pid_reuse(monkeypatch):
|
||||
_tokens(monkeypatch, ["old", "new"])
|
||||
assert process_lifecycle.observe(42, lambda pid: "whose facts?") is None
|
||||
|
||||
|
||||
def test_observe_of_a_gone_pid_reads_nothing(monkeypatch):
|
||||
_tokens(monkeypatch, [None])
|
||||
assert process_lifecycle.observe(42, lambda pid: pytest.fail("read a gone pid")) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sequence", [
|
||||
[process_ownership.InspectionUnavailable("stat")],
|
||||
["t1", process_ownership.InspectionUnavailable("stat")],
|
||||
])
|
||||
def test_observe_keeps_unidentifiable_facts_without_a_token(monkeypatch, sequence):
|
||||
_tokens(monkeypatch, sequence)
|
||||
seen = process_lifecycle.observe(42, lambda pid: "facts")
|
||||
assert seen.identity.start_token is None and seen.facts == "facts"
|
||||
assert seen.identity.verdict() == process_ownership.UNVERIFIABLE
|
||||
|
||||
|
||||
def test_bind_descendants_drops_a_pid_reparented_between_snapshots(monkeypatch):
|
||||
Info = process_ownership.ProcessInfo
|
||||
tables = iter([
|
||||
{10: Info(10, 1, "shell"), 11: Info(11, 10, "server")},
|
||||
{10: Info(10, 1, "shell"), 11: Info(11, 1, "stranger")},
|
||||
])
|
||||
monkeypatch.setattr(process_ownership, "process_table", lambda: next(tables))
|
||||
monkeypatch.setattr(process_ownership, "start_token", lambda pid: f"t{pid}")
|
||||
bound = process_lifecycle.bind_descendants([10])
|
||||
assert [seen.identity.pid for seen in bound] == [10]
|
||||
|
||||
|
||||
def test_bind_descendants_drops_a_pid_whose_token_changed(monkeypatch):
|
||||
Info = process_ownership.ProcessInfo
|
||||
table = {10: Info(10, 1, "shell"), 11: Info(11, 10, "server")}
|
||||
monkeypatch.setattr(process_ownership, "process_table", lambda: dict(table))
|
||||
monkeypatch.setattr(process_ownership, "start_token", lambda pid: f"t{pid}")
|
||||
monkeypatch.setattr(process_ownership, "verify",
|
||||
lambda pid, token: process_ownership.FOREIGN if pid == 11 else process_ownership.OWNED)
|
||||
bound = process_lifecycle.bind_descendants([10], exclude={99})
|
||||
assert [(seen.identity.pid, seen.facts.command) for seen in bound] == [(10, "shell")]
|
||||
@@ -338,6 +338,25 @@ async def test_generate_pty_timeout_says_so_when_the_session_survives(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="POSIX process groups")
|
||||
async def test_terminate_pty_session_never_signals_the_servers_own_group(monkeypatch):
|
||||
"""If setsid did not apply, the child's group is ours: reach the child alone."""
|
||||
import routes.shell_routes as shell_routes
|
||||
|
||||
own = os.getpgid(0)
|
||||
sent = []
|
||||
monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
|
||||
monkeypatch.setattr(shell_routes.process_lifecycle, "pgid_of", lambda _pid: own)
|
||||
monkeypatch.setattr(os, "killpg", lambda pgid, sig: sent.append(("group", pgid, sig)))
|
||||
monkeypatch.setattr(os, "kill", lambda pid, sig: sent.append(("pid", pid, sig)))
|
||||
|
||||
proc = SimpleNamespace(pid=987654, returncode=0, wait=None)
|
||||
assert shell_routes._session_pgid(proc.pid) is None
|
||||
await shell_routes._terminate_pty_session(proc)
|
||||
|
||||
assert sent and all(kind == "pid" and target == 987654 for kind, target, _ in sent), sent
|
||||
|
||||
|
||||
def test_session_alive_treats_a_refused_probe_as_alive(monkeypatch):
|
||||
"""EPERM says the group exists but we may not signal it, not that it died.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user