fix(shell): kill the PTY command's whole session on timeout

/api/shell/stream starts its PTY child under os.setsid, so the child
leads its own session and process group. The timeout, client-disconnect
and error paths all called proc.kill(), which signals only the group
leader. Creating a group and then signalling only its leader is strictly
worse than never creating one: the descendants are detached from the
server's group as well, so nothing else will ever reach them, while the
route reports "Command timed out after Ns" and exit_code -1 as if the
command were gone.

The kernel's controlling-terminal SIGHUP hid this for well-behaved
children, which is why it reads as working. Anything that ignores
SIGHUP — a nohup'ed job, a daemon, a process that means to outlive its
terminal — survives the kill indefinitely.

Signal the whole group instead, escalate to SIGKILL if it outlives the
grace period, and confirm it is actually gone. The timeout response now
says so when containment could not be established rather than claiming
a clean kill it did not get.
This commit is contained in:
Léo
2026-10-01 18:43:28 +02:00
parent 05c6cfcfa1
commit f49e09e59a
2 changed files with 364 additions and 10 deletions
+229
View File
@@ -1,16 +1,20 @@
"""Tests for shell_routes.py helpers."""
import asyncio
import builtins
import importlib
import importlib.util
import json
import os
import shlex
import signal
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from core.platform_compat import pid_alive
from routes.shell_routes import (
_find_line_break,
_host_docker_access_enabled,
@@ -85,6 +89,231 @@ async def test_generate_pty_reports_explicit_unsupported_error(monkeypatch):
]
pty_session = pytest.mark.skipif(
not hasattr(os, "setsid"), reason="process sessions are POSIX-only"
)
async def _spawn_pty_style_session(script: str):
"""Spawn `script` the way _generate_pty does: its own session via setsid."""
return await asyncio.create_subprocess_shell(
script,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
preexec_fn=os.setsid,
)
def _stubborn_child(pid_file: Path, ignore: tuple[str, ...]) -> str:
"""Shell snippet that starts a child ignoring `ignore`, then waits for it.
Killing a PTY session leader makes the kernel send SIGHUP to the
terminal's foreground process group, so a plain `sleep` child looks
contained even when nothing ever signalled the group. A child that ignores
SIGHUP is what an admin actually runs into — a `nohup`ed job, a daemon,
anything meant to outlive its terminal.
The child publishes its own pid only after installing the handlers, and
the snippet blocks until it does, so a test can never signal it while it
is still starting up and read that as teardown having worked.
"""
ignores = "".join(
f"signal.signal(signal.{name}, signal.SIG_IGN); " for name in ignore
)
script = (
f"import os, signal, time; {ignores}"
f"open({str(pid_file)!r}, 'w').write(str(os.getpid())); "
"time.sleep(120)"
)
return (
f"{sys.executable} -c {shlex.quote(script)} & "
f"while [ ! -s {pid_file} ]; do sleep 0.02; done"
)
async def _never_disconnected() -> bool:
return False
def _reap_if_alive(pid: int) -> None:
"""Clean up a descendant the code under test was supposed to have killed."""
if pid_alive(pid):
try:
os.kill(pid, signal.SIGKILL)
except OSError:
pass
async def _read_pid(path: Path, timeout: float = 5.0) -> int:
"""Wait for a child to publish its pid, then return it."""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
while loop.time() < deadline:
if path.exists():
text = path.read_text().strip()
if text:
return int(text)
await asyncio.sleep(0.01)
raise AssertionError(f"child never wrote its pid to {path}")
@pty_session
async def test_terminate_pty_session_kills_descendants(tmp_path):
"""Tearing down a PTY command takes its children, not only the shell."""
import routes.shell_routes as shell_routes
pid_file = tmp_path / "child.pid"
proc = await _spawn_pty_style_session(
f"sleep 120 & echo $! > {pid_file}; sleep 120"
)
try:
child_pid = await _read_pid(pid_file)
assert pid_alive(child_pid)
assert await shell_routes._terminate_pty_session(proc) is True
assert proc.returncode is not None
assert not pid_alive(child_pid)
finally:
await shell_routes._terminate_pty_session(proc)
@pty_session
async def test_terminate_pty_session_escalates_past_ignored_sigterm(tmp_path):
"""A child that ignores SIGTERM is still gone when teardown returns."""
import routes.shell_routes as shell_routes
pid_file = tmp_path / "child.pid"
child = _stubborn_child(pid_file, ("SIGHUP", "SIGTERM"))
proc = await _spawn_pty_style_session(f"{child}; sleep 120")
try:
child_pid = await _read_pid(pid_file)
assert pid_alive(child_pid)
assert await shell_routes._terminate_pty_session(proc) is True
assert not pid_alive(child_pid)
finally:
await shell_routes._terminate_pty_session(proc)
@pty_session
async def test_generate_pty_timeout_kills_the_whole_session(tmp_path):
"""A timed-out PTY command leaves none of its children running."""
import routes.shell_routes as shell_routes
pid_file = tmp_path / "child.pid"
child = _stubborn_child(pid_file, ("SIGHUP",))
cmd = f"{child}; echo ready; sleep 120"
request = SimpleNamespace(is_disconnected=_never_disconnected)
events = [
json.loads(chunk.removeprefix("data: ").strip())
async for chunk in shell_routes._generate_pty(cmd, 1, request)
]
child_pid = await _read_pid(pid_file)
try:
assert events[-1] == {"exit_code": -1}
assert events[-2]["data"].startswith("Command timed out after 1s")
assert not pid_alive(child_pid)
finally:
_reap_if_alive(child_pid)
@pty_session
async def test_generate_pty_disconnect_kills_the_whole_session(tmp_path):
"""Abandoning the stream kills the command's children too."""
import routes.shell_routes as shell_routes
pid_file = tmp_path / "child.pid"
child = _stubborn_child(pid_file, ("SIGHUP",))
cmd = f"{child}; echo ready; sleep 120"
polls = []
async def disconnect_after_first_poll() -> bool:
polls.append(None)
return len(polls) > 1
request = SimpleNamespace(is_disconnected=disconnect_after_first_poll)
async for _ in shell_routes._generate_pty(cmd, 0, request):
pass
child_pid = await _read_pid(pid_file)
try:
assert not pid_alive(child_pid)
finally:
_reap_if_alive(child_pid)
async def test_terminate_pty_session_reports_a_session_it_could_not_kill(
monkeypatch,
):
"""Teardown returns False rather than claiming a surviving session died."""
import routes.shell_routes as shell_routes
monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
monkeypatch.setattr(shell_routes, "_signal_session", lambda *_: True)
monkeypatch.setattr(shell_routes, "_session_alive", lambda *_: True)
monkeypatch.setattr(shell_routes, "_session_pgid", lambda _: 4242)
proc = SimpleNamespace(pid=4242, returncode=0, wait=None)
assert await shell_routes._terminate_pty_session(proc) is False
async def test_terminate_pty_session_escalates_before_giving_up(monkeypatch):
"""SIGTERM then SIGKILL — the group is never signalled only once."""
import routes.shell_routes as shell_routes
sent = []
monkeypatch.setattr(shell_routes, "PTY_KILL_GRACE", 0.01)
monkeypatch.setattr(shell_routes, "_session_pgid", lambda _: 4242)
monkeypatch.setattr(shell_routes, "_session_alive", lambda *_: True)
monkeypatch.setattr(
shell_routes,
"_signal_session",
lambda pgid, pid, sig: sent.append(sig) or True,
)
proc = SimpleNamespace(pid=4242, returncode=0, wait=None)
await shell_routes._terminate_pty_session(proc)
assert sent == [signal.SIGTERM, signal.SIGKILL]
@pty_session
async def test_generate_pty_timeout_says_so_when_the_session_survives(
monkeypatch,
):
"""A timed-out command no longer reports clean termination it didn't get."""
import routes.shell_routes as shell_routes
real_terminate = shell_routes._terminate_pty_session
async def terminate_but_report_failure(proc):
await real_terminate(proc)
return False
monkeypatch.setattr(
shell_routes, "_terminate_pty_session", terminate_but_report_failure
)
request = SimpleNamespace(is_disconnected=_never_disconnected)
events = [
json.loads(chunk.removeprefix("data: ").strip())
async for chunk in shell_routes._generate_pty("echo ready; sleep 30", 1, request)
]
assert events[-1] == {"exit_code": -1}
timed_out = events[-2]
assert timed_out["stream"] == "stderr"
assert timed_out["data"] == (
"Command timed out after 1s" + shell_routes.PTY_KILL_FAILED_HINT
)
class TestFindLineBreak:
"""Test line-break detection in byte buffers."""