settled-reach/tooling/core/process.py

"""Detached execution: spawn a child that outlives its parent (D-263).

Substrate, not a domain — this has no verbs of its own. The verbs (`list`,
`status`, `log`, `wait`) have logic and state and are therefore
`reach jobs …`, which is the `core/` bound doing its job.

**Three things here are easy to get subtly wrong, and each has a comment where
it is handled rather than only here:**

1. *The child must genuinely outlive the parent.* `start_new_session=True` puts
   it in its own session and process group, so a signal to the parent's group —
   or the parent simply being killed on a timeout — does not take the work with
   it. A background shell job would not survive that, which is the whole reason
   detach exists.
2. *The child re-execs `reach` by BARE NAME.* Never an interpreter path, never
   `.venv/bin/reach` (T-1261). An absolute path would freeze the child to
   whichever checkout was current at spawn time, so after `make reach-repoint`
   a detached job would silently run the wrong source with no error anywhere —
   exactly the failure that command exists to fix.
3. *The exit code is recorded by the CHILD as its last act.* Not polled by a
   parent that has already returned. A parent cannot observe an exit it is no
   longer around for, and a runner that loses the failure is the exit-0 trap
   from D-263 relocated somewhere nothing is watching.

Streams stay separated exactly as they are in the foreground: the event stream
to `<id>.jsonl`, the command's real output to `<id>.out`. Merging them would
make the log unparseable for the sake of one fewer file.
"""

from __future__ import annotations

import json
import os
import subprocess
import sys
import time
from pathlib import Path
from typing import Any

from tooling.core import config, jobs
from tooling.core.errors import ReachError

# Under .cache/, which is gitignored and already the repo's scratch space — so a
# wrong answer about retention costs disk, never data.
JOBS_SUBPATH = (".cache", "reach", "jobs")


def jobs_dir() -> Path:
    path = config.path(*JOBS_SUBPATH)
    path.mkdir(parents=True, exist_ok=True)
    return path


def log_path(job_id: str) -> Path:
    return jobs_dir() / f"{job_id}.jsonl"


def output_path(job_id: str) -> Path:
    return jobs_dir() / f"{job_id}.out"


def meta_path(job_id: str) -> Path:
    return jobs_dir() / f"{job_id}.json"


class ProcessTimeout(ReachError):
    """A run that exceeded its `timeout=`.

    Its own class because a timeout is not a failure verdict — a caller that
    treats "the tool ran and said no" and "the tool never answered" alike will
    turn a slow machine into a rejection. Uncaught it still reads as a
    `ReachError` with a remedy.
    """


def run(
    argv: list[str],
    *,
    cwd: Path | None = None,
    env: dict[str, str] | None = None,
    capture: bool = True,
    check: bool = True,
    input: str | None = None,
    timeout: float | None = None,
    fix: str | None = None,
    missing_fix: str | None = None,
) -> subprocess.CompletedProcess[str]:
    """Run an external program, guarded. The one sanctioned `exec` in reach.

    "Rewrite the bash in Python" does not mean reimplementing the operating
    system (D-263). A guarded exec is right for `git`, `godot`, `rustup`,
    `curl`; what must be Python is the *logic* around it — which version is
    wanted, whether it is already there, what the output means. The test of a
    good port is whether the decisions can be exercised without performing them.

    Every guard lives here rather than at each call site, because a per-domain
    `subprocess.run` is exactly where one of them quietly goes missing:

    - **An argv list, never a shell string.** `shell=False` always, so a
      filename containing a space or a semicolon is an argument and not a
      command. Passing a string here is rejected outright rather than helpfully
      split, since the helpful split is the vulnerability.
    - **A non-zero exit becomes a `ReachError`** naming the command and carrying
      a remedy — not a `CalledProcessError` traceback at someone who wanted to
      know what to do next.
    - **A missing binary reports what to install.** `FileNotFoundError` names
      the path that was not found, which is the least useful half of the answer.
    """
    if isinstance(argv, str):  # type: ignore[unreachable]
        raise ReachError(
            "process.run was given a string, not an argument list",
            fix='pass a list — ["git", "status"] — so nothing goes through a shell',
        )

    try:
        result = subprocess.run(
            argv,
            cwd=cwd,
            env=env,
            capture_output=capture,
            text=True,
            shell=False,
            input=input,
            timeout=timeout,
        )
    except FileNotFoundError as exc:
        raise ReachError(
            f"{argv[0]} is not installed or not on PATH",
            fix=missing_fix or f"install {argv[0]}, or check PATH in a non-interactive shell",
        ) from exc
    except subprocess.TimeoutExpired as exc:
        raise ProcessTimeout(
            f"{argv[0]} did not finish within {timeout:g}s",
            fix=f"re-run, or raise the timeout — `{' '.join(argv)}` was killed, "
            "which says nothing about whether it would have succeeded",
        ) from exc

    if check and result.returncode != 0:
        detail = (result.stderr or result.stdout or "").strip()
        tail = f"\n{detail}" if detail else ""
        raise ReachError(
            f"{' '.join(argv)} exited {result.returncode}{tail}",
            fix=fix or f"run `{' '.join(argv)}` directly to see the full output",
            exit_code=result.returncode,
        )
    return result


def cargo_binary(name: str, *args: str, capture: bool = True) -> str:
    """Run a Rust binary from `server/`, building it first if it is stale.

    Three scripts wrote this by hand — `tooling/atlas`, `generate-brands`,
    `generate-corporations` — each ~24 lines of identical bash: check for the
    debug binary, `cargo build --bin` it if missing, `exec` it with every
    argument. Three copies of one idea is a substrate, so it lives here.

    It ALWAYS asks cargo to build, and lets cargo decide whether anything is
    stale. The copies all built only when the binary was MISSING, so an edit
    to the Rust source ran the old binary — and for generate_brands that meant
    `make regen-db` stamped the new source SHA onto output the old code
    produced, the one thing the stamp exists to prevent (found T-1253: a header
    change to main.rs did not reach generated_brands.toml). An up-to-date build
    is incremental and fast; a real rebuild is announced, since ninety silent
    seconds reads as a hang.
    """
    from tooling.core import console

    binary = config.path("server", "target", "debug", name)
    build = run(
        ["cargo", "build", "--bin", name],
        cwd=config.path("server"),
        check=False,
        missing_fix="install Rust — make setup-rust",
    )
    if build.returncode != 0:
        raise ReachError(
            f"could not build {name}\n{(build.stderr or '').strip()}",
            fix=f"cd server && cargo build --bin {name} — for the full error",
            exit_code=build.returncode,
        )
    if "Compiling" in (build.stderr or ""):
        console.event(f"rebuilt the {name} binary — its source changed since the last build", level="warn")

    result = run([str(binary), *args], check=False, capture=capture)
    if result.returncode != 0:
        raise ReachError(
            f"{name} {' '.join(args)} exited {result.returncode}\n"
            + (result.stderr or "").strip(),
            fix=f"run `{binary} {' '.join(args)}` directly for the full output",
            exit_code=result.returncode,
        )
    return result.stdout or ""


def spawn_detached(argv: list[str]) -> str:
    """Run `reach <argv>` in a detached child. Returns the job id immediately.

    The caller is expected to report the id and exit — it must NOT wait, since
    not waiting is the entire point.
    """
    job_id = jobs.new_id()
    directory = jobs_dir()

    # Opened here and inherited by the child, which then owns them. The parent
    # closes its copies below; the child keeps writing after the parent is gone.
    log_file = open(directory / f"{job_id}.jsonl", "wb")
    out_file = open(directory / f"{job_id}.out", "wb")

    child_env = {
        **os.environ,
        jobs.ENV_JOB_ID: job_id,
        # Force machine format: the child's stderr is a file, so isatty would
        # already say JSONL — but being explicit means a future TTY-inheriting
        # spawn cannot silently start writing prose into a log meant to be read
        # back as events.
        "SR_OUTPUT_FORMAT": "json",
    }

    try:
        process = subprocess.Popen(
            ["reach", *argv],  # BARE NAME — see note 2 in the module docstring
            stdout=out_file,
            stderr=log_file,
            stdin=subprocess.DEVNULL,
            start_new_session=True,  # note 1: its own session, survives the parent
            env=child_env,
            cwd=config.repo_root(),
        )
    finally:
        log_file.close()
        out_file.close()

    _write_meta(
        job_id,
        {
            "job": job_id,
            "argv": argv,
            "command": " ".join(["reach", *argv]),
            "pid": process.pid,
            "started_at": _now(),
            "status": "running",
        },
    )

    # At write time, not on a schedule. A retention pass that depends on someone
    # remembering to run it is one that silently never happens.
    prune()
    return job_id


# Measured 2026-08-31: a chatty short job (10 progress events) writes ~1.8 KB
# across its three files. A generator emitting per-body progress might reach
# 100 KB. 100 jobs is therefore single-digit megabytes at worst, inside .cache/
# which is gitignored scratch — so the failure mode of this number being wrong
# is disk, never data. Raise it freely if a real batch proves it tight.
DEFAULT_KEEP = 100

ENV_KEEP = "SR_JOB_KEEP"


def prune(keep: int | None = None) -> list[str]:
    """Delete all but the `keep` most recent jobs. Returns the ids removed.

    **A genuinely running job is never pruned** — but note what that has to
    mean. Skipping anything whose *file* says "running" would make every
    crashed job immortal, because a process killed outright never gets to
    update its own status. Those are precisely the jobs that accumulate, so the
    check is against the process table, not the record.

    Called at spawn time rather than by a sweep: a cleanup nothing invokes is a
    cleanup that does not happen.
    """
    if keep is None:
        raw = os.environ.get(ENV_KEEP)
        keep = int(raw) if raw and raw.isdigit() else DEFAULT_KEEP

    directory = jobs_dir()
    # Ids sort chronologically because they are timestamp-first (T-1276).
    ids = sorted((path.stem for path in directory.glob("*.json")), reverse=True)

    removed: list[str] = []
    for job_id in ids[keep:]:
        meta = read_meta(job_id)
        if meta and meta.get("status") == "running" and is_alive(int(meta.get("pid", -1))):
            continue  # actually running — a long generator may outlive the window
        for path in (log_path(job_id), output_path(job_id), meta_path(job_id)):
            path.unlink(missing_ok=True)
        removed.append(job_id)
    return removed


def finish_if_detached(exit_code: int) -> None:
    """Record completion — called by the CHILD, from the outermost decorator.

    A no-op in a foreground run, which has no metadata file to update. Note 3
    in the module docstring is why this lives on the child's exit path rather
    than in whatever spawned it.
    """
    job_id = os.environ.get(jobs.ENV_JOB_ID)
    if not job_id:
        return
    meta = read_meta(job_id)
    if meta is None:
        return
    meta.update(
        {
            "status": "done" if exit_code == 0 else "failed",
            "exit_code": exit_code,
            "ended_at": _now(),
        }
    )
    _write_meta(job_id, meta)


def read_meta(job_id: str) -> dict[str, Any] | None:
    path = meta_path(job_id)
    if not path.is_file():
        return None
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError:
        return None


def is_alive(pid: int) -> bool:
    """Whether a recorded pid is still running.

    Needed because a child killed outright — SIGKILL, OOM, a crash in the
    interpreter itself — never gets to record its own completion, and its
    metadata would otherwise say "running" forever. Reconciling against the
    process table is what stops a dead job from looking like a busy one.
    """
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        return True  # exists, owned by someone else
    return True


def _write_meta(job_id: str, meta: dict[str, Any]) -> None:
    # Written via a temporary file and renamed, because `jobs list` may read
    # this at any moment and a half-written JSON file is an unreadable job.
    target = meta_path(job_id)
    temporary = target.with_suffix(".json.tmp")
    temporary.write_text(json.dumps(meta, indent=2), encoding="utf-8")
    temporary.replace(target)


def _now() -> str:
    return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())


def current_argv() -> list[str]:
    """The invocation's arguments with `--detach` removed.

    Removed because the child must not detach again — it would fork forever,
    each generation spawning another and none doing the work.
    """
    return [arg for arg in sys.argv[1:] if arg != "--detach"]