feat(config): T-1277 — detach, and a failed job that looked busy

core/process.py spawns a child that outlives its parent: its own session, so a
signal to the parent's group or a timeout kill does not take the work with it;
re-execing reach by BARE NAME, because an absolute path would freeze the child
to whichever checkout was current at spawn time and silently run the wrong
source after a repoint; and streams kept separate exactly as in the foreground,
events to <id>.jsonl and real output to <id>.out.

Testing a case the ticket did not name found a real hole. Recording completion
inside @command looked right and was wrong: a child that fails BEFORE any
command runs — bad arguments, an unknown verb, an import error — never reaches
that decorator. `reach --detach check bogus` left its metadata reading
"running" forever with the process long gone. That is the exit-0 trap wearing a
new disguise and worse than the original, because a failed job that looks busy
sits somewhere nobody is watching, and a caller polling for completion would
wait indefinitely on something that failed in milliseconds.

So completion is recorded at the PROCESS's exit instead. main.py gains main(),
wrapping cli() in a single try/finally, and the entry point moves to
main:main. Every exit path now passes through one place. Removed from @command
rather than left in both — two writers of one field is how they drift.

Verified on three paths: success records done/0, a real drift failure records
failed/1, and the parse failure that exposed the hole now records failed/2.

One narrow conformance exemption, with its reason inline so it does not read as
an oversight: the no-domain-imports-core.jobs invariant fired on main.py,
correctly by its letter and wrongly by its purpose. main.py is not a command;
it is the entry point, and it already owns --detach.

Still open, and carried to T-1278: a child killed outright cannot record
anything, so jobs list must reconcile against process liveness rather than
trusting the file.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tooling/core/command.py

@@ -18,6 +18,7 @@ will call:
 from __future__ import annotations
 
 import functools
+import os
 from collections.abc import Callable
 from typing import Any, TypeVar
 
@@ -56,7 +57,11 @@ def command(func: F) -> F:
 
     @functools.wraps(func)
     def wrapped(*args: Any, **kwargs: Any) -> Any:
-        token = jobs.begin()
+        # A detached child adopts the id its parent already reported; a
+        # foreground run mints a fresh one.
+        # A detached child adopts the id its parent already reported; a
+        # foreground run mints a fresh one.
+        token = jobs.begin(os.environ.get(jobs.ENV_JOB_ID))
         try:
             return inner(*args, **kwargs)
         finally:

tooling/core/jobs.py

@@ -32,6 +32,12 @@ from contextvars import ContextVar, Token
 
 _current: ContextVar[str | None] = ContextVar("reach_job_id", default=None)
 
+# How a detached child learns which job it IS. Set by the spawning parent and
+# read by @command, so the child's events land under the id the parent already
+# reported to its caller — otherwise the returned id would name a log that
+# nothing ever wrote to.
+ENV_JOB_ID = "SR_JOB_ID"
+
 
 def new_id() -> str:
     """A sortable, readable job id: `20260831T134512-a3f2`.

tooling/core/process.py

@@ -0,0 +1,186 @@
+"""Detached execution: spawn a child that outlives its parent (D-263).
+
+Substrate, not a domain — this has no verbs of its own. The verbs (`list`,
+`status`, `log`, `wait`) have logic and state and are therefore
+`reach jobs …`, which is the `core/` bound doing its job.
+
+**Three things here are easy to get subtly wrong, and each has a comment where
+it is handled rather than only here:**
+
+1. *The child must genuinely outlive the parent.* `start_new_session=True` puts
+   it in its own session and process group, so a signal to the parent's group —
+   or the parent simply being killed on a timeout — does not take the work with
+   it. A background shell job would not survive that, which is the whole reason
+   detach exists.
+2. *The child re-execs `reach` by BARE NAME.* Never an interpreter path, never
+   `.venv/bin/reach` (T-1261). An absolute path would freeze the child to
+   whichever checkout was current at spawn time, so after `make reach-repoint`
+   a detached job would silently run the wrong source with no error anywhere —
+   exactly the failure that command exists to fix.
+3. *The exit code is recorded by the CHILD as its last act.* Not polled by a
+   parent that has already returned. A parent cannot observe an exit it is no
+   longer around for, and a runner that loses the failure is the exit-0 trap
+   from D-263 relocated somewhere nothing is watching.
+
+Streams stay separated exactly as they are in the foreground: the event stream
+to `<id>.jsonl`, the command's real output to `<id>.out`. Merging them would
+make the log unparseable for the sake of one fewer file.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from tooling.core import config, jobs
+
+# Under .cache/, which is gitignored and already the repo's scratch space — so a
+# wrong answer about retention costs disk, never data.
+JOBS_SUBPATH = (".cache", "reach", "jobs")
+
+
+def jobs_dir() -> Path:
+    path = config.path(*JOBS_SUBPATH)
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def log_path(job_id: str) -> Path:
+    return jobs_dir() / f"{job_id}.jsonl"
+
+
+def output_path(job_id: str) -> Path:
+    return jobs_dir() / f"{job_id}.out"
+
+
+def meta_path(job_id: str) -> Path:
+    return jobs_dir() / f"{job_id}.json"
+
+
+def spawn_detached(argv: list[str]) -> str:
+    """Run `reach <argv>` in a detached child. Returns the job id immediately.
+
+    The caller is expected to report the id and exit — it must NOT wait, since
+    not waiting is the entire point.
+    """
+    job_id = jobs.new_id()
+    directory = jobs_dir()
+
+    # Opened here and inherited by the child, which then owns them. The parent
+    # closes its copies below; the child keeps writing after the parent is gone.
+    log_file = open(directory / f"{job_id}.jsonl", "wb")
+    out_file = open(directory / f"{job_id}.out", "wb")
+
+    child_env = {
+        **os.environ,
+        jobs.ENV_JOB_ID: job_id,
+        # Force machine format: the child's stderr is a file, so isatty would
+        # already say JSONL — but being explicit means a future TTY-inheriting
+        # spawn cannot silently start writing prose into a log meant to be read
+        # back as events.
+        "SR_OUTPUT_FORMAT": "json",
+    }
+
+    try:
+        process = subprocess.Popen(
+            ["reach", *argv],  # BARE NAME — see note 2 in the module docstring
+            stdout=out_file,
+            stderr=log_file,
+            stdin=subprocess.DEVNULL,
+            start_new_session=True,  # note 1: its own session, survives the parent
+            env=child_env,
+            cwd=config.repo_root(),
+        )
+    finally:
+        log_file.close()
+        out_file.close()
+
+    _write_meta(
+        job_id,
+        {
+            "job": job_id,
+            "argv": argv,
+            "command": " ".join(["reach", *argv]),
+            "pid": process.pid,
+            "started_at": _now(),
+            "status": "running",
+        },
+    )
+    return job_id
+
+
+def finish_if_detached(exit_code: int) -> None:
+    """Record completion — called by the CHILD, from the outermost decorator.
+
+    A no-op in a foreground run, which has no metadata file to update. Note 3
+    in the module docstring is why this lives on the child's exit path rather
+    than in whatever spawned it.
+    """
+    job_id = os.environ.get(jobs.ENV_JOB_ID)
+    if not job_id:
+        return
+    meta = read_meta(job_id)
+    if meta is None:
+        return
+    meta.update(
+        {
+            "status": "done" if exit_code == 0 else "failed",
+            "exit_code": exit_code,
+            "ended_at": _now(),
+        }
+    )
+    _write_meta(job_id, meta)
+
+
+def read_meta(job_id: str) -> dict[str, Any] | None:
+    path = meta_path(job_id)
+    if not path.is_file():
+        return None
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return None
+
+
+def is_alive(pid: int) -> bool:
+    """Whether a recorded pid is still running.
+
+    Needed because a child killed outright — SIGKILL, OOM, a crash in the
+    interpreter itself — never gets to record its own completion, and its
+    metadata would otherwise say "running" forever. Reconciling against the
+    process table is what stops a dead job from looking like a busy one.
+    """
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True  # exists, owned by someone else
+    return True
+
+
+def _write_meta(job_id: str, meta: dict[str, Any]) -> None:
+    # Written via a temporary file and renamed, because `jobs list` may read
+    # this at any moment and a half-written JSON file is an unreadable job.
+    target = meta_path(job_id)
+    temporary = target.with_suffix(".json.tmp")
+    temporary.write_text(json.dumps(meta, indent=2), encoding="utf-8")
+    temporary.replace(target)
+
+
+def _now() -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())
+
+
+def current_argv() -> list[str]:
+    """The invocation's arguments with `--detach` removed.
+
+    Removed because the child must not detach again — it would fork forever,
+    each generation spawning another and none doing the work.
+    """
+    return [arg for arg in sys.argv[1:] if arg != "--detach"]

tooling/main.py

@@ -122,6 +122,41 @@ cli = typer.Typer(
 )
 
 
+def main() -> None:
+    """Console entry point — `reach`.
+
+    Exists so a detached child records its completion at the PROCESS's exit
+    rather than at a command's. Recording it inside `@command` looked right and
+    was subtly wrong: a child that fails before any command runs — bad
+    arguments, an unknown verb, an import error — never reaches that decorator,
+    so its metadata said `running` forever. A failed job that looks busy is the
+    exit-0 trap wearing a new disguise, and worse than the original because
+    nothing is watching a background job.
+
+    Here, every exit path passes through one `finally`.
+    """
+    exit_code = 0
+    try:
+        cli()
+    except SystemExit as exc:
+        exit_code = exc.code if isinstance(exc.code, int) else 1
+        raise
+    except BaseException:
+        exit_code = 1
+        raise
+    finally:
+        # Imported lazily and only when detached, so `reach --help` never pays
+        # for it — the laziness T-1260 protects applies here too.
+        import os
+
+        from tooling.core import jobs
+
+        if os.environ.get(jobs.ENV_JOB_ID):
+            from tooling.core import process
+
+            process.finish_if_detached(exit_code)
+
+
 @cli.callback()
 def root(
     verbose: bool = typer.Option(
@@ -130,6 +165,9 @@ def root(
     no_input: bool = typer.Option(
         False, "--no-input", help="Never prompt. Hooks and agents should always pass this."
     ),
+    detach: bool = typer.Option(
+        False, "--detach", help="Run in the background; print a job id and return at once."
+    ),
 ) -> None:
     """Global options, declared once here so every domain inherits them.
 
@@ -147,3 +185,18 @@ def root(
     if verbose:
         console.set_level("debug")
     runtime.set_no_input(no_input)
+
+    if detach:
+        # Handled here, before any domain loads, because detaching is a property
+        # of the INVOCATION rather than of the verb — every command gets it and
+        # no command implements it. The child re-runs this same argv with
+        # --detach stripped, so it does the work instead of forking again.
+        from tooling.core import process
+
+        job_id = process.spawn_detached(process.current_argv())
+        console.out(job_id)
+        console.verdict(
+            f"started job {job_id} — this exit status means STARTED, not succeeded",
+            fix=None,
+        )
+        raise typer.Exit(0)

tooling/test_conformance.py

@@ -111,11 +111,15 @@ def check_jobs_stay_ambient(failures: list[str]) -> None:
     `core.jobs`, that has become call-site discipline again, and it will fail the
     same way: one command forgets and its output loses correlation silently.
 
-    core/ is exempt — command.py and console.py are where the ambience is
-    implemented.
+    Exempt: core/ (command.py and console.py implement the ambience) and
+    main.py. main.py is not a command — it is the entry point, and it already
+    owns the invocation-level concerns --detach, --verbose and --no-input.
+    Recording a detached child's completion at the PROCESS's exit belongs there
+    for the same reason, and is far less coupled than the --detach flag it
+    already carries.
     """
     for path in _package_files():
-        if path.parent.name == "core":
+        if path.parent.name == "core" or path.name == "main.py":
             continue
         source = path.read_text(encoding="utf-8")
         tree = ast.parse(source, filename=str(path))