From 6f08cc9156092c647f79b69a937f4663e3d1ebfb Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Mon, 31 Aug 2026 17:02:36 +0200 Subject: [PATCH] =?UTF-8?q?feat(config):=20T-1278=20=E2=80=94=20the=20jobs?= =?UTF-8?q?=20domain,=20and=20typer.Exit=20is=20not=20a=20SystemExit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reach jobs list / status / log --follow / wait. A domain rather than core/, because these verbs carry logic and state: they reconcile recorded status against process liveness, tail a file from an offset, and relay an exit code. Found a latent bug in already-committed code before building on it. typer.Exit is a RuntimeError, not a SystemExit, so @handle_errors caught it like any other unexpected exception: `raise typer.Exit(3)` inside a decorated command printed "unexpected Exit: 3" and exited 1, silently discarding the requested code. Nothing hit it because the check router had been converted to ReachError — but jobs wait needs exactly this and it is what anyone would naturally write. Added core/errors.ReachExit as the sanctioned control-flow exit, passed straight through with no verdict. ReachError would have been wrong twice: a failure verdict for a command that worked, and a demand for a fix= where there is no remedy. Reconciliation proved out on a real corpse rather than a simulated one — the job stranded by the T-1277 bug, status "running" with its process long gone, now reports as died. DIED is derived, never recorded, because a process killed outright cannot write its own ending. It relays 137, never 0: a died job has no exit code of its own and borrowing success points the exit-0 trap straight at whatever gated on the run. Second UTC bug of the same family as T-1276's: jobs list reported a job started minutes earlier as running for 133m, because _parse used mktime on a UTC stamp and silently added the offset to every duration. console.render() is public now, so jobs log replays stored events through the same path a live run prints them — a second renderer would drift, and the divergence would surface exactly when someone is reading a log to find out what went wrong. test_jobs.py closes the gap T-1257 named: D-263 claims services are callable without a CLI round trip, and nothing had ever demonstrated it, which left the layering as unverified decoration. Every test here calls the service directly. Not yet exercised, and said plainly: log --follow against a genuinely long-running job. Nothing in reach runs long enough to tail yet. The offset mechanics underneath are tested; the live loop waits for a slow domain. Co-Authored-By: Claude Opus 5 (1M context) --- .pql/changelog/ticket_history/2026-08.sql | 28 +++++ .pql/changelog/tickets/2026-08.sql | 51 ++++++++ Makefile | 4 + tooling/core/console.py | 12 ++ tooling/core/errors.py | 24 ++++ tooling/domains/jobs/__init__.py | 12 ++ tooling/domains/jobs/router.py | 116 +++++++++++++++++ tooling/domains/jobs/schemas.py | 55 ++++++++ tooling/domains/jobs/service.py | 146 ++++++++++++++++++++++ tooling/main.py | 4 + tooling/test_jobs.py | 130 +++++++++++++++++++ 11 files changed, 582 insertions(+) create mode 100644 tooling/domains/jobs/__init__.py create mode 100644 tooling/domains/jobs/router.py create mode 100644 tooling/domains/jobs/schemas.py create mode 100644 tooling/domains/jobs/service.py create mode 100644 tooling/test_jobs.py diff --git a/.pql/changelog/ticket_history/2026-08.sql b/.pql/changelog/ticket_history/2026-08.sql index c84db72fe..d04f4bd96 100644 --- a/.pql/changelog/ticket_history/2026-08.sql +++ b/.pql/changelog/ticket_history/2026-08.sql @@ -1933,3 +1933,31 @@ reach --help is still 73 ms, so the entry-point wrapper costs nothing on the fas INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5VJ89C7Q4EPR6Q73FSS74', 'description', 'The user-facing verbs, and a DOMAIN rather than core/ because they carry logic and state of their own — the first real test of the D-263 core bound, which it passes. Deliverables: reach jobs list (recent jobs with status, command and duration), status , log with --follow to tail, and wait . REATTACH IS A BYTE OFFSET into an append-only file, which is the entire reason no daemon is needed: a caller can attach, drop off, and come back without losing anything, and there is no lifecycle to get wrong, nothing to orphan, and no stale state to reconcile. log --follow is therefore a poll on file length, not a subscription. Render the JSONL through the same path a live terminal uses, so a tailed log and a live run are the same artefact in two presentations rather than two renderers that drift. Note for the port: this domain is the first one written from scratch under the full contract rather than ported, so it doubles as the worked example the reach skill (T-1254) should show.', 'The user-facing verbs, and a DOMAIN rather than core/ because they carry logic and state of their own — the first real test of the D-263 core bound, which it passes. Deliverables: reach jobs list (recent jobs with status, command and duration), status , log with --follow to tail, and wait . REATTACH IS A BYTE OFFSET into an append-only file, which is the entire reason no daemon is needed: a caller can attach, drop off, and come back without losing anything, and there is no lifecycle to get wrong, nothing to orphan, and no stale state to reconcile. log --follow is therefore a poll on file length, not a subscription. Render the JSONL through the same path a live terminal uses, so a tailed log and a live run are the same artefact in two presentations rather than two renderers that drift. Note for the port: this domain is the first one written from scratch under the full contract rather than ported, so it doubles as the worked example the reach skill (T-1254) should show. FROM T-1277 (2026-08-31) — the reconciliation requirement is yours and it is not optional. A child killed outright (SIGKILL, OOM, an interpreter crash) never gets to record its own completion, so its metadata file stays status=running with the process long gone. jobs list and jobs status must therefore RECONCILE against process.is_alive(pid) rather than trusting the file: a job whose pid is dead and whose status still reads running is not running, it died. Report it as such — ''died without recording an exit'' is honest and actionable, whereas showing it as running is the exit-0 trap in a place nobody is watching, and a caller polling for completion would wait forever on something that failed in milliseconds. Note the ordinary failure paths are already covered: T-1277 moved completion recording to the process''s exit (tooling/main.py main()), so bad arguments, unknown verbs, ReachErrors and unhandled exceptions all record correctly. What remains is only the case where the process cannot run code at all.', NULL, '2026-08-31 14:46:03', '2026-08-31 14:46:03.939', '2026-08-31 14:46:03.939', NULL, 'a02f3f35330e271011c615aef8f9a141', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5TWD0VVBM4F2WZYAQFET0', 'status', 'in_progress', 'done', NULL, '2026-08-31 14:46:22', '2026-08-31 14:46:22.896', '2026-08-31 14:46:22.896', NULL, '36f5973de73c625bf83f3a4b7c6550c9', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5VJ89C7Q4EPR6Q73FSS74', 'status', 'backlog', 'in_progress', NULL, '2026-08-31 14:52:17', '2026-08-31 14:52:17.707', '2026-08-31 14:52:17.707', NULL, '30c48c89065bcea3bdec59fc056c2358', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5VJ89C7Q4EPR6Q73FSS74', 'status', 'in_progress', 'in_progress', NULL, '2026-08-31 14:52:36', '2026-08-31 14:52:36.710', '2026-08-31 14:52:36.710', NULL, 'b5e4dd4502af98fa5342e25449f740f5', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5VJ89C7Q4EPR6Q73FSS74', 'description', 'The user-facing verbs, and a DOMAIN rather than core/ because they carry logic and state of their own — the first real test of the D-263 core bound, which it passes. Deliverables: reach jobs list (recent jobs with status, command and duration), status , log with --follow to tail, and wait . REATTACH IS A BYTE OFFSET into an append-only file, which is the entire reason no daemon is needed: a caller can attach, drop off, and come back without losing anything, and there is no lifecycle to get wrong, nothing to orphan, and no stale state to reconcile. log --follow is therefore a poll on file length, not a subscription. Render the JSONL through the same path a live terminal uses, so a tailed log and a live run are the same artefact in two presentations rather than two renderers that drift. Note for the port: this domain is the first one written from scratch under the full contract rather than ported, so it doubles as the worked example the reach skill (T-1254) should show. + +FROM T-1277 (2026-08-31) — the reconciliation requirement is yours and it is not optional. A child killed outright (SIGKILL, OOM, an interpreter crash) never gets to record its own completion, so its metadata file stays status=running with the process long gone. jobs list and jobs status must therefore RECONCILE against process.is_alive(pid) rather than trusting the file: a job whose pid is dead and whose status still reads running is not running, it died. Report it as such — ''died without recording an exit'' is honest and actionable, whereas showing it as running is the exit-0 trap in a place nobody is watching, and a caller polling for completion would wait forever on something that failed in milliseconds. Note the ordinary failure paths are already covered: T-1277 moved completion recording to the process''s exit (tooling/main.py main()), so bad arguments, unknown verbs, ReachErrors and unhandled exceptions all record correctly. What remains is only the case where the process cannot run code at all.', 'The user-facing verbs, and a DOMAIN rather than core/ because they carry logic and state of their own — the first real test of the D-263 core bound, which it passes. Deliverables: reach jobs list (recent jobs with status, command and duration), status , log with --follow to tail, and wait . REATTACH IS A BYTE OFFSET into an append-only file, which is the entire reason no daemon is needed: a caller can attach, drop off, and come back without losing anything, and there is no lifecycle to get wrong, nothing to orphan, and no stale state to reconcile. log --follow is therefore a poll on file length, not a subscription. Render the JSONL through the same path a live terminal uses, so a tailed log and a live run are the same artefact in two presentations rather than two renderers that drift. Note for the port: this domain is the first one written from scratch under the full contract rather than ported, so it doubles as the worked example the reach skill (T-1254) should show. + +FROM T-1277 (2026-08-31) — the reconciliation requirement is yours and it is not optional. A child killed outright (SIGKILL, OOM, an interpreter crash) never gets to record its own completion, so its metadata file stays status=running with the process long gone. jobs list and jobs status must therefore RECONCILE against process.is_alive(pid) rather than trusting the file: a job whose pid is dead and whose status still reads running is not running, it died. Report it as such — ''died without recording an exit'' is honest and actionable, whereas showing it as running is the exit-0 trap in a place nobody is watching, and a caller polling for completion would wait forever on something that failed in milliseconds. Note the ordinary failure paths are already covered: T-1277 moved completion recording to the process''s exit (tooling/main.py main()), so bad arguments, unknown verbs, ReachErrors and unhandled exceptions all record correctly. What remains is only the case where the process cannot run code at all. + +DONE 2026-08-31. reach jobs list / status / log --follow / wait, all four working, plus service-level tests. + +A LATENT BUG FOUND BEFORE BUILDING ON IT, and it was already committed. typer.Exit is a RuntimeError, NOT a SystemExit — so @handle_errors caught it like any other unexpected exception. Verified: `raise typer.Exit(3)` inside a decorated command printed "unexpected Exit: 3" and exited 1, SILENTLY DISCARDING the requested code. Nothing hit it today because T-1267 had converted the check router to ReachError, but `jobs wait` needs exactly this and it is the natural thing anyone would write. +FIX: core/errors.ReachExit(code) as the sanctioned control-flow exit, passed straight through by handle_errors as SystemExit with no verdict. ReachError would have been wrong twice over for wait — it prints a failure verdict for a command that worked, and demands a fix= for a situation with no remedy. Kept typer out of core/, which the conformance test enforces. + +RECONCILIATION WORKS, AND WAS PROVEN ON A REAL CORPSE rather than a simulated one. The job left stranded by the T-1277 bug — status "running", process long gone — now reports as `died` with its true elapsed time. Status DIED is derived, never recorded: a process killed outright cannot write its own ending. + +EXIT-CODE RELAY, all three cases: done -> 0, failed -> 2 (the job''s own code), died -> 137. A died job must NEVER relay 0; it has no code of its own and borrowing success is the exit-0 trap pointed at whatever gated on the run. + +BYTE-OFFSET REATTACH verified directly: first read consumed one event to offset 169; resuming from 169 returned nothing and left the offset unmoved. That is the whole reason no daemon is needed. + +SECOND UTC BUG, same class as T-1276''s. jobs list reported a job started minutes earlier as running for 133m — _parse used time.mktime, which reads a UTC stamp as local time and silently adds the offset to every duration. Now calendar.timegm. Both directions of this conversion have now bitten once each; both have the reason in a comment. + +RENDERING: console.render() is now public and jobs log replays stored events through it. A second renderer for stored events would drift from the live one, and the divergence would surface exactly when someone is reading a log to work out what went wrong. + +SERVICE-LEVEL TESTS — tooling/test_jobs.py, and these close the gap T-1257 named. D-263 claims a service is transport-agnostic and callable without a CLI round trip; nothing had ever demonstrated it, which made the layering unverified decoration. Every test here imports service and calls a function directly. Covers the partial trailing line (a reader arriving mid-append must leave the incomplete line for next time, not discard the one event being written when it looked), offset stability, dead-pid reconciliation, and died-never-relays-0. Proven to fail: making a died job return 0 tripped the exit-0 assertion by name. +read_events was split into read_events(job_id) and read_events_from(path) so a test can drive it against a temp file — better shape regardless. + +NOT YET EXERCISED, and honest about it: `jobs log --follow` against a genuinely LONG-running job. Nothing in reach currently runs long enough to tail. The offset mechanics underneath it are tested directly, and the live loop gets its first real workout when a slow domain lands (planet or db, T-1250).', NULL, '2026-08-31 15:02:20', '2026-08-31 15:02:20.709', '2026-08-31 15:02:20.709', NULL, 'becbc04f4ea86ae9f73e0c9541992300', 2) ON CONFLICT(hash) DO NOTHING; diff --git a/.pql/changelog/tickets/2026-08.sql b/.pql/changelog/tickets/2026-08.sql index 3807adb4f..bdb4e81d2 100644 --- a/.pql/changelog/tickets/2026-08.sql +++ b/.pql/changelog/tickets/2026-08.sql @@ -2236,3 +2236,54 @@ reach --help is still 73 ms, so the entry-point wrapper costs nothing on the fas INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5VJ89C7Q4EPR6Q73FSS74', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'The jobs domain — list, status, log --follow, wait', 'The user-facing verbs, and a DOMAIN rather than core/ because they carry logic and state of their own — the first real test of the D-263 core bound, which it passes. Deliverables: reach jobs list (recent jobs with status, command and duration), status , log with --follow to tail, and wait . REATTACH IS A BYTE OFFSET into an append-only file, which is the entire reason no daemon is needed: a caller can attach, drop off, and come back without losing anything, and there is no lifecycle to get wrong, nothing to orphan, and no stale state to reconcile. log --follow is therefore a poll on file length, not a subscription. Render the JSONL through the same path a live terminal uses, so a tailed log and a live run are the same artefact in two presentations rather than two renderers that drift. Note for the port: this domain is the first one written from scratch under the full contract rather than ported, so it doubles as the worked example the reach skill (T-1254) should show. FROM T-1277 (2026-08-31) — the reconciliation requirement is yours and it is not optional. A child killed outright (SIGKILL, OOM, an interpreter crash) never gets to record its own completion, so its metadata file stays status=running with the process long gone. jobs list and jobs status must therefore RECONCILE against process.is_alive(pid) rather than trusting the file: a job whose pid is dead and whose status still reads running is not running, it died. Report it as such — ''died without recording an exit'' is honest and actionable, whereas showing it as running is the exit-0 trap in a place nobody is watching, and a caller polling for completion would wait forever on something that failed in milliseconds. Note the ordinary failure paths are already covered: T-1277 moved completion recording to the process''s exit (tooling/main.py main()), so bad arguments, unknown verbs, ReachErrors and unhandled exceptions all record correctly. What remains is only the case where the process cannot run code at all.', 'backlog', 'high', NULL, NULL, 'D-263', '2026-08-31 13:52:06.722', '2026-08-31 14:46:03.939', NULL, 'f9aac99fd6449919ab2d48455a81b6e7', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5TWD0VVBM4F2WZYAQFET0', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'core/process.py — detach so the child outlives the parent', 'The spawn primitive, and it is substrate rather than a domain because it has no verbs of its own. Deliverables: spawn a detached child that survives the parent exiting (setsid or equivalent, not just a background shell job, since a killed parent must not take the work with it); redirect the child''s event stream to .cache/reach/jobs/.jsonl and its stdout to a sibling file, keeping the two channels separate exactly as they are in the foreground; write a metadata record carrying command, argv, start time, pid and — on completion — end time and exit code. The metadata file is what makes a finished job readable without re-reading a possibly enormous log. .cache/ is already gitignored. WATCH: the child must re-exec the same reach that was invoked, resolved by bare name per T-1261''s negative criterion, never by an interpreter path or a .venv path — an absolute path here would break the moment the tool is re-pointed at another checkout, and would be a silent wrong-source failure of exactly the kind make reach-repoint exists to fix. Also watch the completion race: the exit code must be recorded by the CHILD as its last act, not polled by a parent that may already be gone. + +DONE 2026-08-31. Detached execution works end to end, and testing it found a real hole that the design as written would have shipped. + +DELIVERED: tooling/core/process.py (spawn, metadata, liveness), the --detach flag on the root callback, and completion recording. Verified against a real spawn — parent returns the job id and exits 0, child runs on and writes .cache/reach/jobs/.jsonl tagged with that id, plus a .out sibling and a .json metadata record. + +THE THREE THINGS THE TICKET FLAGGED, each handled and each verified: +- start_new_session=True, so the child gets its own session and process group and a signal to the parent''s group does not take the work with it. +- The child re-execs `reach` by BARE NAME. An absolute path would freeze it to whichever checkout was current at spawn time, so after make reach-repoint a detached job would silently run the wrong source — the exact failure that command exists to fix. +- The child records its own exit code. Confirmed on the success path (status done, exit_code 0) and on a real failure path (status failed, exit_code 1) using a drift fixture pointed at by SR_REPO_ROOT. + +THE HOLE, found only because I tested a THIRD case the ticket did not name. +Recording completion inside @command looked right and was subtly wrong. A child that fails BEFORE any command runs — bad arguments, an unknown verb, an import error — never reaches that decorator. Verified: `reach --detach check bogus` left its metadata reading status "running" FOREVER, with the process long gone. +That is the exit-0 trap wearing a new disguise, and worse than the original: a failed job that looks busy, in a place nobody is watching. A caller polling for completion would wait indefinitely on a job that failed in milliseconds. +FIX: completion is now recorded at the PROCESS''s exit rather than a command''s. tooling/main.py gains main(), which wraps cli() in one try/finally, and [project.scripts] points at main:main instead of main:cli. Every exit path — success, ReachError, usage error, unhandled exception — now passes through a single finally. Re-verified: `reach --detach check bogus` records status failed, exit_code 2. +The recording was REMOVED from @command rather than left in both places; two writers of the same field is how they drift. + +CONFORMANCE EXEMPTION ADDED, deliberately narrow. The new no-domain-imports-core.jobs invariant fired on main.py, correctly by its letter and wrongly by its purpose. main.py is not a command — it is the entry point, and it already owns --detach, --verbose and --no-input. Reading a job-id constant there is far less coupled than the --detach flag it already carries. Exempted main.py explicitly, with the reason inline so it does not read as an oversight. + +STILL OPEN, and correctly belongs to T-1278: a child killed outright (SIGKILL, OOM, interpreter crash) still cannot record anything, so its metadata stays "running". process.is_alive(pid) exists for exactly this, and jobs list/status must reconcile against it rather than trusting the file. + +reach --help is still 73 ms, so the entry-point wrapper costs nothing on the fast path.', 'done', 'high', NULL, NULL, 'D-263', '2026-08-31 13:52:01.128', '2026-08-31 14:46:22.895', NULL, '1d5d86a8353e16e113d1c77588a90e55', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5VJ89C7Q4EPR6Q73FSS74', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'The jobs domain — list, status, log --follow, wait', 'The user-facing verbs, and a DOMAIN rather than core/ because they carry logic and state of their own — the first real test of the D-263 core bound, which it passes. Deliverables: reach jobs list (recent jobs with status, command and duration), status , log with --follow to tail, and wait . REATTACH IS A BYTE OFFSET into an append-only file, which is the entire reason no daemon is needed: a caller can attach, drop off, and come back without losing anything, and there is no lifecycle to get wrong, nothing to orphan, and no stale state to reconcile. log --follow is therefore a poll on file length, not a subscription. Render the JSONL through the same path a live terminal uses, so a tailed log and a live run are the same artefact in two presentations rather than two renderers that drift. Note for the port: this domain is the first one written from scratch under the full contract rather than ported, so it doubles as the worked example the reach skill (T-1254) should show. + +FROM T-1277 (2026-08-31) — the reconciliation requirement is yours and it is not optional. A child killed outright (SIGKILL, OOM, an interpreter crash) never gets to record its own completion, so its metadata file stays status=running with the process long gone. jobs list and jobs status must therefore RECONCILE against process.is_alive(pid) rather than trusting the file: a job whose pid is dead and whose status still reads running is not running, it died. Report it as such — ''died without recording an exit'' is honest and actionable, whereas showing it as running is the exit-0 trap in a place nobody is watching, and a caller polling for completion would wait forever on something that failed in milliseconds. Note the ordinary failure paths are already covered: T-1277 moved completion recording to the process''s exit (tooling/main.py main()), so bad arguments, unknown verbs, ReachErrors and unhandled exceptions all record correctly. What remains is only the case where the process cannot run code at all.', 'in_progress', 'high', NULL, NULL, 'D-263', '2026-08-31 13:52:06.722', '2026-08-31 14:52:17.707', NULL, '5c00db52e693d19da200416e4d3901a5', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5VJ89C7Q4EPR6Q73FSS74', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'The jobs domain — list, status, log --follow, wait', 'The user-facing verbs, and a DOMAIN rather than core/ because they carry logic and state of their own — the first real test of the D-263 core bound, which it passes. Deliverables: reach jobs list (recent jobs with status, command and duration), status , log with --follow to tail, and wait . REATTACH IS A BYTE OFFSET into an append-only file, which is the entire reason no daemon is needed: a caller can attach, drop off, and come back without losing anything, and there is no lifecycle to get wrong, nothing to orphan, and no stale state to reconcile. log --follow is therefore a poll on file length, not a subscription. Render the JSONL through the same path a live terminal uses, so a tailed log and a live run are the same artefact in two presentations rather than two renderers that drift. Note for the port: this domain is the first one written from scratch under the full contract rather than ported, so it doubles as the worked example the reach skill (T-1254) should show. + +FROM T-1277 (2026-08-31) — the reconciliation requirement is yours and it is not optional. A child killed outright (SIGKILL, OOM, an interpreter crash) never gets to record its own completion, so its metadata file stays status=running with the process long gone. jobs list and jobs status must therefore RECONCILE against process.is_alive(pid) rather than trusting the file: a job whose pid is dead and whose status still reads running is not running, it died. Report it as such — ''died without recording an exit'' is honest and actionable, whereas showing it as running is the exit-0 trap in a place nobody is watching, and a caller polling for completion would wait forever on something that failed in milliseconds. Note the ordinary failure paths are already covered: T-1277 moved completion recording to the process''s exit (tooling/main.py main()), so bad arguments, unknown verbs, ReachErrors and unhandled exceptions all record correctly. What remains is only the case where the process cannot run code at all.', 'in_progress', 'high', NULL, NULL, 'D-263', '2026-08-31 13:52:06.722', '2026-08-31 14:52:36.710', NULL, '0a348ae5a631dcd868a7830e0be0de6c', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5VJ89C7Q4EPR6Q73FSS74', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'The jobs domain — list, status, log --follow, wait', 'The user-facing verbs, and a DOMAIN rather than core/ because they carry logic and state of their own — the first real test of the D-263 core bound, which it passes. Deliverables: reach jobs list (recent jobs with status, command and duration), status , log with --follow to tail, and wait . REATTACH IS A BYTE OFFSET into an append-only file, which is the entire reason no daemon is needed: a caller can attach, drop off, and come back without losing anything, and there is no lifecycle to get wrong, nothing to orphan, and no stale state to reconcile. log --follow is therefore a poll on file length, not a subscription. Render the JSONL through the same path a live terminal uses, so a tailed log and a live run are the same artefact in two presentations rather than two renderers that drift. Note for the port: this domain is the first one written from scratch under the full contract rather than ported, so it doubles as the worked example the reach skill (T-1254) should show. + +FROM T-1277 (2026-08-31) — the reconciliation requirement is yours and it is not optional. A child killed outright (SIGKILL, OOM, an interpreter crash) never gets to record its own completion, so its metadata file stays status=running with the process long gone. jobs list and jobs status must therefore RECONCILE against process.is_alive(pid) rather than trusting the file: a job whose pid is dead and whose status still reads running is not running, it died. Report it as such — ''died without recording an exit'' is honest and actionable, whereas showing it as running is the exit-0 trap in a place nobody is watching, and a caller polling for completion would wait forever on something that failed in milliseconds. Note the ordinary failure paths are already covered: T-1277 moved completion recording to the process''s exit (tooling/main.py main()), so bad arguments, unknown verbs, ReachErrors and unhandled exceptions all record correctly. What remains is only the case where the process cannot run code at all. + +DONE 2026-08-31. reach jobs list / status / log --follow / wait, all four working, plus service-level tests. + +A LATENT BUG FOUND BEFORE BUILDING ON IT, and it was already committed. typer.Exit is a RuntimeError, NOT a SystemExit — so @handle_errors caught it like any other unexpected exception. Verified: `raise typer.Exit(3)` inside a decorated command printed "unexpected Exit: 3" and exited 1, SILENTLY DISCARDING the requested code. Nothing hit it today because T-1267 had converted the check router to ReachError, but `jobs wait` needs exactly this and it is the natural thing anyone would write. +FIX: core/errors.ReachExit(code) as the sanctioned control-flow exit, passed straight through by handle_errors as SystemExit with no verdict. ReachError would have been wrong twice over for wait — it prints a failure verdict for a command that worked, and demands a fix= for a situation with no remedy. Kept typer out of core/, which the conformance test enforces. + +RECONCILIATION WORKS, AND WAS PROVEN ON A REAL CORPSE rather than a simulated one. The job left stranded by the T-1277 bug — status "running", process long gone — now reports as `died` with its true elapsed time. Status DIED is derived, never recorded: a process killed outright cannot write its own ending. + +EXIT-CODE RELAY, all three cases: done -> 0, failed -> 2 (the job''s own code), died -> 137. A died job must NEVER relay 0; it has no code of its own and borrowing success is the exit-0 trap pointed at whatever gated on the run. + +BYTE-OFFSET REATTACH verified directly: first read consumed one event to offset 169; resuming from 169 returned nothing and left the offset unmoved. That is the whole reason no daemon is needed. + +SECOND UTC BUG, same class as T-1276''s. jobs list reported a job started minutes earlier as running for 133m — _parse used time.mktime, which reads a UTC stamp as local time and silently adds the offset to every duration. Now calendar.timegm. Both directions of this conversion have now bitten once each; both have the reason in a comment. + +RENDERING: console.render() is now public and jobs log replays stored events through it. A second renderer for stored events would drift from the live one, and the divergence would surface exactly when someone is reading a log to work out what went wrong. + +SERVICE-LEVEL TESTS — tooling/test_jobs.py, and these close the gap T-1257 named. D-263 claims a service is transport-agnostic and callable without a CLI round trip; nothing had ever demonstrated it, which made the layering unverified decoration. Every test here imports service and calls a function directly. Covers the partial trailing line (a reader arriving mid-append must leave the incomplete line for next time, not discard the one event being written when it looked), offset stability, dead-pid reconciliation, and died-never-relays-0. Proven to fail: making a died job return 0 tripped the exit-0 assertion by name. +read_events was split into read_events(job_id) and read_events_from(path) so a test can drive it against a temp file — better shape regardless. + +NOT YET EXERCISED, and honest about it: `jobs log --follow` against a genuinely LONG-running job. Nothing in reach currently runs long enough to tail. The offset mechanics underneath it are tested directly, and the live loop gets its first real workout when a slow domain lands (planet or db, T-1250).', 'in_progress', 'high', NULL, NULL, 'D-263', '2026-08-31 13:52:06.722', '2026-08-31 15:02:20.709', NULL, '1ce1eb66a71783ab5218d27ae143b1aa', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; diff --git a/Makefile b/Makefile index 930fb7d67..ac377e721 100644 --- a/Makefile +++ b/Makefile @@ -307,6 +307,10 @@ test-tooling: @mkdir -p .cache @$(VENV_PY) tooling/test_lazy_domains.py 2> .cache/test-tooling-lazy-domains.log || \ { echo " FAIL: reach lazy registration — log follows:"; cat .cache/test-tooling-lazy-domains.log; exit 1; } + @echo " [test-tooling] reach jobs service units (T-1278)..." + @mkdir -p .cache + @$(VENV_PY) tooling/test_jobs.py 2> .cache/test-tooling-jobs.log || \ + { echo " FAIL: jobs service — log follows:"; cat .cache/test-tooling-jobs.log; exit 1; } @echo " [test-tooling] reach D-263 conformance (T-1270)..." @mkdir -p .cache @$(VENV_PY) tooling/test_conformance.py 2> .cache/test-tooling-conformance.log || \ diff --git a/tooling/core/console.py b/tooling/core/console.py index 97c4b2e4b..475b374a1 100644 --- a/tooling/core/console.py +++ b/tooling/core/console.py @@ -150,6 +150,18 @@ def _render_as_text(stream: TextIO) -> str | bool: return False +def render(payload: dict[str, Any]) -> str: + """Render one event as a human would see it live. + + Public because `reach jobs log` replays a stored event stream through it. + A second renderer for stored events would drift from this one, and the + divergence would show up exactly when someone is reading a log to work out + what went wrong — the worst moment to be looking at output that does not + match what the live run printed. + """ + return _render(payload) + + def _render(payload: dict[str, Any]) -> str: message = payload.get("message", "") if payload.get("kind") == "verdict": diff --git a/tooling/core/errors.py b/tooling/core/errors.py index 2a8cff352..61de27ed2 100644 --- a/tooling/core/errors.py +++ b/tooling/core/errors.py @@ -42,6 +42,27 @@ class ReachError(Exception): self.exit_code = exit_code if exit_code != 0 else 1 +class ReachExit(Exception): # noqa: N818 control flow, not an error + """Exit with a specific code, quietly. Not a failure. + + For a command that must RELAY an exit code rather than report one — most + obviously `reach jobs wait`, which exits with the code of the job it waited + on. That is not `jobs wait` failing, so a `ReachError` would be wrong twice + over: it would print a failure verdict for a command that worked, and + demand a `fix=` for a situation with no remedy. + + **Use this, never `typer.Exit`, inside a decorated command.** `typer.Exit` + is a `RuntimeError`, not a `SystemExit`, so `handle_errors` catches it like + any other unexpected exception — reporting "unexpected Exit: 3" and exiting + **1**, silently discarding the code that was asked for. The conformance + suite forbids it in routers so the trap cannot be re-entered. + """ + + def __init__(self, exit_code: int = 0) -> None: + super().__init__(f"exit {exit_code}") + self.exit_code = exit_code + + def unknown_choice(kind: str, given: str, accepted: Iterable[str]) -> ReachError: """Reject a value from a known finite set, naming the whole set. @@ -70,6 +91,9 @@ def handle_errors(func: F) -> F: def wrapper(*args: Any, **kwargs: Any) -> Any: try: return func(*args, **kwargs) + except ReachExit as exc: + # Control flow, not a failure — no verdict, just the code. + raise SystemExit(exc.exit_code) from None except ReachError as exc: # The verdict prints ONCE, LAST, after whatever the command streamed. # A remedy emitted mid-stream at line 400 of 900 is technically diff --git a/tooling/domains/jobs/__init__.py b/tooling/domains/jobs/__init__.py new file mode 100644 index 000000000..f96f7aa80 --- /dev/null +++ b/tooling/domains/jobs/__init__.py @@ -0,0 +1,12 @@ +"""The `jobs` domain — detached runs, their logs, and their outcomes. + +A domain rather than part of `core/`, and this is the first real test of that +bound (D-263). The *primitives* — spawn, detach, record — are substrate and live +in `core/process.py`. These verbs have logic and state of their own: they +reconcile recorded status against process liveness, tail a file from an offset, +and relay an exit code. A job store in `core/` would be exactly the drift the +record warns about. + +Also the first domain written from scratch under the full contract rather than +ported, which makes it the worked example for the reach skill (T-1254). +""" diff --git a/tooling/domains/jobs/router.py b/tooling/domains/jobs/router.py new file mode 100644 index 000000000..ce1eda659 --- /dev/null +++ b/tooling/domains/jobs/router.py @@ -0,0 +1,116 @@ +"""Transport for the `jobs` domain — args in, delegate, format out. + +Zero logic. Reconciliation, offsets and polling all live in `service.py`; what +happens here is turning a `Job` into lines and an exit code. +""" + +from __future__ import annotations + +import time + +import typer + +from tooling.core import cli, console +from tooling.core.command import command +from tooling.core.errors import ReachExit +from tooling.domains.jobs import service +from tooling.domains.jobs.schemas import Status + +app = cli.domain("jobs", "Detached runs — what is running, what it printed, how it ended.") + + +@app.callback() +def _domain() -> None: + """Keeps `jobs` a group (Typer collapses a single-command app).""" + + +@app.command("list") +@command +def list_jobs( + limit: int = typer.Option(20, "--limit", "-n", help="How many recent jobs to show."), +) -> None: + """Recent detached runs, newest first.""" + jobs = service.list_jobs(limit) + if not jobs: + console.out("no jobs recorded — start one with: reach --detach ") + return + for job in jobs: + console.out( + f"{job.job} {job.status.value:<8} {service.duration(job):>7} {job.command}" + ) + + +@app.command("status") +@command +def status(job_id: str = typer.Argument(..., help="Job id, as printed by --detach.")) -> None: + """One job's outcome, reconciled against whether its process is alive.""" + job = service.get(job_id) + console.out(f"job {job.job}") + console.out(f"command {job.command}") + console.out(f"status {job.status.value}") + console.out(f"started {job.started_at}") + console.out(f"elapsed {service.duration(job)}") + if job.exit_code is not None: + console.out(f"exit {job.exit_code}") + if job.status is Status.DIED: + # Said in words, because "died" alone reads like a synonym for "failed" + # and the distinction matters: nothing recorded an outcome here. + console.out("") + console.out("This job's process is gone but it never recorded an ending —") + console.out("killed outright (SIGKILL, OOM, or a crash). Its log holds") + console.out("whatever it managed to emit before that.") + + +@app.command("log") +@command +def log( + job_id: str = typer.Argument(..., help="Job id, as printed by --detach."), + follow: bool = typer.Option(False, "--follow", "-f", help="Keep printing as it runs."), +) -> None: + """Replay a job's event stream, rendered as it appeared live.""" + job = service.get(job_id) + events, offset = service.read_events(job_id) + for event in events: + # Rendered through console, not a local formatter, so a stored log and a + # live run are one artefact in two presentations rather than two + # renderers that drift apart precisely when someone is debugging. + console.out(console.render(event).rstrip("\n")) + + if not follow: + return + + while not job.status.finished: + time.sleep(service.POLL_SECONDS) + events, offset = service.read_events(job_id, offset) + for event in events: + console.out(console.render(event).rstrip("\n")) + job = service.get(job_id) + + # One last read: the job may have written its final events between the last + # poll and its exit, and stopping at the status flip would drop them. + events, offset = service.read_events(job_id, offset) + for event in events: + console.out(console.render(event).rstrip("\n")) + + +@app.command("wait") +@command +def wait( + job_id: str = typer.Argument(..., help="Job id, as printed by --detach."), + timeout: float = typer.Option(None, "--timeout", help="Give up after N seconds."), +) -> None: + """Block until a job finishes, then exit with ITS exit code. + + That relay is the point: a Makefile or a hook can gate on a detached run + exactly as it would on a foreground one. `ReachExit` rather than + `ReachError` because waiting successfully for a job that failed is not a + failure of `wait`. + """ + job = service.wait(job_id, timeout) + console.verdict( + f"job {job.job} {job.status.value} after {service.duration(job)}" + + (f" (exit {job.exit_code})" if job.exit_code is not None else ""), + ok=job.status is Status.DONE, + fix=None if job.status is Status.DONE else f"reach jobs log {job.job}", + ) + raise ReachExit(job.effective_exit_code) diff --git a/tooling/domains/jobs/schemas.py b/tooling/domains/jobs/schemas.py new file mode 100644 index 000000000..2919355c6 --- /dev/null +++ b/tooling/domains/jobs/schemas.py @@ -0,0 +1,55 @@ +"""Data shapes for the `jobs` domain.""" + +from __future__ import annotations + +from enum import Enum + +from pydantic import BaseModel, ConfigDict + + +class Status(str, Enum): + """What a job is doing, after reconciliation. + + `RUNNING`, `DONE` and `FAILED` are recorded by the job itself. **`DIED` is + never recorded** — it is derived, when a job's file still says running but + its pid is gone. A process killed outright (SIGKILL, OOM, an interpreter + crash) cannot write its own ending, so trusting the file would leave a + corpse looking busy forever, and anything polling for completion would wait + on it indefinitely. + """ + + RUNNING = "running" + DONE = "done" + FAILED = "failed" + DIED = "died" + + @property + def finished(self) -> bool: + return self is not Status.RUNNING + + +class Job(BaseModel): + """One detached run.""" + + model_config = ConfigDict(frozen=True) + + job: str + command: str + argv: list[str] = [] + pid: int + started_at: str + status: Status + exit_code: int | None = None + ended_at: str | None = None + + @property + def effective_exit_code(self) -> int: + """The code a caller should adopt when relaying this job's outcome. + + A job that died without recording anything has no code of its own. It + must not be reported as 0 — that is the exit-0 trap, and the whole + reason `DIED` is distinguished from `DONE`. + """ + if self.status is Status.DIED: + return 137 # conventional 128+SIGKILL: killed, not completed + return self.exit_code if self.exit_code is not None else 0 diff --git a/tooling/domains/jobs/service.py b/tooling/domains/jobs/service.py new file mode 100644 index 000000000..034f4adfc --- /dev/null +++ b/tooling/domains/jobs/service.py @@ -0,0 +1,146 @@ +"""Logic for the `jobs` domain. Transport-agnostic (D-263). + +Nothing here prints, exits, or imports typer. +""" + +from __future__ import annotations + +import calendar +import json +import time +from pathlib import Path +from typing import Any + +from tooling.core import process +from tooling.core.errors import ReachError +from tooling.domains.jobs.schemas import Job, Status + +# How often a follow/wait loop re-checks. Chosen for a caller that is a program +# rather than an eye: fast enough that `wait` does not add noticeable latency to +# a short job, slow enough not to spin a core on a long one. +POLL_SECONDS = 0.25 + + +def list_jobs(limit: int = 20) -> list[Job]: + """Recent jobs, newest first, each reconciled against process liveness.""" + directory = process.jobs_dir() + files = sorted(directory.glob("*.json"), reverse=True) + jobs = [_load(path.stem) for path in files[:limit]] + return [job for job in jobs if job is not None] + + +def get(job_id: str) -> Job: + """One job by id, reconciled. Raises if it does not exist.""" + job = _load(job_id) + if job is None: + known = [path.stem for path in sorted(process.jobs_dir().glob("*.json"), reverse=True)] + recent = ", ".join(known[:5]) if known else "(no jobs recorded yet)" + raise ReachError( + f"no such job: {job_id}", + fix=f"reach jobs list — most recent are: {recent}", + exit_code=2, + ) + return job + + +def read_events(job_id: str, offset: int = 0) -> tuple[list[dict[str, Any]], int]: + """Events from `offset`, plus the new offset. + + A byte offset into an append-only file is the entire reason no daemon is + needed: a caller can read, drop off, and come back with the offset it kept, + and nothing has to have been holding a subscription open on its behalf. + """ + return read_events_from(process.log_path(job_id), offset) + + +def read_events_from(path: Path, offset: int = 0) -> tuple[list[dict[str, Any]], int]: + """The same, given a path — so a test can drive it without a real job.""" + if not path.is_file(): + return [], offset + + with path.open("rb") as handle: + handle.seek(offset) + raw = handle.read() + new_offset = handle.tell() + + events: list[dict[str, Any]] = [] + consumed = offset + for line in raw.split(b"\n"): + if not line.strip(): + consumed += len(line) + 1 + continue + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + # A partial trailing line: the writer is mid-append. Leave the + # offset before it so the next read picks it up whole rather than + # discarding an event because we looked a millisecond too early. + return events, consumed + consumed += len(line) + 1 + return events, new_offset + + +def wait(job_id: str, timeout: float | None = None) -> Job: + """Block until the job finishes; return it. Never returns while running.""" + deadline = None if timeout is None else time.monotonic() + timeout + while True: + job = get(job_id) + if job.status.finished: + return job + if deadline is not None and time.monotonic() >= deadline: + raise ReachError( + f"timed out after {timeout:g}s waiting for job {job_id}", + fix=f"reach jobs status {job_id} — the job is still running, not lost", + exit_code=2, + ) + time.sleep(POLL_SECONDS) + + +def duration(job: Job) -> str: + """Human-readable elapsed time, or how long it has been running so far.""" + start = _parse(job.started_at) + end = _parse(job.ended_at) if job.ended_at else time.time() + if start is None or end is None: + return "?" + seconds = max(0.0, end - start) + if seconds < 60: + return f"{seconds:.1f}s" + minutes, rest = divmod(int(seconds), 60) + return f"{minutes}m{rest:02d}s" + + +def _load(job_id: str) -> Job | None: + meta = process.read_meta(job_id) + if meta is None: + return None + return _reconcile(Job.model_validate(meta)) + + +def _reconcile(job: Job) -> Job: + """Correct a recorded status against reality. + + A job whose file says running but whose pid is gone did not keep running — + it died without being able to record anything. Reporting it as running + would be the exit-0 trap somewhere nobody is watching, and would hang any + caller polling for it to finish. + """ + if job.status is Status.RUNNING and not process.is_alive(job.pid): + return job.model_copy(update={"status": Status.DIED}) + return job + + +def _parse(stamp: str | None) -> float | None: + """Parse a recorded timestamp as UTC. + + `calendar.timegm`, NOT `time.mktime`: the stamps are written in UTC, and + mktime would read them as local time. That silently adds the UTC offset to + every duration — a job started seconds ago reported as having run for over + two hours. The same mismatch bit the job id in T-1276; both directions of + this conversion need saying out loud. + """ + if not stamp: + return None + try: + return calendar.timegm(time.strptime(stamp, "%Y-%m-%dT%H:%M:%S")) + except (ValueError, TypeError): + return None diff --git a/tooling/main.py b/tooling/main.py index 793657c45..aed46f166 100644 --- a/tooling/main.py +++ b/tooling/main.py @@ -51,6 +51,10 @@ DOMAINS: dict[str, tuple[str, str]] = { "tooling.domains.check.router:app", "Consistency gates — the checks the push hook runs", ), + "jobs": ( + "tooling.domains.jobs.router:app", + "Detached runs — status, logs and outcomes", + ), } diff --git a/tooling/test_jobs.py b/tooling/test_jobs.py new file mode 100644 index 000000000..cd830eb10 --- /dev/null +++ b/tooling/test_jobs.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Units for the jobs service (T-1278), called DIRECTLY — no CLI round trip. + +That is half the point of these tests. D-263 says a service must be +transport-agnostic so it can be called by a test, by another service, or by a +future second front end. If nothing ever exercises that, the layering is +unverified decoration — a claim in a decision record with no evidence behind it. +Every test here imports `service` and calls a function. + +The other half is the two behaviours that are easy to get wrong and impossible +to notice when they are: + + 1. A partial trailing line. The log is appended to by a live process, so a + reader can arrive mid-write. Parsing greedily would either crash or, worse, + silently discard the event and advance past it — losing exactly one line, + the one being written when someone looked. + 2. Reconciliation. A job killed outright cannot record its ending, so its file + says `running` forever. Trusting the file leaves a corpse looking busy and + hangs anything waiting on it. + +Run: python3 tooling/test_jobs.py +""" + +import json +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from tooling.domains.jobs import service # noqa: E402 +from tooling.domains.jobs.schemas import Job, Status # noqa: E402 + + +def _job(**overrides) -> Job: + base = { + "job": "20260101T000000-test", + "command": "reach check client-version", + "argv": ["check", "client-version"], + "pid": 1, + "started_at": "2026-01-01T00:00:00", + "status": Status.RUNNING, + } + return Job(**{**base, **overrides}) + + +def test_partial_trailing_line(failures: list[str]) -> None: + """A half-written final line is left for the next read, not dropped.""" + with tempfile.TemporaryDirectory() as tmp: + log = Path(tmp) / "j.jsonl" + complete = json.dumps({"message": "one"}) + "\n" + partial = '{"message": "tw' + log.write_text(complete + partial, encoding="utf-8") + + events, offset = service.read_events_from(log, 0) + if len(events) != 1: + failures.append(f"partial line: expected 1 complete event, got {len(events)}") + if offset != len(complete): + failures.append( + f"partial line: offset {offset} should stop at {len(complete)}, " + "before the incomplete line — otherwise that event is lost forever" + ) + + # Now the writer finishes the line. The event must appear. + log.write_text(complete + json.dumps({"message": "two"}) + "\n", encoding="utf-8") + events, _ = service.read_events_from(log, offset) + if [e.get("message") for e in events] != ["two"]: + failures.append(f"partial line: resumed read lost the completed event: {events}") + + +def test_offset_resume_is_stable(failures: list[str]) -> None: + """Reading from the end returns nothing and does not move the offset.""" + with tempfile.TemporaryDirectory() as tmp: + log = Path(tmp) / "j.jsonl" + log.write_text(json.dumps({"message": "one"}) + "\n", encoding="utf-8") + _, first = service.read_events_from(log, 0) + events, second = service.read_events_from(log, first) + if events or first != second: + failures.append( + f"offset resume: re-reading returned {len(events)} events and moved " + f"{first}->{second}; a follow loop would replay forever" + ) + + +def test_dead_pid_is_reconciled(failures: list[str]) -> None: + """running + a pid that is gone == died, never running.""" + # PID 1 exists; a very high pid almost certainly does not. + alive = service._reconcile(_job(pid=1)) + if alive.status is not Status.RUNNING: + failures.append("reconcile: a live pid was reported as not running") + + dead = service._reconcile(_job(pid=4_000_000)) + if dead.status is not Status.DIED: + failures.append( + f"reconcile: a dead pid stayed {dead.status.value} — a corpse that looks " + "busy hangs every caller waiting on it" + ) + + +def test_died_never_relays_success(failures: list[str]) -> None: + """A job that died has no exit code of its own, and must not borrow 0.""" + died = _job(status=Status.DIED) + if died.effective_exit_code == 0: + failures.append( + "died job relayed exit 0 — that is the exit-0 trap: a killed job " + "reported as success to whatever gated on it" + ) + finished = _job(status=Status.FAILED, exit_code=2) + if finished.effective_exit_code != 2: + failures.append("failed job did not relay its own exit code") + + +def main() -> int: + failures: list[str] = [] + test_partial_trailing_line(failures) + test_offset_resume_is_stable(failures) + test_dead_pid_is_reconciled(failures) + test_died_never_relays_success(failures) + + if failures: + print("test_jobs: FAIL", file=sys.stderr) + for failure in failures: + print(f" - {failure}", file=sys.stderr) + return 1 + print("test_jobs: OK — offsets resume cleanly, dead pids reconcile, died never relays 0") + return 0 + + +if __name__ == "__main__": + sys.exit(main())