diff --git a/.pql/changelog/ticket_history/2026-08.sql b/.pql/changelog/ticket_history/2026-08.sql index 5a21fd23b..341362684 100644 --- a/.pql/changelog/ticket_history/2026-08.sql +++ b/.pql/changelog/ticket_history/2026-08.sql @@ -1983,3 +1983,20 @@ CLOSED T-1278''s OPEN GAP as a side effect. `jobs log --follow` had never been e PROVEN TO FAIL: made effective_exit_code always return 0 — the exit-0 trap itself — and both the fast and slow cases failed by name, reporting the expected code against the 0 they got. Restored, green. NOTE ON WHAT --follow RETURNS: `jobs log --follow` exits 0 even when the job it tailed failed. That is correct and deliberate — log succeeded at logging. `wait` is the verb that relays an outcome, and conflating the two would mean you could not read a failed job''s log without your shell treating the read as a failure.', NULL, '2026-08-31 15:14:09', '2026-08-31 15:14:09.188', '2026-08-31 15:14:09.188', NULL, 'a48443a46dc1cc27d6089e81695a0b5c', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5WNGDMENP7AY9VMB5G3XM', 'status', 'in_progress', 'done', NULL, '2026-08-31 15:14:26', '2026-08-31 15:14:26.448', '2026-08-31 15:14:26.448', NULL, 'cb92a2183c01e69b8b0210ceba273faa', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5XC1XX5179S0X6VFW0TM0', 'status', 'backlog', 'in_progress', NULL, '2026-08-31 15:15:08', '2026-08-31 15:15:08.418', '2026-08-31 15:15:08.418', NULL, 'a155a557722498e38a8cb7a25d2df543', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5XC1XX5179S0X6VFW0TM0', 'status', 'in_progress', 'in_progress', NULL, '2026-08-31 15:15:15', '2026-08-31 15:15:15.934', '2026-08-31 15:15:15.934', NULL, 'ead1e0927bbb7d89bd9f6f359981eb6d', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5XC1XX5179S0X6VFW0TM0', 'description', 'An append-only log directory with no retention is a disk leak that nobody notices until it matters. Deliverables: a cap by count or age, applied at write time rather than by a separate sweep — a cleanup nothing invokes is a cleanup that does not happen; reach jobs prune as the explicit escape hatch; and a rule that a RUNNING job is never pruned regardless of age, since a long generator can outlive the window. Bound the whole thing to .cache/, which is gitignored and already the repo''s scratch space, so a wrong answer here costs disk rather than data. Deliberately last in the epic: retention pressure is not real until jobs exist and the log sizes are observable, and guessing a cap before seeing a Blender batch''s output volume would just be a number to change later.', 'An append-only log directory with no retention is a disk leak that nobody notices until it matters. Deliverables: a cap by count or age, applied at write time rather than by a separate sweep — a cleanup nothing invokes is a cleanup that does not happen; reach jobs prune as the explicit escape hatch; and a rule that a RUNNING job is never pruned regardless of age, since a long generator can outlive the window. Bound the whole thing to .cache/, which is gitignored and already the repo''s scratch space, so a wrong answer here costs disk rather than data. Deliberately last in the epic: retention pressure is not real until jobs exist and the log sizes are observable, and guessing a cap before seeing a Blender batch''s output volume would just be a number to change later. + +DONE 2026-08-31. Retention applied at write time, plus reach jobs prune as the explicit escape hatch. + +MEASURED BEFORE CHOOSING A NUMBER, which is why this ticket was scheduled last. A chatty short job (10 progress events) writes ~1.8 KB across its three files; 20 jobs came to 164 KB. A generator emitting per-body progress might reach 100 KB, so a cap of 100 jobs is single-digit megabytes at worst, inside .cache/ which is gitignored scratch. The failure mode of that number being wrong is disk, never data. Overridable with SR_JOB_KEEP, and worth raising once a real Blender batch has run. + +APPLIED AT SPAWN TIME, not on a schedule — a retention pass that depends on someone remembering to run it is one that silently never happens. reach jobs prune --keep N exists as the deliberate escape hatch for reclaiming space now, or --keep 0 to clear everything. + +THE SUBTLETY THE TICKET''S WORDING HIDES, and it is the whole of the design. "A running job is never pruned" has to mean GENUINELY running, checked against the process table — not "the file says running". A process killed outright never updates its own status, so skipping anything whose record reads running would make every crashed job IMMORTAL. Those are exactly the jobs that accumulate, so the naive reading of the rule produces the opposite of retention: the only files that never go away are the ones nobody wants. + +VERIFIED BOTH DIRECTIONS. Live: started a 30-second job, immediately spawned another with SR_JOB_KEEP=1, and the running job survived alongside the new one. Pinned in tooling/test_jobs.py with a fixture carrying four jobs — a finished one, a corpse (status running, dead pid), a genuinely alive one (status running, os.getpid()), and another finished one — asserting the live one survives AND the corpse is removed. +Proven to fail: dropping the liveness check made the test report the immortal-corpse case by name. + +A FALSE ALARM WORTH RECORDING, because it looked exactly like the bug. My first live test appeared to show a running job being pruned. It was not: my commands ran two minutes apart, so the "20-second" job had long finished before the prune happened. The test was invalid, not the guard. Re-run in a single command with no gap, it behaved correctly. Worth noting that a multi-turn shell makes any timing-sensitive check unreliable unless the whole sequence is one command.', NULL, '2026-08-31 15:27:12', '2026-08-31 15:27:12.804', '2026-08-31 15:27:12.804', NULL, 'fa9421f34a00276707a264ea4341d330', 2) ON CONFLICT(hash) DO NOTHING; diff --git a/.pql/changelog/tickets/2026-08.sql b/.pql/changelog/tickets/2026-08.sql index 0f4daf198..0cf1d699b 100644 --- a/.pql/changelog/tickets/2026-08.sql +++ b/.pql/changelog/tickets/2026-08.sql @@ -2331,3 +2331,38 @@ CLOSED T-1278''s OPEN GAP as a side effect. `jobs log --follow` had never been e PROVEN TO FAIL: made effective_exit_code always return 0 — the exit-0 trap itself — and both the fast and slow cases failed by name, reporting the expected code against the 0 they got. Restored, green. NOTE ON WHAT --follow RETURNS: `jobs log --follow` exits 0 even when the job it tailed failed. That is correct and deliberate — log succeeded at logging. `wait` is the verb that relays an outcome, and conflating the two would mean you could not read a failed job''s log without your shell treating the read as a failure.', 'in_progress', 'high', NULL, NULL, 'D-263', '2026-08-31 13:52:15.747', '2026-08-31 15:14:09.188', NULL, '3bd99b15e59df3b447ab404740ee63fa', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5WNGDMENP7AY9VMB5G3XM', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'Exit codes survive a detached run — the non-negotiable', 'A runner that reports started and then loses the failure is the exit-0 trap from the top of D-263 relocated somewhere nothing is watching, which is strictly worse than the original: at least a foreground exit 0 is in front of someone. Requirements: reach jobs wait EXITS WITH THE JOB EXIT CODE, so a Makefile or hook can gate on a detached run exactly as it would on a foreground one; a failed job that nobody waited on is visible as failed in reach jobs list rather than merely absent; and reach --detach itself exits 0 for SUCCESSFULLY STARTING, which is a different claim from the job succeeding, so the output must say so in words rather than leaving a reader to infer it. TEST THE FAILING PATH FIRST and give it a deliberately failing command, since a job runner that has only ever run successful jobs has never been tested — this is the same discipline that caught the gate tests. Also test the case where the job fails BEFORE the parent has exited, and where it fails long AFTER, since those exercise different halves of the recording path. + +DONE 2026-08-31. All four properties verified and pinned in tooling/test_job_exit_codes.py, wired into make test-tooling. + +BUILT A REAL VERB TO TEST WITH, rather than a test hook. The two timing cases this ticket names — fails BEFORE the parent exits, fails long AFTER — cannot be distinguished when every command in reach finishes in milliseconds. So `reach dev selftest --seconds N --fail --exit-code C` now exists: it emits progress events over N seconds then optionally fails. It is a genuine diagnostic ("does streaming work end to end on this machine, can I tail it, does a failure survive detach?"), it lives in the `dev` domain the map already planned, and it is the only command slow enough to answer those by observation instead of argument. A test-only hook would have been the same code with less value. + +VERIFIED, and the slow case is the one that proves the design: +- FAST failure (0s): wait relays exit 3. +- SLOW failure (6s real / 2s in the test): --detach returned in 75 ms while the child ran 6 seconds, so the parent was demonstrably gone long before the child failed — and wait still relayed exit 7. That is the half of the recording path only this case reaches, and it is why T-1277 moved completion recording to the child. +- SUCCESS: wait relays 0. +- A failed job nobody waited on shows as `failed` in jobs list with its full command. +- --detach exits 0 AND says "this exit status means STARTED, not succeeded" in words. The test asserts on that wording, not just the code, because a bare 0 invites exactly the wrong reading. +- Every event in a detached job''s log carries the job id. + +CLOSED T-1278''s OPEN GAP as a side effect. `jobs log --follow` had never been exercised against a genuinely long job because none existed. With selftest it now has been: attached to a running 6-second job mid-flight, streamed the remaining steps live as they happened, and caught the final failure verdict with its remedy after the job ended. The follow loop and its final post-completion read both work. + +PROVEN TO FAIL: made effective_exit_code always return 0 — the exit-0 trap itself — and both the fast and slow cases failed by name, reporting the expected code against the 0 they got. Restored, green. + +NOTE ON WHAT --follow RETURNS: `jobs log --follow` exits 0 even when the job it tailed failed. That is correct and deliberate — log succeeded at logging. `wait` is the verb that relays an outcome, and conflating the two would mean you could not read a failed job''s log without your shell treating the read as a failure.', 'done', 'high', NULL, NULL, 'D-263', '2026-08-31 13:52:15.747', '2026-08-31 15:14:26.448', NULL, 'fab9a47c11bfc1072bf94ddfd0a74c97', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5XC1XX5179S0X6VFW0TM0', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'Job log retention, so .cache/reach/jobs does not grow forever', 'An append-only log directory with no retention is a disk leak that nobody notices until it matters. Deliverables: a cap by count or age, applied at write time rather than by a separate sweep — a cleanup nothing invokes is a cleanup that does not happen; reach jobs prune as the explicit escape hatch; and a rule that a RUNNING job is never pruned regardless of age, since a long generator can outlive the window. Bound the whole thing to .cache/, which is gitignored and already the repo''s scratch space, so a wrong answer here costs disk rather than data. Deliberately last in the epic: retention pressure is not real until jobs exist and the log sizes are observable, and guessing a cap before seeing a Blender batch''s output volume would just be a number to change later.', 'in_progress', 'low', NULL, NULL, 'D-263', '2026-08-31 13:52:21.519', '2026-08-31 15:15:08.418', NULL, 'ce3d53e7205ef41b98745a593449f0c6', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5XC1XX5179S0X6VFW0TM0', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'Job log retention, so .cache/reach/jobs does not grow forever', 'An append-only log directory with no retention is a disk leak that nobody notices until it matters. Deliverables: a cap by count or age, applied at write time rather than by a separate sweep — a cleanup nothing invokes is a cleanup that does not happen; reach jobs prune as the explicit escape hatch; and a rule that a RUNNING job is never pruned regardless of age, since a long generator can outlive the window. Bound the whole thing to .cache/, which is gitignored and already the repo''s scratch space, so a wrong answer here costs disk rather than data. Deliberately last in the epic: retention pressure is not real until jobs exist and the log sizes are observable, and guessing a cap before seeing a Blender batch''s output volume would just be a number to change later.', 'in_progress', 'low', NULL, NULL, 'D-263', '2026-08-31 13:52:21.519', '2026-08-31 15:15:15.933', NULL, '8d8c5bea8bb17815a8804663d4e0980e', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06G5G5XC1XX5179S0X6VFW0TM0', 'task', '06G1S7NVJR0GT9KWS9QVYNNFMM', 'Job log retention, so .cache/reach/jobs does not grow forever', 'An append-only log directory with no retention is a disk leak that nobody notices until it matters. Deliverables: a cap by count or age, applied at write time rather than by a separate sweep — a cleanup nothing invokes is a cleanup that does not happen; reach jobs prune as the explicit escape hatch; and a rule that a RUNNING job is never pruned regardless of age, since a long generator can outlive the window. Bound the whole thing to .cache/, which is gitignored and already the repo''s scratch space, so a wrong answer here costs disk rather than data. Deliberately last in the epic: retention pressure is not real until jobs exist and the log sizes are observable, and guessing a cap before seeing a Blender batch''s output volume would just be a number to change later. + +DONE 2026-08-31. Retention applied at write time, plus reach jobs prune as the explicit escape hatch. + +MEASURED BEFORE CHOOSING A NUMBER, which is why this ticket was scheduled last. A chatty short job (10 progress events) writes ~1.8 KB across its three files; 20 jobs came to 164 KB. A generator emitting per-body progress might reach 100 KB, so a cap of 100 jobs is single-digit megabytes at worst, inside .cache/ which is gitignored scratch. The failure mode of that number being wrong is disk, never data. Overridable with SR_JOB_KEEP, and worth raising once a real Blender batch has run. + +APPLIED AT SPAWN TIME, not on a schedule — a retention pass that depends on someone remembering to run it is one that silently never happens. reach jobs prune --keep N exists as the deliberate escape hatch for reclaiming space now, or --keep 0 to clear everything. + +THE SUBTLETY THE TICKET''S WORDING HIDES, and it is the whole of the design. "A running job is never pruned" has to mean GENUINELY running, checked against the process table — not "the file says running". A process killed outright never updates its own status, so skipping anything whose record reads running would make every crashed job IMMORTAL. Those are exactly the jobs that accumulate, so the naive reading of the rule produces the opposite of retention: the only files that never go away are the ones nobody wants. + +VERIFIED BOTH DIRECTIONS. Live: started a 30-second job, immediately spawned another with SR_JOB_KEEP=1, and the running job survived alongside the new one. Pinned in tooling/test_jobs.py with a fixture carrying four jobs — a finished one, a corpse (status running, dead pid), a genuinely alive one (status running, os.getpid()), and another finished one — asserting the live one survives AND the corpse is removed. +Proven to fail: dropping the liveness check made the test report the immortal-corpse case by name. + +A FALSE ALARM WORTH RECORDING, because it looked exactly like the bug. My first live test appeared to show a running job being pruned. It was not: my commands ran two minutes apart, so the "20-second" job had long finished before the prune happened. The test was invalid, not the guard. Re-run in a single command with no gap, it behaved correctly. Worth noting that a multi-turn shell makes any timing-sensitive check unreliable unless the whole sequence is one command.', 'in_progress', 'low', NULL, NULL, 'D-263', '2026-08-31 13:52:21.519', '2026-08-31 15:27:12.804', NULL, '7e4e9641ba882047bca763ced09c7500', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at; diff --git a/tooling/core/process.py b/tooling/core/process.py index ac4dea368..dae8d89fd 100644 --- a/tooling/core/process.py +++ b/tooling/core/process.py @@ -111,9 +111,54 @@ def spawn_detached(argv: list[str]) -> str: "status": "running", }, ) + + # At write time, not on a schedule. A retention pass that depends on someone + # remembering to run it is one that silently never happens. + prune() return job_id +# Measured 2026-08-31: a chatty short job (10 progress events) writes ~1.8 KB +# across its three files. A generator emitting per-body progress might reach +# 100 KB. 100 jobs is therefore single-digit megabytes at worst, inside .cache/ +# which is gitignored scratch — so the failure mode of this number being wrong +# is disk, never data. Raise it freely if a real batch proves it tight. +DEFAULT_KEEP = 100 + +ENV_KEEP = "SR_JOB_KEEP" + + +def prune(keep: int | None = None) -> list[str]: + """Delete all but the `keep` most recent jobs. Returns the ids removed. + + **A genuinely running job is never pruned** — but note what that has to + mean. Skipping anything whose *file* says "running" would make every + crashed job immortal, because a process killed outright never gets to + update its own status. Those are precisely the jobs that accumulate, so the + check is against the process table, not the record. + + Called at spawn time rather than by a sweep: a cleanup nothing invokes is a + cleanup that does not happen. + """ + if keep is None: + raw = os.environ.get(ENV_KEEP) + keep = int(raw) if raw and raw.isdigit() else DEFAULT_KEEP + + directory = jobs_dir() + # Ids sort chronologically because they are timestamp-first (T-1276). + ids = sorted((path.stem for path in directory.glob("*.json")), reverse=True) + + removed: list[str] = [] + for job_id in ids[keep:]: + meta = read_meta(job_id) + if meta and meta.get("status") == "running" and is_alive(int(meta.get("pid", -1))): + continue # actually running — a long generator may outlive the window + for path in (log_path(job_id), output_path(job_id), meta_path(job_id)): + path.unlink(missing_ok=True) + removed.append(job_id) + return removed + + def finish_if_detached(exit_code: int) -> None: """Record completion — called by the CHILD, from the outermost decorator. diff --git a/tooling/domains/jobs/router.py b/tooling/domains/jobs/router.py index ce1eda659..2b7c343bd 100644 --- a/tooling/domains/jobs/router.py +++ b/tooling/domains/jobs/router.py @@ -10,7 +10,7 @@ import time import typer -from tooling.core import cli, console +from tooling.core import cli, console, process from tooling.core.command import command from tooling.core.errors import ReachExit from tooling.domains.jobs import service @@ -93,6 +93,26 @@ def log( console.out(console.render(event).rstrip("\n")) +@app.command("prune") +@command +def prune( + keep: int = typer.Option( + process.DEFAULT_KEEP, "--keep", help="How many recent jobs to keep." + ), +) -> None: + """Delete old job logs. Running jobs are kept regardless of age. + + Pruning also happens automatically whenever a job is started, so this is the + explicit escape hatch rather than the only path — reach for it to reclaim + space now, or with --keep 0 to clear everything. + """ + removed = service.prune(keep) + if not removed: + console.verdict(f"nothing to prune — {keep} or fewer jobs on record") + return + console.verdict(f"pruned {len(removed)} job(s), kept the {keep} most recent") + + @app.command("wait") @command def wait( diff --git a/tooling/domains/jobs/service.py b/tooling/domains/jobs/service.py index 034f4adfc..fdcd107c3 100644 --- a/tooling/domains/jobs/service.py +++ b/tooling/domains/jobs/service.py @@ -96,6 +96,22 @@ def wait(job_id: str, timeout: float | None = None) -> Job: time.sleep(POLL_SECONDS) +def prune(keep: int) -> list[str]: + """Delete all but the `keep` most recent jobs; return the ids removed. + + Genuinely running jobs survive regardless of age — see `process.prune`, + which checks the process table rather than the recorded status so that a + crashed job is prunable and a live generator is not. + """ + if keep < 0: + raise ReachError( + f"cannot keep {keep} jobs", + fix="pass --keep 0 to remove everything, or a positive number to keep some", + exit_code=2, + ) + return process.prune(keep) + + def duration(job: Job) -> str: """Human-readable elapsed time, or how long it has been running so far.""" start = _parse(job.started_at) diff --git a/tooling/test_jobs.py b/tooling/test_jobs.py index cd830eb10..75ccf711d 100644 --- a/tooling/test_jobs.py +++ b/tooling/test_jobs.py @@ -110,12 +110,85 @@ def test_died_never_relays_success(failures: list[str]) -> None: failures.append("failed job did not relay its own exit code") +def _fixture_jobs(root: Path, specs: list[tuple[str, str, int]]) -> None: + """Write job metadata into a throwaway repo root. specs: (id, status, pid).""" + directory = root / ".cache" / "reach" / "jobs" + directory.mkdir(parents=True, exist_ok=True) + for job_id, status, pid in specs: + (directory / f"{job_id}.json").write_text( + json.dumps( + { + "job": job_id, + "command": "reach dev selftest", + "argv": ["dev", "selftest"], + "pid": pid, + "started_at": "2026-01-01T00:00:00", + "status": status, + } + ), + encoding="utf-8", + ) + (directory / f"{job_id}.jsonl").write_text("", encoding="utf-8") + + +def test_prune_spares_running_but_not_corpses(failures: list[str]) -> None: + """The retention rule, and the half of it that is easy to get wrong. + + A job that is genuinely running survives pruning however old it is — a + generator can outlive the window. But a job whose FILE says running while + its process is gone must be prunable, or every crashed job becomes immortal + and those are exactly what accumulates. The check has to consult the process + table, not the record. + """ + import os + + from tooling.core import process + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "project.yaml").write_text("version: 0.0.0\n", encoding="utf-8") + _fixture_jobs( + root, + [ + ("20260101T000001-aaaa", "done", 1), + ("20260101T000002-bbbb", "running", 4_000_000), # corpse: dead pid + ("20260101T000003-cccc", "running", os.getpid()), # genuinely alive + ("20260101T000004-dddd", "done", 1), + ], + ) + + previous = os.environ.get("SR_REPO_ROOT") + os.environ["SR_REPO_ROOT"] = str(root) + try: + removed = set(process.prune(keep=1)) + finally: + if previous is None: + del os.environ["SR_REPO_ROOT"] + else: + os.environ["SR_REPO_ROOT"] = previous + + if "20260101T000003-cccc" in removed: + failures.append( + "prune: removed a job whose process is alive — a long generator " + "would lose its own log while still writing to it" + ) + if "20260101T000002-bbbb" not in removed: + failures.append( + "prune: kept a job whose file says running but whose process is " + "gone — a crashed job must not be immortal, and those are exactly " + "the ones that accumulate" + ) + if "20260101T000001-aaaa" not in removed: + failures.append("prune: kept a finished job beyond the keep window") + + def main() -> int: failures: list[str] = [] test_partial_trailing_line(failures) test_offset_resume_is_stable(failures) test_dead_pid_is_reconciled(failures) test_died_never_relays_success(failures) + test_prune_spares_running_but_not_corpses(failures) if failures: print("test_jobs: FAIL", file=sys.stderr)