settled-reach/tooling/domains/dev/clerk.py

#!/usr/bin/env python3
"""
Clerk pre-push review — checks D-record consistency, ticket drift, and decision
contradictions against the diff that would be pushed.

Reviewed one commit at a time — commits are the logical units (each /git-commit is
one coherent change), so each clerk agent reviews a self-contained change *and its
commit message*. Commits are reviewed by a bounded pool of parallel clerk agents
and the per-commit verdicts aggregated.

Three outcomes per commit:
  APPROVED    no contradiction found.
  REJECTED    a concrete contradiction with a named, active D-record. HARD BLOCK.
  INCOMPLETE  the review could not finish — timed out, ran out of turns, or emitted
              no clear verdict. This is NOT a contradiction; it is non-blocking by
              default (the push proceeds with a warning). Raise the budget knobs or
              add a `Clerk-Skip:` trailer to avoid it.

Why INCOMPLETE exists: an unfinished review must never read as "hard contradiction
found." The previous version defaulted no-verdict / max-turns to REJECTED, which
turned every slow review into a false block (e.g. a CHANGELOG-only commit).

Safety valve: a commit message with a `Clerk-Skip:` trailer line is auto-approved
without spawning an agent — for bulk content commits (e.g. shipping thousands of
generated planetary description files) where D-record review is moot. A *trailer*
(a line starting with `Clerk-Skip:`) is required, so merely mentioning the token in
prose or a subject line does not trip the valve.

Outputs exactly one word as the LAST stdout line: APPROVED, REJECTED, or INCOMPLETE.
Writes verbose findings to .cache/pre-push-review.md.

Exit codes:
  0 = APPROVED or INCOMPLETE (non-blocking)
  1 = REJECTED (>=1 commit contradicts an active D-record)

Env knobs:
  SR_CLERK_COMMIT_BUDGET  per-commit diff char cap   (default 120000, truncates)
  SR_CLERK_WORKERS        parallel clerk agents       (default 4)
  SR_CLERK_MAX_TURNS      turns per clerk agent       (default 15)
  SR_CLERK_TIMEOUT        per-commit timeout seconds  (default 300)

Usage:
  reach dev clerk            run the review, print verdict
  reach dev clerk --plan     print the per-commit plan only (no clerk spawned)
"""

from __future__ import annotations

import os
import re
from concurrent.futures import ThreadPoolExecutor, as_completed

from tooling.core import config, console, process
from tooling.core.errors import ReachError

REPO_ROOT = config.repo_root()

CACHE_DIR = REPO_ROOT / ".cache"
FINDINGS_FILE = CACHE_DIR / "pre-push-review.md"

COMMIT_BUDGET = int(os.environ.get("SR_CLERK_COMMIT_BUDGET", "120000"))
WORKERS = int(os.environ.get("SR_CLERK_WORKERS", "4"))
MAX_TURNS = os.environ.get("SR_CLERK_MAX_TURNS", "15")
TIMEOUT_SECONDS = int(os.environ.get("SR_CLERK_TIMEOUT", "300"))

# Safety valve: a commit message with a "Clerk-Skip:" trailer line is auto-approved
# (no agent). A trailer (line-start) is required so that merely mentioning the token
# in prose or a subject line does not trip the valve.
SKIP_TRAILER = re.compile(r"(?im)^[ \t]*clerk-skip[ \t]*:")

CLERK_PROMPT = """You are the CLERK, the institutional guardrail for The Settled Reach.

You are reviewing ONE COMMIT ({label}) from a larger pre-push for D-record
consistency, ticket drift, and decision contradictions.

## governance/ domain index

{index}

## The commit (message + diff)

{diff}

## Your task

Be efficient — a few targeted greps of governance/decisions/*.md, then conclude. Check:
- Does this commit contradict any active D-record?
- If the commit's code/text cites a D/Q/R-ID, does that ID exist and is it active?
- If the commit message cites ticket #NNN, does the change match the ticket?
Note any relevant open Q-records.

## Output format (REQUIRED)

Write brief findings, then on the VERY LAST LINE output exactly one word:
APPROVED or REJECTED.

- REJECTED *only* for a concrete contradiction with a named, active D-record.
- Drift, stale references, open Q-records, and suggestions are findings, NOT blocks -> APPROVED.
- If you did not find a concrete contradiction, output APPROVED — even if you could
  not check everything. "Unsure / didn't finish" means APPROVED, never REJECTED.
"""


def _git(*args: str) -> str:
    return process.run(["git", *args], cwd=REPO_ROOT).stdout


def base_range() -> str:
    """The commit range that would be pushed (base..HEAD)."""
    branch = _git("branch", "--show-current").strip()
    for ref in [f"origin/{branch}", "origin/main"]:
        probe = process.run(
            ["git", "rev-parse", "--verify", ref], cwd=REPO_ROOT, check=False
        )
        if probe.returncode == 0:
            return f"{ref}..HEAD"
    return "HEAD~1..HEAD"


def list_commits(rng: str) -> list[str]:
    """SHAs in the push range, oldest first."""
    return [s for s in _git("rev-list", "--reverse", rng).splitlines() if s.strip()]


def commit_unit(sha):
    """Return (subject, text, skip) for one commit.

    `skip` is True when the commit message has a `Clerk-Skip:` trailer (the safety
    valve); `text` is the message + diff, truncated to COMMIT_BUDGET.
    """
    subject = _git("show", "-s", "--format=%h %s", sha).strip()
    message = _git("show", "-s", "--format=%B", sha)
    skip = bool(SKIP_TRAILER.search(message))
    text = _git("show", "--format=fuller", sha)
    if len(text) > COMMIT_BUDGET:
        text = text[:COMMIT_BUDGET] + f"\n\n... (commit diff truncated; {len(text)} total chars)\n"
    return subject, text, skip


def run_clerk(label, diff_text, index):
    """Spawn one clerk agent over a commit; return (verdict, findings).

    verdict is APPROVED, REJECTED, or INCOMPLETE. INCOMPLETE covers timeout, empty
    output, and no-clear-verdict — none of which is a contradiction.
    """
    prompt = CLERK_PROMPT.format(label=label, index=index, diff=diff_text)
    try:
        result = process.run(
            ["claude", "-p", "--model", "sonnet", "--max-turns", str(MAX_TURNS)],
            cwd=REPO_ROOT,
            input=prompt,
            timeout=TIMEOUT_SECONDS,
            check=False,
            missing_fix="install the claude CLI, or set SR_CLERK=0 to skip the review",
        )
        output = result.stdout.strip()
    except process.ProcessTimeout:
        return "INCOMPLETE", f"{label}: review timed out after {TIMEOUT_SECONDS}s (not a contradiction)."

    if not output:
        return "INCOMPLETE", f"{label}: clerk produced no output (not a contradiction)."

    last_line = output.splitlines()[-1].strip().upper()
    if last_line == "APPROVED":
        return "APPROVED", output
    if last_line == "REJECTED":
        return "REJECTED", output
    return "INCOMPLETE", output + "\n\n(No clear verdict on last line — recorded as INCOMPLETE, not a contradiction.)"


def load_index() -> str:
    """The decision-domain index the clerk greps from.

    `governance/README.md` since the DQR tree moved; the old `decisions/README.md`
    path silently resolved to "not found", which handed every clerk an empty
    index and made it grep blind.
    """
    readme = REPO_ROOT / "governance" / "README.md"
    return readme.read_text()[:8000] if readme.exists() else "(governance/README.md not found)"


def run(plan_only: bool = False) -> str:
    """Review the push range commit by commit. Returns the overall verdict."""
    CACHE_DIR.mkdir(parents=True, exist_ok=True)

    rng = base_range()
    commits = list_commits(rng)
    if not commits:
        FINDINGS_FILE.write_text("# Clerk Review\n\nNo commits to review.\n\nVerdict: APPROVED\n")
        console.event("clerk: no commits to review")
        console.out("APPROVED")
        return "APPROVED"

    units = [commit_unit(sha) for sha in commits]  # [(subject, text, skip), ...]

    if plan_only:
        console.out(f"clerk plan: {len(commits)} commit(s) over {rng} "
                    f"(budget {COMMIT_BUDGET} chars, {WORKERS} workers, "
                    f"{MAX_TURNS} turns each)")
        for i, (subject, text, skip) in enumerate(units):
            tag = "SKIP (Clerk-Skip:)" if skip else "review"
            console.out(
                f"  commit {i + 1}/{len(commits)}: {len(text):>9} chars  [{tag}]  {subject}"
            )
        return "PLAN"

    index = load_index()
    reviewable = sum(1 for _, _, skip in units if not skip)
    console.event(
        f"clerk: {len(commits)} commit(s) — {reviewable} to review, "
        f"{len(commits) - reviewable} auto-approved (Clerk-Skip:); {WORKERS} parallel",
        phase="clerk",
    )

    def task(i, subject, text, skip):
        label = f"commit {i + 1}/{len(commits)} ({subject})"
        if skip:
            return i, label, "APPROVED", f"{label}: auto-approved via Clerk-Skip: trailer."
        verdict, findings = run_clerk(label, text, index)
        return i, label, verdict, findings

    results = [None] * len(commits)
    with ThreadPoolExecutor(max_workers=WORKERS) as ex:
        futures = [ex.submit(task, i, s, t, skip) for i, (s, t, skip) in enumerate(units)]
        for fut in as_completed(futures):
            i, label, verdict, findings = fut.result()
            results[i] = (label, verdict, findings)
            console.event(f"clerk: {label} — {verdict}", phase="clerk")

    verdicts = [r[1] for r in results]
    if "REJECTED" in verdicts:
        overall = "REJECTED"
    elif "INCOMPLETE" in verdicts:
        overall = "INCOMPLETE"
    else:
        overall = "APPROVED"

    n_rej = verdicts.count("REJECTED")
    n_inc = verdicts.count("INCOMPLETE")
    parts = [
        f"# Clerk Pre-Push Review\n\n**Overall verdict: {overall}**\n",
        f"Reviewed {len(commits)} commit(s) over {rng} — "
        f"{verdicts.count('APPROVED')} approved, {n_rej} rejected, {n_inc} incomplete.\n",
    ]
    for label, verdict, findings in results:
        parts.append(f"\n---\n\n## {label} — {verdict}\n\n{findings}\n")
    FINDINGS_FILE.write_text("\n".join(parts))

    console.event(
        f"clerk: overall verdict — {overall} (details: {FINDINGS_FILE})", phase="clerk"
    )
    console.out(overall)

    if overall == "REJECTED":
        raise ReachError(
            f"clerk rejected {n_rej} commit(s) — a named, active D-record is contradicted",
            fix=f"read {FINDINGS_FILE.relative_to(REPO_ROOT)}; amend the commit or "
            "the decision, or add a `Clerk-Skip:` trailer if the clerk is wrong",
        )
    return overall