settled-reach/tooling/domains/dev/perf.py

#!/usr/bin/env python3
"""Performance baseline — tick timing, memory, and shadowcast scaling.

Builds the server in release, runs the benchmark suite, and either writes
tests/perf/baseline.json or compares against the committed one. The compare
path exits non-zero on a >20% regression or a p95 over the D-026 tick budget.
"""

from __future__ import annotations

import json
import re
from datetime import datetime, timezone

from tooling.core import config, console, process
from tooling.core.errors import ReachError

ROOT = config.repo_root()
PERF_DIR = ROOT / "tests" / "perf"
BASELINE_FILE = PERF_DIR / "baseline.json"

# Tick budget from D-026: 100ms per tick at 10 tps floor (D-031).
# If tick rate changes, update this constant.
TICK_BUDGET_US = 100_000  # 100ms


def run_command(cmd, **kwargs):
    """Run a command in the server directory and return the result."""
    return process.run(cmd, cwd=ROOT / "server", check=False, **kwargs)


def get_git_info():
    """Current commit and branch. check=False — a shallow or detached tree is
    not an error here, it just means the stamp carries less."""
    commit = process.run(
        ["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, check=False
    ).stdout.strip()
    branch = process.run(
        ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=ROOT, check=False
    ).stdout.strip()
    return {"commit": commit, "branch": branch}


def run_tick_benchmark():
    """Run perf_tick_timing test and parse PERF_RESULT JSON."""
    console.event("running tick timing benchmark (release mode)", phase="bench")
    result = run_command([
        "cargo", "test", "--release", "--test", "perf_bench",
        "--", "--ignored", "--nocapture", "perf_tick_timing",
    ])

    if result.returncode != 0:
        console.event(f"tick benchmark exited {result.returncode}", level="error")
        if result.stderr:
            # Last 20 lines of stderr are the diagnostic; the rest is build noise.
            for line in result.stderr.strip().splitlines()[-20:]:
                console.event(line, level="error")
        return None

    # Parse PERF_RESULT: line from stdout
    for line in result.stdout.splitlines():
        if line.startswith("PERF_RESULT:"):
            json_str = line[len("PERF_RESULT:"):]
            return json.loads(json_str)

    console.event("no PERF_RESULT found in test output", level="warn")
    return None


def run_shadowcast_benchmark():
    """Run shadowcast benchmark and parse structured output."""
    console.event("running shadowcast benchmark (release mode)", phase="bench")
    result = run_command([
        "cargo", "test", "--release", "--test", "shadowcast_bench",
        "--", "--ignored", "--nocapture", "benchmark_symmetric_vs_recursive",
    ])

    if result.returncode != 0:
        console.event(
            f"shadowcast benchmark exited {result.returncode}", level="error"
        )
        return None

    configs = []
    current = {}
    for line in result.stdout.splitlines():
        line = line.strip()

        m = re.match(
            r"Map: (\d+)x(\d+), Density: (.+), Range: (\d+), Iterations: (\d+)",
            line,
        )
        if m:
            # New config block — flush previous if complete
            if current.get("map_size"):
                configs.append(current)
            current = {
                "map_size": int(m.group(1)),
                "density": m.group(3),
                "range": int(m.group(4)),
                "iterations": int(m.group(5)),
            }
            continue

        m = re.match(r"Symmetric:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line)
        if m:
            current["symmetric_total_ms"] = float(m.group(1))
            current["symmetric_per_call_us"] = float(m.group(2))
            continue

        m = re.match(r"Recursive:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line)
        if m:
            current["recursive_total_ms"] = float(m.group(1))
            current["recursive_per_call_us"] = float(m.group(2))
            continue

    # Flush last config
    if current.get("map_size"):
        configs.append(current)

    return {"configs": configs} if configs else None


def compare_baselines(old, new):
    """Compare two baselines and report regressions. Returns list of regression strings."""
    regressions = []
    improvements = []

    old_tick = old.get("tick_timing", {})
    new_tick = new.get("tick_timing", {})

    if old_tick and new_tick:
        # Mean tick time regression (>20% = warning)
        old_mean = old_tick.get("mean_us", 0)
        new_mean = new_tick.get("mean_us", 0)
        if old_mean > 0:
            change = (new_mean - old_mean) / old_mean * 100
            if change > 20:
                regressions.append(
                    f"mean tick time {old_mean}us -> {new_mean}us (+{change:.1f}%)"
                )
            elif change < -20:
                improvements.append(
                    f"mean tick time {old_mean}us -> {new_mean}us ({change:.1f}%)"
                )

        # p95 tick time regression
        old_p95 = old_tick.get("p95_us", 0)
        new_p95 = new_tick.get("p95_us", 0)
        if old_p95 > 0:
            change = (new_p95 - old_p95) / old_p95 * 100
            if change > 20:
                regressions.append(
                    f"p95 tick time {old_p95}us -> {new_p95}us (+{change:.1f}%)"
                )
            elif change < -20:
                improvements.append(
                    f"p95 tick time {old_p95}us -> {new_p95}us ({change:.1f}%)"
                )

    # Absolute budget check
    new_p95 = new.get("tick_timing", {}).get("p95_us", 0)
    if new_p95 > TICK_BUDGET_US:
        regressions.append(
            f"p95 {new_p95}us exceeds {TICK_BUDGET_US}us tick budget (D-026)"
        )

    return regressions, improvements


def run(compare_mode: bool = False) -> None:
    """Build release, run the benchmarks, then write or compare the baseline."""
    console.event("building server (release)", phase="build")
    build = run_command(["cargo", "build", "--release"])
    if build.returncode != 0:
        for line in build.stderr.strip().splitlines()[-20:]:
            console.event(line, level="error")
        raise ReachError(
            "the release build failed, so there is nothing to benchmark",
            fix="fix the build errors above, then re-run — a perf number from a "
            "stale binary is worse than no number",
        )

    tick_results = run_tick_benchmark()
    shadowcast_results = run_shadowcast_benchmark()

    if not tick_results:
        raise ReachError(
            "the tick benchmark produced no result — no baseline generated",
            fix="run `cargo test --release --test perf_bench -- --ignored "
            "--nocapture perf_tick_timing` in server/ to see why it failed",
        )

    # Assemble baseline
    git_info = get_git_info()
    baseline = {
        "timestamp": datetime.now(timezone.utc).isoformat(),
        "git": git_info,
        "tick_timing": tick_results.get("tick_timing", {}),
        "entities": tick_results.get("entities", {}),
        "memory": tick_results.get("memory", {}),
    }
    if shadowcast_results:
        baseline["shadowcast"] = shadowcast_results

    # Report
    tt = baseline["tick_timing"]
    console.out("--- Results ---")
    console.out(f"Git: {git_info['commit']} ({git_info['branch']})")
    console.out(f"Tick timing ({tt.get('measured_ticks', '?')} ticks, "
                f"{tt.get('warmup_ticks', '?')} warmup):")
    console.out(f"  min:  {tt.get('min_us', '?')}us")
    console.out(f"  mean: {tt.get('mean_us', '?')}us")
    console.out(f"  p95:  {tt.get('p95_us', '?')}us")
    console.out(f"  max:  {tt.get('max_us', '?')}us")

    ent = baseline["entities"]
    console.out(f"Entities: avg {ent.get('avg_per_snapshot', '?')}, "
                f"max {ent.get('max_per_snapshot', '?')}")

    rss = baseline["memory"].get("rss_kb")
    if rss:
        console.out(f"Memory: {rss} KB RSS ({rss / 1024:.1f} MB)")

    if shadowcast_results:
        n = len(shadowcast_results.get("configs", []))
        console.out(f"Shadowcast: {n} configurations benchmarked")

    # Budget check
    p95 = tt.get("p95_us", 0)
    if p95 > TICK_BUDGET_US:
        console.out(f"BUDGET EXCEEDED: p95 {p95}us > {TICK_BUDGET_US}us (D-026)")
    else:
        budget_pct = p95 / TICK_BUDGET_US * 100 if TICK_BUDGET_US else 0
        console.out(
            f"Budget: {budget_pct:.1f}% of {TICK_BUDGET_US}us tick budget (D-026)"
        )

    # Compare with previous baseline if it exists
    regressions: list[str] = []
    if BASELINE_FILE.exists():
        old_baseline = json.loads(BASELINE_FILE.read_text())
        old_commit = old_baseline.get("git", {}).get("commit", "?")
        console.out(f"--- Comparison vs {old_commit} ---")
        regressions, improvements = compare_baselines(old_baseline, baseline)
        for r in regressions:
            console.out(f"  REGRESSION: {r}")
        for i in improvements:
            console.out(f"  IMPROVEMENT: {i}")
        if not regressions and not improvements:
            console.out("  No significant changes.")
    elif compare_mode:
        raise ReachError(
            f"--compare needs a saved baseline at {BASELINE_FILE.relative_to(ROOT)}",
            fix="run `reach dev perf` once without --compare to create one",
        )

    if compare_mode:
        if regressions:
            raise ReachError(
                f"{len(regressions)} performance regression(s) detected",
                fix="see the REGRESSION lines above; if the change is intended, "
                "re-run without --compare to accept it as the new baseline",
            )
        console.verdict("perf: no regressions against the saved baseline")
        return

    # Save baseline (strip per-tick array — too noisy for git diffs)
    PERF_DIR.mkdir(parents=True, exist_ok=True)
    committed = json.loads(json.dumps(baseline))
    committed["tick_timing"].pop("all_us", None)
    BASELINE_FILE.write_text(json.dumps(committed, indent=2) + "\n")

    console.verdict(f"baseline written to {BASELINE_FILE.relative_to(ROOT)}")