settled-reach/tooling/perf-baseline

#!/usr/bin/env python3
"""Performance baseline tooling.

Runs the server benchmark suite, captures tick timing, memory usage, and entity
count scaling metrics, outputs results to tests/perf/.

Usage:
  tooling/perf-baseline             Run benchmarks and save baseline
  tooling/perf-baseline --compare   Compare current run against saved baseline (no save)

Exit code 0 = success, 1 = failure or regression detected (--compare mode).
"""

import json
import re
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
PERF_DIR = ROOT / "tests" / "perf"
BASELINE_FILE = PERF_DIR / "baseline.json"

# Tick budget from D-026: 100ms per tick at 10 tps floor (D-031).
# If tick rate changes, update this constant.
TICK_BUDGET_US = 100_000  # 100ms


def run_command(cmd, **kwargs):
    """Run a command in the server directory and return the result."""
    return subprocess.run(
        cmd, capture_output=True, text=True, cwd=ROOT / "server", **kwargs
    )


def get_git_info():
    """Get current git commit and branch."""
    commit = subprocess.run(
        ["git", "rev-parse", "--short", "HEAD"],
        capture_output=True, text=True, cwd=ROOT,
    ).stdout.strip()
    branch = subprocess.run(
        ["git", "rev-parse", "--abbrev-ref", "HEAD"],
        capture_output=True, text=True, cwd=ROOT,
    ).stdout.strip()
    return {"commit": commit, "branch": branch}


def run_tick_benchmark():
    """Run perf_tick_timing test and parse PERF_RESULT JSON."""
    print("  Running tick timing benchmark (release mode)...")
    result = run_command([
        "cargo", "test", "--release", "--test", "perf_bench",
        "--", "--ignored", "--nocapture", "perf_tick_timing",
    ])

    if result.returncode != 0:
        print(f"  FAILED: tick benchmark exited {result.returncode}")
        if result.stderr:
            # Print last 20 lines of stderr for diagnostics
            lines = result.stderr.strip().splitlines()
            for line in lines[-20:]:
                print(f"    {line}")
        return None

    # Parse PERF_RESULT: line from stdout
    for line in result.stdout.splitlines():
        if line.startswith("PERF_RESULT:"):
            json_str = line[len("PERF_RESULT:"):]
            return json.loads(json_str)

    print("  WARNING: No PERF_RESULT found in test output")
    return None


def run_shadowcast_benchmark():
    """Run shadowcast benchmark and parse structured output."""
    print("  Running shadowcast benchmark (release mode)...")
    result = run_command([
        "cargo", "test", "--release", "--test", "shadowcast_bench",
        "--", "--ignored", "--nocapture", "benchmark_symmetric_vs_recursive",
    ])

    if result.returncode != 0:
        print(f"  FAILED: shadowcast benchmark exited {result.returncode}")
        return None

    configs = []
    current = {}
    for line in result.stdout.splitlines():
        line = line.strip()

        m = re.match(
            r"Map: (\d+)x(\d+), Density: (.+), Range: (\d+), Iterations: (\d+)",
            line,
        )
        if m:
            # New config block — flush previous if complete
            if current.get("map_size"):
                configs.append(current)
            current = {
                "map_size": int(m.group(1)),
                "density": m.group(3),
                "range": int(m.group(4)),
                "iterations": int(m.group(5)),
            }
            continue

        m = re.match(r"Symmetric:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line)
        if m:
            current["symmetric_total_ms"] = float(m.group(1))
            current["symmetric_per_call_us"] = float(m.group(2))
            continue

        m = re.match(r"Recursive:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line)
        if m:
            current["recursive_total_ms"] = float(m.group(1))
            current["recursive_per_call_us"] = float(m.group(2))
            continue

    # Flush last config
    if current.get("map_size"):
        configs.append(current)

    return {"configs": configs} if configs else None


def compare_baselines(old, new):
    """Compare two baselines and report regressions. Returns list of regression strings."""
    regressions = []
    improvements = []

    old_tick = old.get("tick_timing", {})
    new_tick = new.get("tick_timing", {})

    if old_tick and new_tick:
        # Mean tick time regression (>20% = warning)
        old_mean = old_tick.get("mean_us", 0)
        new_mean = new_tick.get("mean_us", 0)
        if old_mean > 0:
            change = (new_mean - old_mean) / old_mean * 100
            if change > 20:
                regressions.append(
                    f"mean tick time {old_mean}us -> {new_mean}us (+{change:.1f}%)"
                )
            elif change < -20:
                improvements.append(
                    f"mean tick time {old_mean}us -> {new_mean}us ({change:.1f}%)"
                )

        # p95 tick time regression
        old_p95 = old_tick.get("p95_us", 0)
        new_p95 = new_tick.get("p95_us", 0)
        if old_p95 > 0:
            change = (new_p95 - old_p95) / old_p95 * 100
            if change > 20:
                regressions.append(
                    f"p95 tick time {old_p95}us -> {new_p95}us (+{change:.1f}%)"
                )
            elif change < -20:
                improvements.append(
                    f"p95 tick time {old_p95}us -> {new_p95}us ({change:.1f}%)"
                )

    # Absolute budget check
    new_p95 = new.get("tick_timing", {}).get("p95_us", 0)
    if new_p95 > TICK_BUDGET_US:
        regressions.append(
            f"p95 {new_p95}us exceeds {TICK_BUDGET_US}us tick budget (D-026)"
        )

    return regressions, improvements


def main():
    compare_mode = "--compare" in sys.argv

    print("=== Performance Baseline ===\n")

    # Build in release mode first
    print("Building server (release)...")
    build = run_command(["cargo", "build", "--release"])
    if build.returncode != 0:
        print("BUILD FAILED")
        lines = build.stderr.strip().splitlines()
        for line in lines[-20:]:
            print(f"  {line}")
        return 1

    print("\nRunning benchmarks...\n")

    tick_results = run_tick_benchmark()
    shadowcast_results = run_shadowcast_benchmark()

    if not tick_results:
        print("\nFATAL: tick benchmark failed -- no baseline generated")
        return 1

    # Assemble baseline
    git_info = get_git_info()
    baseline = {
        "timestamp": datetime.now(timezone.utc).isoformat(),
        "git": git_info,
        "tick_timing": tick_results.get("tick_timing", {}),
        "entities": tick_results.get("entities", {}),
        "memory": tick_results.get("memory", {}),
    }
    if shadowcast_results:
        baseline["shadowcast"] = shadowcast_results

    # Report
    tt = baseline["tick_timing"]
    print(f"\n--- Results ---")
    print(f"Git: {git_info['commit']} ({git_info['branch']})")
    print(f"Tick timing ({tt.get('measured_ticks', '?')} ticks, "
          f"{tt.get('warmup_ticks', '?')} warmup):")
    print(f"  min:  {tt.get('min_us', '?')}us")
    print(f"  mean: {tt.get('mean_us', '?')}us")
    print(f"  p95:  {tt.get('p95_us', '?')}us")
    print(f"  max:  {tt.get('max_us', '?')}us")

    ent = baseline["entities"]
    print(f"Entities: avg {ent.get('avg_per_snapshot', '?')}, "
          f"max {ent.get('max_per_snapshot', '?')}")

    mem = baseline["memory"]
    rss = mem.get("rss_kb")
    if rss:
        print(f"Memory: {rss} KB RSS ({rss / 1024:.1f} MB)")

    if shadowcast_results:
        n = len(shadowcast_results.get("configs", []))
        print(f"Shadowcast: {n} configurations benchmarked")

    # Budget check
    p95 = tt.get("p95_us", 0)
    if p95 > TICK_BUDGET_US:
        print(f"\nBUDGET EXCEEDED: p95 {p95}us > {TICK_BUDGET_US}us (D-026)")
    else:
        budget_pct = p95 / TICK_BUDGET_US * 100 if TICK_BUDGET_US else 0
        print(f"\nBudget: {budget_pct:.1f}% of {TICK_BUDGET_US}us tick budget (D-026)")

    # Compare with previous baseline if it exists
    if BASELINE_FILE.exists():
        with open(BASELINE_FILE) as f:
            old_baseline = json.load(f)
        old_commit = old_baseline.get("git", {}).get("commit", "?")
        print(f"\n--- Comparison vs {old_commit} ---")
        regressions, improvements = compare_baselines(old_baseline, baseline)
        for r in regressions:
            print(f"  REGRESSION: {r}")
        for i in improvements:
            print(f"  IMPROVEMENT: {i}")
        if not regressions and not improvements:
            print("  No significant changes.")
        if compare_mode and regressions:
            print(f"\n{len(regressions)} regression(s) detected.")
            return 1
    elif compare_mode:
        print(f"\nERROR: --compare requires a saved baseline at {BASELINE_FILE.relative_to(ROOT)}")
        print("Run `make perf-baseline` first to create one.")
        return 1

    if compare_mode:
        return 0

    # Save baseline (strip per-tick array — too noisy for git diffs)
    PERF_DIR.mkdir(parents=True, exist_ok=True)
    committed = json.loads(json.dumps(baseline))
    committed["tick_timing"].pop("all_us", None)

    with open(BASELINE_FILE, "w") as f:
        json.dump(committed, f, indent=2)
        f.write("\n")

    print(f"\nBaseline written to {BASELINE_FILE.relative_to(ROOT)}")
    return 0


if __name__ == "__main__":
    sys.exit(main())