#!/usr/bin/env python3 """Performance baseline — tick timing, memory, and shadowcast scaling. Builds the server in release, runs the benchmark suite, and either writes tests/perf/baseline.json or compares against the committed one. The compare path exits non-zero on a >20% regression or a p95 over the D-026 tick budget. """ from __future__ import annotations import json import re from datetime import datetime, timezone from tooling.core import config, console, process from tooling.core.errors import ReachError ROOT = config.repo_root() PERF_DIR = ROOT / "tests" / "perf" BASELINE_FILE = PERF_DIR / "baseline.json" # Tick budget from D-026: 100ms per tick at 10 tps floor (D-031). # If tick rate changes, update this constant. TICK_BUDGET_US = 100_000 # 100ms def run_command(cmd, **kwargs): """Run a command in the server directory and return the result.""" return process.run(cmd, cwd=ROOT / "server", check=False, **kwargs) def get_git_info(): """Current commit and branch. check=False — a shallow or detached tree is not an error here, it just means the stamp carries less.""" commit = process.run( ["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, check=False ).stdout.strip() branch = process.run( ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=ROOT, check=False ).stdout.strip() return {"commit": commit, "branch": branch} def run_tick_benchmark(): """Run perf_tick_timing test and parse PERF_RESULT JSON.""" console.event("running tick timing benchmark (release mode)", phase="bench") result = run_command([ "cargo", "test", "--release", "--test", "perf_bench", "--", "--ignored", "--nocapture", "perf_tick_timing", ]) if result.returncode != 0: console.event(f"tick benchmark exited {result.returncode}", level="error") if result.stderr: # Last 20 lines of stderr are the diagnostic; the rest is build noise. for line in result.stderr.strip().splitlines()[-20:]: console.event(line, level="error") return None # Parse PERF_RESULT: line from stdout for line in result.stdout.splitlines(): if line.startswith("PERF_RESULT:"): json_str = line[len("PERF_RESULT:"):] return json.loads(json_str) console.event("no PERF_RESULT found in test output", level="warn") return None def run_shadowcast_benchmark(): """Run shadowcast benchmark and parse structured output.""" console.event("running shadowcast benchmark (release mode)", phase="bench") result = run_command([ "cargo", "test", "--release", "--test", "shadowcast_bench", "--", "--ignored", "--nocapture", "benchmark_symmetric_vs_recursive", ]) if result.returncode != 0: console.event( f"shadowcast benchmark exited {result.returncode}", level="error" ) return None configs = [] current = {} for line in result.stdout.splitlines(): line = line.strip() m = re.match( r"Map: (\d+)x(\d+), Density: (.+), Range: (\d+), Iterations: (\d+)", line, ) if m: # New config block — flush previous if complete if current.get("map_size"): configs.append(current) current = { "map_size": int(m.group(1)), "density": m.group(3), "range": int(m.group(4)), "iterations": int(m.group(5)), } continue m = re.match(r"Symmetric:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line) if m: current["symmetric_total_ms"] = float(m.group(1)) current["symmetric_per_call_us"] = float(m.group(2)) continue m = re.match(r"Recursive:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line) if m: current["recursive_total_ms"] = float(m.group(1)) current["recursive_per_call_us"] = float(m.group(2)) continue # Flush last config if current.get("map_size"): configs.append(current) return {"configs": configs} if configs else None def compare_baselines(old, new): """Compare two baselines and report regressions. Returns list of regression strings.""" regressions = [] improvements = [] old_tick = old.get("tick_timing", {}) new_tick = new.get("tick_timing", {}) if old_tick and new_tick: # Mean tick time regression (>20% = warning) old_mean = old_tick.get("mean_us", 0) new_mean = new_tick.get("mean_us", 0) if old_mean > 0: change = (new_mean - old_mean) / old_mean * 100 if change > 20: regressions.append( f"mean tick time {old_mean}us -> {new_mean}us (+{change:.1f}%)" ) elif change < -20: improvements.append( f"mean tick time {old_mean}us -> {new_mean}us ({change:.1f}%)" ) # p95 tick time regression old_p95 = old_tick.get("p95_us", 0) new_p95 = new_tick.get("p95_us", 0) if old_p95 > 0: change = (new_p95 - old_p95) / old_p95 * 100 if change > 20: regressions.append( f"p95 tick time {old_p95}us -> {new_p95}us (+{change:.1f}%)" ) elif change < -20: improvements.append( f"p95 tick time {old_p95}us -> {new_p95}us ({change:.1f}%)" ) # Absolute budget check new_p95 = new.get("tick_timing", {}).get("p95_us", 0) if new_p95 > TICK_BUDGET_US: regressions.append( f"p95 {new_p95}us exceeds {TICK_BUDGET_US}us tick budget (D-026)" ) return regressions, improvements def run(compare_mode: bool = False) -> None: """Build release, run the benchmarks, then write or compare the baseline.""" console.event("building server (release)", phase="build") build = run_command(["cargo", "build", "--release"]) if build.returncode != 0: for line in build.stderr.strip().splitlines()[-20:]: console.event(line, level="error") raise ReachError( "the release build failed, so there is nothing to benchmark", fix="fix the build errors above, then re-run — a perf number from a " "stale binary is worse than no number", ) tick_results = run_tick_benchmark() shadowcast_results = run_shadowcast_benchmark() if not tick_results: raise ReachError( "the tick benchmark produced no result — no baseline generated", fix="run `cargo test --release --test perf_bench -- --ignored " "--nocapture perf_tick_timing` in server/ to see why it failed", ) # Assemble baseline git_info = get_git_info() baseline = { "timestamp": datetime.now(timezone.utc).isoformat(), "git": git_info, "tick_timing": tick_results.get("tick_timing", {}), "entities": tick_results.get("entities", {}), "memory": tick_results.get("memory", {}), } if shadowcast_results: baseline["shadowcast"] = shadowcast_results # Report tt = baseline["tick_timing"] console.out("--- Results ---") console.out(f"Git: {git_info['commit']} ({git_info['branch']})") console.out(f"Tick timing ({tt.get('measured_ticks', '?')} ticks, " f"{tt.get('warmup_ticks', '?')} warmup):") console.out(f" min: {tt.get('min_us', '?')}us") console.out(f" mean: {tt.get('mean_us', '?')}us") console.out(f" p95: {tt.get('p95_us', '?')}us") console.out(f" max: {tt.get('max_us', '?')}us") ent = baseline["entities"] console.out(f"Entities: avg {ent.get('avg_per_snapshot', '?')}, " f"max {ent.get('max_per_snapshot', '?')}") rss = baseline["memory"].get("rss_kb") if rss: console.out(f"Memory: {rss} KB RSS ({rss / 1024:.1f} MB)") if shadowcast_results: n = len(shadowcast_results.get("configs", [])) console.out(f"Shadowcast: {n} configurations benchmarked") # Budget check p95 = tt.get("p95_us", 0) if p95 > TICK_BUDGET_US: console.out(f"BUDGET EXCEEDED: p95 {p95}us > {TICK_BUDGET_US}us (D-026)") else: budget_pct = p95 / TICK_BUDGET_US * 100 if TICK_BUDGET_US else 0 console.out( f"Budget: {budget_pct:.1f}% of {TICK_BUDGET_US}us tick budget (D-026)" ) # Compare with previous baseline if it exists regressions: list[str] = [] if BASELINE_FILE.exists(): old_baseline = json.loads(BASELINE_FILE.read_text()) old_commit = old_baseline.get("git", {}).get("commit", "?") console.out(f"--- Comparison vs {old_commit} ---") regressions, improvements = compare_baselines(old_baseline, baseline) for r in regressions: console.out(f" REGRESSION: {r}") for i in improvements: console.out(f" IMPROVEMENT: {i}") if not regressions and not improvements: console.out(" No significant changes.") elif compare_mode: raise ReachError( f"--compare needs a saved baseline at {BASELINE_FILE.relative_to(ROOT)}", fix="run `reach dev perf` once without --compare to create one", ) if compare_mode: if regressions: raise ReachError( f"{len(regressions)} performance regression(s) detected", fix="see the REGRESSION lines above; if the change is intended, " "re-run without --compare to accept it as the new baseline", ) console.verdict("perf: no regressions against the saved baseline") return # Save baseline (strip per-tick array — too noisy for git diffs) PERF_DIR.mkdir(parents=True, exist_ok=True) committed = json.loads(json.dumps(baseline)) committed["tick_timing"].pop("all_us", None) BASELINE_FILE.write_text(json.dumps(committed, indent=2) + "\n") console.verdict(f"baseline written to {BASELINE_FILE.relative_to(ROOT)}")