feat(ci): add performance baseline tooling (#499)
Adds `make perf-baseline` — boots the full server plugin stack with real content, measures 50 ticks (5 warmup), captures per-tick timing, entity counts, and process RSS. Includes shadowcast benchmarks. Saves structured JSON to tests/perf/baseline.json for regression detection. Supports --compare mode (>20% threshold). First baseline: mean 366µs, p95 526µs (0.5% of D-026 100ms budget). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
Executable
+273
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Performance baseline tooling.
|
||||
|
||||
Runs the server benchmark suite, captures tick timing, memory usage, and entity
|
||||
count scaling metrics, outputs results to tests/perf/.
|
||||
|
||||
Usage:
|
||||
tooling/perf-baseline Run benchmarks and save baseline
|
||||
tooling/perf-baseline --compare Compare current run against saved baseline (no save)
|
||||
|
||||
Exit code 0 = success, 1 = failure or regression detected (--compare mode).
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
PERF_DIR = ROOT / "tests" / "perf"
|
||||
BASELINE_FILE = PERF_DIR / "baseline.json"
|
||||
|
||||
# Tick budget from D-026
|
||||
TICK_BUDGET_US = 100_000 # 100ms
|
||||
|
||||
|
||||
def run_command(cmd, **kwargs):
|
||||
"""Run a command in the server directory and return the result."""
|
||||
return subprocess.run(
|
||||
cmd, capture_output=True, text=True, cwd=ROOT / "server", **kwargs
|
||||
)
|
||||
|
||||
|
||||
def get_git_info():
|
||||
"""Get current git commit and branch."""
|
||||
commit = subprocess.run(
|
||||
["git", "rev-parse", "--short", "HEAD"],
|
||||
capture_output=True, text=True, cwd=ROOT,
|
||||
).stdout.strip()
|
||||
branch = subprocess.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
capture_output=True, text=True, cwd=ROOT,
|
||||
).stdout.strip()
|
||||
return {"commit": commit, "branch": branch}
|
||||
|
||||
|
||||
def run_tick_benchmark():
|
||||
"""Run perf_tick_timing test and parse PERF_RESULT JSON."""
|
||||
print(" Running tick timing benchmark (release mode)...")
|
||||
result = run_command([
|
||||
"cargo", "test", "--release", "--test", "perf_bench",
|
||||
"--", "--ignored", "--nocapture", "perf_tick_timing",
|
||||
])
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f" FAILED: tick benchmark exited {result.returncode}")
|
||||
if result.stderr:
|
||||
# Print last 20 lines of stderr for diagnostics
|
||||
lines = result.stderr.strip().splitlines()
|
||||
for line in lines[-20:]:
|
||||
print(f" {line}")
|
||||
return None
|
||||
|
||||
# Parse PERF_RESULT: line from stdout
|
||||
for line in result.stdout.splitlines():
|
||||
if line.startswith("PERF_RESULT:"):
|
||||
json_str = line[len("PERF_RESULT:"):]
|
||||
return json.loads(json_str)
|
||||
|
||||
print(" WARNING: No PERF_RESULT found in test output")
|
||||
return None
|
||||
|
||||
|
||||
def run_shadowcast_benchmark():
|
||||
"""Run shadowcast benchmark and parse structured output."""
|
||||
print(" Running shadowcast benchmark (release mode)...")
|
||||
result = run_command([
|
||||
"cargo", "test", "--release", "--test", "shadowcast_bench",
|
||||
"--", "--ignored", "--nocapture", "benchmark_symmetric_vs_recursive",
|
||||
])
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f" FAILED: shadowcast benchmark exited {result.returncode}")
|
||||
return None
|
||||
|
||||
configs = []
|
||||
current = {}
|
||||
for line in result.stdout.splitlines():
|
||||
line = line.strip()
|
||||
|
||||
m = re.match(
|
||||
r"Map: (\d+)x(\d+), Density: (.+), Range: (\d+), Iterations: (\d+)",
|
||||
line,
|
||||
)
|
||||
if m:
|
||||
current = {
|
||||
"map_size": int(m.group(1)),
|
||||
"density": m.group(3),
|
||||
"range": int(m.group(4)),
|
||||
"iterations": int(m.group(5)),
|
||||
}
|
||||
continue
|
||||
|
||||
m = re.match(r"Symmetric:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line)
|
||||
if m:
|
||||
current["symmetric_total_ms"] = float(m.group(1))
|
||||
current["symmetric_per_call_us"] = float(m.group(2))
|
||||
continue
|
||||
|
||||
m = re.match(r"Recursive:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line)
|
||||
if m:
|
||||
current["recursive_total_ms"] = float(m.group(1))
|
||||
current["recursive_per_call_us"] = float(m.group(2))
|
||||
if current.get("map_size"):
|
||||
configs.append(current)
|
||||
current = {}
|
||||
continue
|
||||
|
||||
return {"configs": configs} if configs else None
|
||||
|
||||
|
||||
def compare_baselines(old, new):
|
||||
"""Compare two baselines and report regressions. Returns list of regression strings."""
|
||||
regressions = []
|
||||
improvements = []
|
||||
|
||||
old_tick = old.get("tick_timing", {})
|
||||
new_tick = new.get("tick_timing", {})
|
||||
|
||||
if old_tick and new_tick:
|
||||
# Mean tick time regression (>20% = warning)
|
||||
old_mean = old_tick.get("mean_us", 0)
|
||||
new_mean = new_tick.get("mean_us", 0)
|
||||
if old_mean > 0:
|
||||
change = (new_mean - old_mean) / old_mean * 100
|
||||
if change > 20:
|
||||
regressions.append(
|
||||
f"mean tick time {old_mean}us -> {new_mean}us (+{change:.1f}%)"
|
||||
)
|
||||
elif change < -20:
|
||||
improvements.append(
|
||||
f"mean tick time {old_mean}us -> {new_mean}us ({change:.1f}%)"
|
||||
)
|
||||
|
||||
# p95 tick time regression
|
||||
old_p95 = old_tick.get("p95_us", 0)
|
||||
new_p95 = new_tick.get("p95_us", 0)
|
||||
if old_p95 > 0:
|
||||
change = (new_p95 - old_p95) / old_p95 * 100
|
||||
if change > 20:
|
||||
regressions.append(
|
||||
f"p95 tick time {old_p95}us -> {new_p95}us (+{change:.1f}%)"
|
||||
)
|
||||
elif change < -20:
|
||||
improvements.append(
|
||||
f"p95 tick time {old_p95}us -> {new_p95}us ({change:.1f}%)"
|
||||
)
|
||||
|
||||
# Absolute budget check
|
||||
new_p95 = new.get("tick_timing", {}).get("p95_us", 0)
|
||||
if new_p95 > TICK_BUDGET_US:
|
||||
regressions.append(
|
||||
f"p95 {new_p95}us exceeds {TICK_BUDGET_US}us tick budget (D-026)"
|
||||
)
|
||||
|
||||
return regressions, improvements
|
||||
|
||||
|
||||
def main():
|
||||
compare_mode = "--compare" in sys.argv
|
||||
|
||||
print("=== Performance Baseline ===\n")
|
||||
|
||||
# Build in release mode first
|
||||
print("Building server (release)...")
|
||||
build = run_command(["cargo", "build", "--release"])
|
||||
if build.returncode != 0:
|
||||
print("BUILD FAILED")
|
||||
lines = build.stderr.strip().splitlines()
|
||||
for line in lines[-20:]:
|
||||
print(f" {line}")
|
||||
return 1
|
||||
|
||||
print("\nRunning benchmarks...\n")
|
||||
|
||||
tick_results = run_tick_benchmark()
|
||||
shadowcast_results = run_shadowcast_benchmark()
|
||||
|
||||
if not tick_results:
|
||||
print("\nFATAL: tick benchmark failed -- no baseline generated")
|
||||
return 1
|
||||
|
||||
# Assemble baseline
|
||||
git_info = get_git_info()
|
||||
baseline = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"git": git_info,
|
||||
"tick_timing": tick_results.get("tick_timing", {}),
|
||||
"entities": tick_results.get("entities", {}),
|
||||
"memory": tick_results.get("memory", {}),
|
||||
}
|
||||
if shadowcast_results:
|
||||
baseline["shadowcast"] = shadowcast_results
|
||||
|
||||
# Report
|
||||
tt = baseline["tick_timing"]
|
||||
print(f"\n--- Results ---")
|
||||
print(f"Git: {git_info['commit']} ({git_info['branch']})")
|
||||
print(f"Tick timing ({tt.get('measured_ticks', '?')} ticks, "
|
||||
f"{tt.get('warmup_ticks', '?')} warmup):")
|
||||
print(f" min: {tt.get('min_us', '?')}us")
|
||||
print(f" mean: {tt.get('mean_us', '?')}us")
|
||||
print(f" p95: {tt.get('p95_us', '?')}us")
|
||||
print(f" max: {tt.get('max_us', '?')}us")
|
||||
|
||||
ent = baseline["entities"]
|
||||
print(f"Entities: avg {ent.get('avg_per_snapshot', '?')}, "
|
||||
f"max {ent.get('max_per_snapshot', '?')}")
|
||||
|
||||
mem = baseline["memory"]
|
||||
rss = mem.get("rss_kb")
|
||||
if rss:
|
||||
print(f"Memory: {rss} KB RSS ({rss / 1024:.1f} MB)")
|
||||
|
||||
if shadowcast_results:
|
||||
n = len(shadowcast_results.get("configs", []))
|
||||
print(f"Shadowcast: {n} configurations benchmarked")
|
||||
|
||||
# Budget check
|
||||
p95 = tt.get("p95_us", 0)
|
||||
if p95 > TICK_BUDGET_US:
|
||||
print(f"\nBUDGET EXCEEDED: p95 {p95}us > {TICK_BUDGET_US}us (D-026)")
|
||||
else:
|
||||
budget_pct = p95 / TICK_BUDGET_US * 100 if TICK_BUDGET_US else 0
|
||||
print(f"\nBudget: {budget_pct:.1f}% of {TICK_BUDGET_US}us tick budget (D-026)")
|
||||
|
||||
# Compare with previous baseline if it exists
|
||||
if BASELINE_FILE.exists():
|
||||
with open(BASELINE_FILE) as f:
|
||||
old_baseline = json.load(f)
|
||||
old_commit = old_baseline.get("git", {}).get("commit", "?")
|
||||
print(f"\n--- Comparison vs {old_commit} ---")
|
||||
regressions, improvements = compare_baselines(old_baseline, baseline)
|
||||
for r in regressions:
|
||||
print(f" REGRESSION: {r}")
|
||||
for i in improvements:
|
||||
print(f" IMPROVEMENT: {i}")
|
||||
if not regressions and not improvements:
|
||||
print(" No significant changes.")
|
||||
if compare_mode and regressions:
|
||||
print(f"\n{len(regressions)} regression(s) detected.")
|
||||
return 1
|
||||
|
||||
if compare_mode:
|
||||
return 0
|
||||
|
||||
# Save baseline (strip per-tick array — too noisy for git diffs)
|
||||
PERF_DIR.mkdir(parents=True, exist_ok=True)
|
||||
committed = json.loads(json.dumps(baseline))
|
||||
committed["tick_timing"].pop("all_us", None)
|
||||
|
||||
with open(BASELINE_FILE, "w") as f:
|
||||
json.dump(committed, f, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
print(f"\nBaseline written to {BASELINE_FILE.relative_to(ROOT)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user