refactor(tooling): T-1286 — generate, pr and dev become reach domains
Twelve scripts retired, three domains registered. `reach` now covers nine. generate: `generate-brands` and `generate-corporations` were the second and third copies of the same 24-line build-if-missing-then-exec bash `tooling/atlas` carried, so they collapsed into `core.process.cargo_binary` rather than being ported. `import_economics` shelled out to the first of those, so it now calls that helper — `generated_brands.toml` comes back byte-identical, and the stamp registry swaps the retired wrapper for `core/process.py`. pr: `watchlist-diff` derives its watched set from `generator_sources.py` instead of restating it, so it cannot drift from the stamp check. dev: the environment scripts split decision from performing, per D-263's guarded-exec rule. `godot_plan()` and `worktree_plan()` decide what would happen; `install_godot()`, `install_rust()` and `setup_worktree()` do it. `tooling/test_environment.py` pins the version pin, both override precedences, the already-current skip, the platform refusal and both worktree refusals — none of them performed. `make setup` now installs reach first, since the targets that install rust and godot are reach verbs. Two live bugs found while porting: - The clerk read its decision index from `decisions/README.md`, a path that stopped existing when the DQR tree moved to `governance/`. Every clerk agent has been grepping blind; its prompt pointed at the same dead directory. - The conformance exec-check matched any `x.system()` regardless of receiver, so `platform.system()` read as `os.system()`. Narrowed and re-proved against a real mutant. `process.run` gains `input=`, `timeout=` and a `ProcessTimeout` subclass so a killed run stays distinguishable from a verdict. The pre-push hook no longer merges the clerk's stderr into its stdout — under streaming the last merged line is a JSONL event, which would read as an unrecognised verdict and block. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
Executable
+274
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Performance baseline — tick timing, memory, and shadowcast scaling.
|
||||
|
||||
Builds the server in release, runs the benchmark suite, and either writes
|
||||
tests/perf/baseline.json or compares against the committed one. The compare
|
||||
path exits non-zero on a >20% regression or a p95 over the D-026 tick budget.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from tooling.core import config, console, process
|
||||
from tooling.core.errors import ReachError
|
||||
|
||||
ROOT = config.repo_root()
|
||||
PERF_DIR = ROOT / "tests" / "perf"
|
||||
BASELINE_FILE = PERF_DIR / "baseline.json"
|
||||
|
||||
# Tick budget from D-026: 100ms per tick at 10 tps floor (D-031).
|
||||
# If tick rate changes, update this constant.
|
||||
TICK_BUDGET_US = 100_000 # 100ms
|
||||
|
||||
|
||||
def run_command(cmd, **kwargs):
|
||||
"""Run a command in the server directory and return the result."""
|
||||
return process.run(cmd, cwd=ROOT / "server", check=False, **kwargs)
|
||||
|
||||
|
||||
def get_git_info():
|
||||
"""Current commit and branch. check=False — a shallow or detached tree is
|
||||
not an error here, it just means the stamp carries less."""
|
||||
commit = process.run(
|
||||
["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, check=False
|
||||
).stdout.strip()
|
||||
branch = process.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=ROOT, check=False
|
||||
).stdout.strip()
|
||||
return {"commit": commit, "branch": branch}
|
||||
|
||||
|
||||
def run_tick_benchmark():
|
||||
"""Run perf_tick_timing test and parse PERF_RESULT JSON."""
|
||||
console.event("running tick timing benchmark (release mode)", phase="bench")
|
||||
result = run_command([
|
||||
"cargo", "test", "--release", "--test", "perf_bench",
|
||||
"--", "--ignored", "--nocapture", "perf_tick_timing",
|
||||
])
|
||||
|
||||
if result.returncode != 0:
|
||||
console.event(f"tick benchmark exited {result.returncode}", level="error")
|
||||
if result.stderr:
|
||||
# Last 20 lines of stderr are the diagnostic; the rest is build noise.
|
||||
for line in result.stderr.strip().splitlines()[-20:]:
|
||||
console.event(line, level="error")
|
||||
return None
|
||||
|
||||
# Parse PERF_RESULT: line from stdout
|
||||
for line in result.stdout.splitlines():
|
||||
if line.startswith("PERF_RESULT:"):
|
||||
json_str = line[len("PERF_RESULT:"):]
|
||||
return json.loads(json_str)
|
||||
|
||||
console.event("no PERF_RESULT found in test output", level="warn")
|
||||
return None
|
||||
|
||||
|
||||
def run_shadowcast_benchmark():
|
||||
"""Run shadowcast benchmark and parse structured output."""
|
||||
console.event("running shadowcast benchmark (release mode)", phase="bench")
|
||||
result = run_command([
|
||||
"cargo", "test", "--release", "--test", "shadowcast_bench",
|
||||
"--", "--ignored", "--nocapture", "benchmark_symmetric_vs_recursive",
|
||||
])
|
||||
|
||||
if result.returncode != 0:
|
||||
console.event(
|
||||
f"shadowcast benchmark exited {result.returncode}", level="error"
|
||||
)
|
||||
return None
|
||||
|
||||
configs = []
|
||||
current = {}
|
||||
for line in result.stdout.splitlines():
|
||||
line = line.strip()
|
||||
|
||||
m = re.match(
|
||||
r"Map: (\d+)x(\d+), Density: (.+), Range: (\d+), Iterations: (\d+)",
|
||||
line,
|
||||
)
|
||||
if m:
|
||||
# New config block — flush previous if complete
|
||||
if current.get("map_size"):
|
||||
configs.append(current)
|
||||
current = {
|
||||
"map_size": int(m.group(1)),
|
||||
"density": m.group(3),
|
||||
"range": int(m.group(4)),
|
||||
"iterations": int(m.group(5)),
|
||||
}
|
||||
continue
|
||||
|
||||
m = re.match(r"Symmetric:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line)
|
||||
if m:
|
||||
current["symmetric_total_ms"] = float(m.group(1))
|
||||
current["symmetric_per_call_us"] = float(m.group(2))
|
||||
continue
|
||||
|
||||
m = re.match(r"Recursive:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line)
|
||||
if m:
|
||||
current["recursive_total_ms"] = float(m.group(1))
|
||||
current["recursive_per_call_us"] = float(m.group(2))
|
||||
continue
|
||||
|
||||
# Flush last config
|
||||
if current.get("map_size"):
|
||||
configs.append(current)
|
||||
|
||||
return {"configs": configs} if configs else None
|
||||
|
||||
|
||||
def compare_baselines(old, new):
|
||||
"""Compare two baselines and report regressions. Returns list of regression strings."""
|
||||
regressions = []
|
||||
improvements = []
|
||||
|
||||
old_tick = old.get("tick_timing", {})
|
||||
new_tick = new.get("tick_timing", {})
|
||||
|
||||
if old_tick and new_tick:
|
||||
# Mean tick time regression (>20% = warning)
|
||||
old_mean = old_tick.get("mean_us", 0)
|
||||
new_mean = new_tick.get("mean_us", 0)
|
||||
if old_mean > 0:
|
||||
change = (new_mean - old_mean) / old_mean * 100
|
||||
if change > 20:
|
||||
regressions.append(
|
||||
f"mean tick time {old_mean}us -> {new_mean}us (+{change:.1f}%)"
|
||||
)
|
||||
elif change < -20:
|
||||
improvements.append(
|
||||
f"mean tick time {old_mean}us -> {new_mean}us ({change:.1f}%)"
|
||||
)
|
||||
|
||||
# p95 tick time regression
|
||||
old_p95 = old_tick.get("p95_us", 0)
|
||||
new_p95 = new_tick.get("p95_us", 0)
|
||||
if old_p95 > 0:
|
||||
change = (new_p95 - old_p95) / old_p95 * 100
|
||||
if change > 20:
|
||||
regressions.append(
|
||||
f"p95 tick time {old_p95}us -> {new_p95}us (+{change:.1f}%)"
|
||||
)
|
||||
elif change < -20:
|
||||
improvements.append(
|
||||
f"p95 tick time {old_p95}us -> {new_p95}us ({change:.1f}%)"
|
||||
)
|
||||
|
||||
# Absolute budget check
|
||||
new_p95 = new.get("tick_timing", {}).get("p95_us", 0)
|
||||
if new_p95 > TICK_BUDGET_US:
|
||||
regressions.append(
|
||||
f"p95 {new_p95}us exceeds {TICK_BUDGET_US}us tick budget (D-026)"
|
||||
)
|
||||
|
||||
return regressions, improvements
|
||||
|
||||
|
||||
def run(compare_mode: bool = False) -> None:
|
||||
"""Build release, run the benchmarks, then write or compare the baseline."""
|
||||
console.event("building server (release)", phase="build")
|
||||
build = run_command(["cargo", "build", "--release"])
|
||||
if build.returncode != 0:
|
||||
for line in build.stderr.strip().splitlines()[-20:]:
|
||||
console.event(line, level="error")
|
||||
raise ReachError(
|
||||
"the release build failed, so there is nothing to benchmark",
|
||||
fix="fix the build errors above, then re-run — a perf number from a "
|
||||
"stale binary is worse than no number",
|
||||
)
|
||||
|
||||
tick_results = run_tick_benchmark()
|
||||
shadowcast_results = run_shadowcast_benchmark()
|
||||
|
||||
if not tick_results:
|
||||
raise ReachError(
|
||||
"the tick benchmark produced no result — no baseline generated",
|
||||
fix="run `cargo test --release --test perf_bench -- --ignored "
|
||||
"--nocapture perf_tick_timing` in server/ to see why it failed",
|
||||
)
|
||||
|
||||
# Assemble baseline
|
||||
git_info = get_git_info()
|
||||
baseline = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"git": git_info,
|
||||
"tick_timing": tick_results.get("tick_timing", {}),
|
||||
"entities": tick_results.get("entities", {}),
|
||||
"memory": tick_results.get("memory", {}),
|
||||
}
|
||||
if shadowcast_results:
|
||||
baseline["shadowcast"] = shadowcast_results
|
||||
|
||||
# Report
|
||||
tt = baseline["tick_timing"]
|
||||
console.out("--- Results ---")
|
||||
console.out(f"Git: {git_info['commit']} ({git_info['branch']})")
|
||||
console.out(f"Tick timing ({tt.get('measured_ticks', '?')} ticks, "
|
||||
f"{tt.get('warmup_ticks', '?')} warmup):")
|
||||
console.out(f" min: {tt.get('min_us', '?')}us")
|
||||
console.out(f" mean: {tt.get('mean_us', '?')}us")
|
||||
console.out(f" p95: {tt.get('p95_us', '?')}us")
|
||||
console.out(f" max: {tt.get('max_us', '?')}us")
|
||||
|
||||
ent = baseline["entities"]
|
||||
console.out(f"Entities: avg {ent.get('avg_per_snapshot', '?')}, "
|
||||
f"max {ent.get('max_per_snapshot', '?')}")
|
||||
|
||||
rss = baseline["memory"].get("rss_kb")
|
||||
if rss:
|
||||
console.out(f"Memory: {rss} KB RSS ({rss / 1024:.1f} MB)")
|
||||
|
||||
if shadowcast_results:
|
||||
n = len(shadowcast_results.get("configs", []))
|
||||
console.out(f"Shadowcast: {n} configurations benchmarked")
|
||||
|
||||
# Budget check
|
||||
p95 = tt.get("p95_us", 0)
|
||||
if p95 > TICK_BUDGET_US:
|
||||
console.out(f"BUDGET EXCEEDED: p95 {p95}us > {TICK_BUDGET_US}us (D-026)")
|
||||
else:
|
||||
budget_pct = p95 / TICK_BUDGET_US * 100 if TICK_BUDGET_US else 0
|
||||
console.out(
|
||||
f"Budget: {budget_pct:.1f}% of {TICK_BUDGET_US}us tick budget (D-026)"
|
||||
)
|
||||
|
||||
# Compare with previous baseline if it exists
|
||||
regressions: list[str] = []
|
||||
if BASELINE_FILE.exists():
|
||||
old_baseline = json.loads(BASELINE_FILE.read_text())
|
||||
old_commit = old_baseline.get("git", {}).get("commit", "?")
|
||||
console.out(f"--- Comparison vs {old_commit} ---")
|
||||
regressions, improvements = compare_baselines(old_baseline, baseline)
|
||||
for r in regressions:
|
||||
console.out(f" REGRESSION: {r}")
|
||||
for i in improvements:
|
||||
console.out(f" IMPROVEMENT: {i}")
|
||||
if not regressions and not improvements:
|
||||
console.out(" No significant changes.")
|
||||
elif compare_mode:
|
||||
raise ReachError(
|
||||
f"--compare needs a saved baseline at {BASELINE_FILE.relative_to(ROOT)}",
|
||||
fix="run `reach dev perf` once without --compare to create one",
|
||||
)
|
||||
|
||||
if compare_mode:
|
||||
if regressions:
|
||||
raise ReachError(
|
||||
f"{len(regressions)} performance regression(s) detected",
|
||||
fix="see the REGRESSION lines above; if the change is intended, "
|
||||
"re-run without --compare to accept it as the new baseline",
|
||||
)
|
||||
console.verdict("perf: no regressions against the saved baseline")
|
||||
return
|
||||
|
||||
# Save baseline (strip per-tick array — too noisy for git diffs)
|
||||
PERF_DIR.mkdir(parents=True, exist_ok=True)
|
||||
committed = json.loads(json.dumps(baseline))
|
||||
committed["tick_timing"].pop("all_us", None)
|
||||
BASELINE_FILE.write_text(json.dumps(committed, indent=2) + "\n")
|
||||
|
||||
console.verdict(f"baseline written to {BASELINE_FILE.relative_to(ROOT)}")
|
||||
Reference in New Issue
Block a user