Golden targets: make golden-diff shows color diff if simulation output changed, make golden-update regenerates and stages for review. Checklist schema: JSON Schema for 7 condition types evaluable from ObserverSnapshot. Per-room YAML checklists for 3 Gauntlet rooms plus cross-room checks. Validation script + make checklist-validate/generate. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
Executable
+190
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate Gauntlet checklist YAML files against the checklist JSON schema.
|
||||
|
||||
Usage:
|
||||
validate-checklist Validate all checklists + print summary
|
||||
validate-checklist --check Schema validation only (for pre-PR chain)
|
||||
|
||||
Exit code 0 = all valid, 1 = validation errors found.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import jsonschema
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCHEMA_PATH = ROOT / "content" / "_schema" / "checklist.schema.json"
|
||||
GAUNTLET_DIR = ROOT / "content" / "gauntlet"
|
||||
|
||||
|
||||
def load_schema():
|
||||
with open(SCHEMA_PATH) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def find_checklists():
|
||||
"""Find all checklist YAML files under content/gauntlet/."""
|
||||
files = []
|
||||
if not GAUNTLET_DIR.exists():
|
||||
return files
|
||||
for path in sorted(GAUNTLET_DIR.rglob("checklist.yaml")):
|
||||
files.append(path)
|
||||
cross = GAUNTLET_DIR / "cross_room_checks.yaml"
|
||||
if cross.exists():
|
||||
files.append(cross)
|
||||
return files
|
||||
|
||||
|
||||
def _rel(path: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(ROOT))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def validate_schema(files, schema):
|
||||
"""Pass 1: JSON Schema validation. Returns (validated, errors)."""
|
||||
errors = 0
|
||||
validated = 0
|
||||
for path in files:
|
||||
rel = _rel(path)
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = yaml.safe_load(f)
|
||||
except yaml.YAMLError as e:
|
||||
print(f"YAML ERROR: {rel}: {e}")
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
if data is None:
|
||||
print(f"EMPTY: {rel}")
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
jsonschema.validate(instance=data, schema=schema)
|
||||
validated += 1
|
||||
except jsonschema.ValidationError as e:
|
||||
print(f"INVALID: {rel}")
|
||||
print(f" Error: {e.message}")
|
||||
if e.absolute_path:
|
||||
print(f" Path: {'.'.join(str(p) for p in e.absolute_path)}")
|
||||
errors += 1
|
||||
|
||||
return validated, errors
|
||||
|
||||
|
||||
def check_id_uniqueness(files):
|
||||
"""Pass 2: Condition ID uniqueness within and across files."""
|
||||
errors = 0
|
||||
global_ids: dict[str, Path] = {}
|
||||
|
||||
for path in files:
|
||||
rel = _rel(path)
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = yaml.safe_load(f)
|
||||
except (yaml.YAMLError, OSError):
|
||||
continue
|
||||
if not data or not isinstance(data, dict):
|
||||
continue
|
||||
|
||||
local_seen: set[str] = set()
|
||||
for cond in data.get("conditions", []):
|
||||
cid = cond.get("id")
|
||||
if not cid:
|
||||
continue
|
||||
if cid in local_seen:
|
||||
print(f'ID ERROR: duplicate condition id "{cid}" in {rel}')
|
||||
errors += 1
|
||||
local_seen.add(cid)
|
||||
|
||||
if cid in global_ids and global_ids[cid] != path:
|
||||
print(f'ID ERROR: condition id "{cid}" used in multiple files')
|
||||
print(f" First: {_rel(global_ids[cid])}")
|
||||
print(f" Also: {rel}")
|
||||
errors += 1
|
||||
elif cid not in global_ids:
|
||||
global_ids[cid] = path
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def summarize(files):
|
||||
"""Print per-room condition counts and type breakdown."""
|
||||
total = 0
|
||||
type_counts: dict[str, int] = {}
|
||||
|
||||
for path in files:
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = yaml.safe_load(f)
|
||||
except (yaml.YAMLError, OSError):
|
||||
continue
|
||||
if not data or not isinstance(data, dict):
|
||||
continue
|
||||
|
||||
conditions = data.get("conditions", [])
|
||||
count = len(conditions)
|
||||
total += count
|
||||
room = data.get("room_id", "cross_room")
|
||||
print(f" {room}: {count} conditions")
|
||||
|
||||
for cond in conditions:
|
||||
ct = cond.get("condition_type", "unknown")
|
||||
type_counts[ct] = type_counts.get(ct, 0) + 1
|
||||
|
||||
print(f"\n Total: {total} conditions across {len(files)} files")
|
||||
if type_counts:
|
||||
print(" By type:")
|
||||
for ct in sorted(type_counts):
|
||||
print(f" {ct}: {type_counts[ct]}")
|
||||
|
||||
|
||||
def main():
|
||||
check_only = "--check" in sys.argv
|
||||
|
||||
if not GAUNTLET_DIR.exists():
|
||||
print(f"No gauntlet directory at {_rel(GAUNTLET_DIR)}")
|
||||
print("Checklist validation skipped (no content yet).")
|
||||
return 0
|
||||
|
||||
files = find_checklists()
|
||||
if not files:
|
||||
print("No checklist files found under content/gauntlet/.")
|
||||
print("Checklist validation skipped.")
|
||||
return 0
|
||||
|
||||
schema = load_schema()
|
||||
|
||||
# Pass 1: Schema validation
|
||||
validated, schema_errors = validate_schema(files, schema)
|
||||
print(f"Pass 1 (schema): {validated} valid, {schema_errors} errors")
|
||||
|
||||
if schema_errors > 0:
|
||||
print(f"\nSchema validation failed ({schema_errors} errors) — skipping ID checks")
|
||||
return 1
|
||||
|
||||
# Pass 2: Condition ID uniqueness
|
||||
id_errors = check_id_uniqueness(files)
|
||||
if id_errors > 0:
|
||||
print(f"Pass 2 (IDs): {id_errors} errors")
|
||||
|
||||
total_errors = schema_errors + id_errors
|
||||
print(f"\nChecklist validation: {validated} valid, {total_errors} errors")
|
||||
|
||||
if total_errors > 0:
|
||||
return 1
|
||||
|
||||
if not check_only:
|
||||
print("\nChecklist summary:")
|
||||
summarize(files)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user