From d8d005169a02338d420a7c64d46e76ff679c8469 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Wed, 18 Feb 2026 10:44:33 +0100 Subject: [PATCH] feat(ci): add golden file targets and checklist schema (#486, #497) Golden targets: make golden-diff shows color diff if simulation output changed, make golden-update regenerates and stages for review. Checklist schema: JSON Schema for 7 condition types evaluable from ObserverSnapshot. Per-room YAML checklists for 3 Gauntlet rooms plus cross-room checks. Validation script + make checklist-validate/generate. Co-Authored-By: Claude Opus 4.6 --- Makefile | 38 +++- content/_schema/checklist.schema.json | 130 ++++++++++++ content/gauntlet/cross_room_checks.yaml | 6 + .../rooms/inventory_warehouse/checklist.yaml | 42 ++++ .../rooms/occlusion_corridor/checklist.yaml | 41 ++++ .../rooms/pause_chamber/checklist.yaml | 26 +++ docs/DEVOPS.md | 31 +++ tooling/validate-checklist | 190 ++++++++++++++++++ 8 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 content/_schema/checklist.schema.json create mode 100644 content/gauntlet/cross_room_checks.yaml create mode 100644 content/gauntlet/rooms/inventory_warehouse/checklist.yaml create mode 100644 content/gauntlet/rooms/occlusion_corridor/checklist.yaml create mode 100644 content/gauntlet/rooms/pause_chamber/checklist.yaml create mode 100755 tooling/validate-checklist diff --git a/Makefile b/Makefile index b2ed08a7d..04ca7bc27 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,8 @@ GODOT := $(shell command -v godot4 2>/dev/null || command -v godot 2>/dev/null) db-backup db-install validate-content content-ron check-fact-ids setup-hooks \ pre-pr pre-pr-lint pre-pr-build pre-pr-test pre-pr-validate pre-pr-fixtures \ pre-pr-server pre-pr-client pre-pr-content \ - fixtures-client + fixtures-client golden-diff golden-update \ + checklist-validate checklist-generate # --- Configuration --- @@ -39,6 +40,10 @@ help: @echo " make check-fact-ids Check fact_id references against knowledge catalogs" @echo " make content-ron Convert content YAML to RON (build-time)" @echo " make fixtures-client Generate GDScript->Rust cross-encoder fixtures (#475)" + @echo " make golden-diff Show diff if golden file output has changed" + @echo " make golden-update Regenerate golden file and stage for commit" + @echo " make checklist-validate Validate checklist YAML against schema" + @echo " make checklist-generate Validate checklists + print condition summary" @echo "" @echo " make pre-pr Run all pre-PR checks (lint, build, test, validate, fixtures)" @echo " make pre-pr-server Server-scoped pre-PR (lint, build, test, fixtures)" @@ -121,6 +126,31 @@ fixtures-client: cd server && cargo test --test serialization gdscript_generated_fixtures @echo "--- Fixtures-client: PASS ---" +golden-diff: + @echo "Regenerating golden output for comparison..." + @mkdir -p .cache/golden + @cp server/tests/golden/proof_room_tick_10.json .cache/golden/before.json + @cd server && UPDATE_GOLDEN=1 cargo test --test golden_suite -- proof_room_tick_10_matches_golden + @cp server/tests/golden/proof_room_tick_10.json .cache/golden/after.json + @cp .cache/golden/before.json server/tests/golden/proof_room_tick_10.json + @if diff -q .cache/golden/before.json .cache/golden/after.json >/dev/null 2>&1; then \ + echo "--- Golden file: UP TO DATE ---"; \ + else \ + echo "--- Golden file has CHANGED ---"; \ + echo ""; \ + diff --color -u .cache/golden/before.json .cache/golden/after.json || true; \ + echo ""; \ + echo "Run 'make golden-update' to accept changes."; \ + exit 1; \ + fi + +golden-update: + @echo "Regenerating golden file..." + @cd server && UPDATE_GOLDEN=1 cargo test --test golden_suite -- proof_room_tick_10_matches_golden + @git add server/tests/golden/ + @echo "--- Golden file updated and staged ---" + @echo "Review with: git diff --cached -- server/tests/golden/" + test-client: @test -n "$(GODOT)" || { echo "Godot not found. Run 'make setup' first."; exit 1; } $(GODOT) --headless --path client -s res://addons/gdUnit4/bin/GdUnitCmdTool.gd --ignoreHeadlessMode -a res://tests/ @@ -240,6 +270,12 @@ validate-content: check-fact-ids: @tooling/check-fact-ids +checklist-validate: + @tooling/validate-checklist --check + +checklist-generate: + @tooling/validate-checklist + content-ron: cd tooling/content-converter && cargo build --release tooling/content-converter/target/release/content-converter --input content --output content-ron --verbose diff --git a/content/_schema/checklist.schema.json b/content/_schema/checklist.schema.json new file mode 100644 index 000000000..4b05f272d --- /dev/null +++ b/content/_schema/checklist.schema.json @@ -0,0 +1,130 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "checklist.schema.json", + "title": "Gauntlet QA Checklist", + "description": "Per-room QA checklist with conditions evaluable from ObserverSnapshot (D-030). Feeds into #503 auto-checklist progress tracking.", + "type": "object", + "required": ["conditions"], + "additionalProperties": false, + "properties": { + "room_id": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*$", + "description": "Room identifier matching constants.rs room name" + }, + "room_name": { + "type": "string", + "minLength": 1, + "description": "Human-readable room name" + }, + "description": { + "type": "string" + }, + "conditions": { + "type": "array", + "items": { + "oneOf": [ + { "$ref": "#/$defs/player_near" }, + { "$ref": "#/$defs/player_facing" }, + { "$ref": "#/$defs/entity_present" }, + { "$ref": "#/$defs/entity_absent" }, + { "$ref": "#/$defs/expected_monologue" }, + { "$ref": "#/$defs/expected_dialogue" }, + { "$ref": "#/$defs/expected_interaction_verb" } + ] + }, + "minItems": 1 + } + }, + "$defs": { + "condition_id": { + "type": "string", + "pattern": "^[a-z][a-z0-9_-]*$", + "description": "Unique condition identifier within this checklist" + }, + "player_near": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description", "condition_type", "x", "y", "radius"], + "properties": { + "id": { "$ref": "#/$defs/condition_id" }, + "description": { "type": "string" }, + "condition_type": { "const": "player_near" }, + "x": { "type": "integer", "description": "Target X coordinate (absolute sim tiles)" }, + "y": { "type": "integer", "description": "Target Y coordinate (absolute sim tiles)" }, + "radius": { "type": "number", "minimum": 0, "description": "Proximity radius in tiles" } + } + }, + "player_facing": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description", "condition_type", "direction"], + "properties": { + "id": { "$ref": "#/$defs/condition_id" }, + "description": { "type": "string" }, + "condition_type": { "const": "player_facing" }, + "direction": { + "type": "string", + "enum": ["North", "South", "East", "West"], + "description": "Expected player facing direction" + } + } + }, + "entity_present": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description", "condition_type", "entity_id"], + "properties": { + "id": { "$ref": "#/$defs/condition_id" }, + "description": { "type": "string" }, + "condition_type": { "const": "entity_present" }, + "entity_id": { "type": "integer", "minimum": 0, "description": "StableId of expected entity" } + } + }, + "entity_absent": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description", "condition_type", "entity_id"], + "properties": { + "id": { "$ref": "#/$defs/condition_id" }, + "description": { "type": "string" }, + "condition_type": { "const": "entity_absent" }, + "entity_id": { "type": "integer", "minimum": 0, "description": "StableId of entity that should NOT be visible" } + } + }, + "expected_monologue": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description", "condition_type", "contains"], + "properties": { + "id": { "$ref": "#/$defs/condition_id" }, + "description": { "type": "string" }, + "condition_type": { "const": "expected_monologue" }, + "contains": { "type": "string", "minLength": 1, "description": "Substring expected in current_monologue text" } + } + }, + "expected_dialogue": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description", "condition_type", "contains"], + "properties": { + "id": { "$ref": "#/$defs/condition_id" }, + "description": { "type": "string" }, + "condition_type": { "const": "expected_dialogue" }, + "contains": { "type": "string", "minLength": 1, "description": "Substring expected in dialogue_response text" } + } + }, + "expected_interaction_verb": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description", "condition_type", "entity_id", "verb"], + "properties": { + "id": { "$ref": "#/$defs/condition_id" }, + "description": { "type": "string" }, + "condition_type": { "const": "expected_interaction_verb" }, + "entity_id": { "type": "integer", "minimum": 0, "description": "StableId of target entity" }, + "verb": { "type": "string", "minLength": 1, "description": "Expected interaction verb (e.g., Talk, Pickup, Examine)" } + } + } + } +} diff --git a/content/gauntlet/cross_room_checks.yaml b/content/gauntlet/cross_room_checks.yaml new file mode 100644 index 000000000..6890cd850 --- /dev/null +++ b/content/gauntlet/cross_room_checks.yaml @@ -0,0 +1,6 @@ +description: "Cross-room checks applied to every Gauntlet room visit." +conditions: + - id: cross-player-entity-present + description: "Player entity (StableId 0) always present in snapshot" + condition_type: entity_present + entity_id: 0 diff --git a/content/gauntlet/rooms/inventory_warehouse/checklist.yaml b/content/gauntlet/rooms/inventory_warehouse/checklist.yaml new file mode 100644 index 000000000..09cefdf8a --- /dev/null +++ b/content/gauntlet/rooms/inventory_warehouse/checklist.yaml @@ -0,0 +1,42 @@ +room_id: inventory_warehouse +room_name: Inventory Warehouse +description: "Tests D-065 (9-slot inventory), pickup/drop verbs, CarriedBy component. 10 crates (StableId 13-22) + 1 NPC (StableId 23)." +conditions: + - id: inv-player-spawn + description: "Player starts near warehouse spawn position" + condition_type: player_near + x: 17 + y: 54 + radius: 2.0 + + - id: inv-player-facing-east + description: "Player faces east at spawn" + condition_type: player_facing + direction: East + + - id: inv-crate-01-visible + description: "First crate (Keycard) is visible from spawn" + condition_type: entity_present + entity_id: 13 + + - id: inv-crate-10-visible + description: "Last crate (Chip) is visible" + condition_type: entity_present + entity_id: 22 + + - id: inv-npc-warehouse-visible + description: "Warehouse supervisor NPC is present" + condition_type: entity_present + entity_id: 23 + + - id: inv-crate-pickup-verb + description: "Pickup verb available on crate when nearby" + condition_type: expected_interaction_verb + entity_id: 13 + verb: Pickup + + - id: inv-npc-talk-verb + description: "Talk verb available on warehouse NPC" + condition_type: expected_interaction_verb + entity_id: 23 + verb: Talk diff --git a/content/gauntlet/rooms/occlusion_corridor/checklist.yaml b/content/gauntlet/rooms/occlusion_corridor/checklist.yaml new file mode 100644 index 000000000..f45b711c9 --- /dev/null +++ b/content/gauntlet/rooms/occlusion_corridor/checklist.yaml @@ -0,0 +1,41 @@ +room_id: occlusion_corridor +room_name: Occlusion Corridor +description: "Tests D-035 (symmetric shadowcasting), D-017 (perception modes), D-015 (vision cone sectors). 4 NPCs (StableId 9-12)." +conditions: + - id: occ-player-spawn + description: "Player starts at corridor entrance" + condition_type: player_near + x: 84 + y: 58 + radius: 2.0 + + - id: occ-player-facing-east + description: "Player faces east down the corridor" + condition_type: player_facing + direction: East + + - id: occ-guard-visible + description: "Guard NPC in clear LOS from spawn" + condition_type: entity_present + entity_id: 9 + + - id: occ-hidden-npc-absent + description: "NPC behind north wall segment is NOT visible from spawn" + condition_type: entity_absent + entity_id: 10 + + - id: occ-peripheral-npc-visible + description: "South alcove NPC visible in peripheral vision sector" + condition_type: entity_present + entity_id: 11 + + - id: occ-far-end-visible + description: "Far end NPC visible down corridor" + condition_type: entity_present + entity_id: 12 + + - id: occ-guard-talk-verb + description: "Talk verb available on guard NPC when nearby" + condition_type: expected_interaction_verb + entity_id: 9 + verb: Talk diff --git a/content/gauntlet/rooms/pause_chamber/checklist.yaml b/content/gauntlet/rooms/pause_chamber/checklist.yaml new file mode 100644 index 000000000..ccba2d03d --- /dev/null +++ b/content/gauntlet/rooms/pause_chamber/checklist.yaml @@ -0,0 +1,26 @@ +room_id: pause_chamber +room_name: Pause Chamber +description: "Tests D-031 (pause/unpause), Bug #3 regression (movement while paused). 1 NPC (StableId 29)." +conditions: + - id: pause-player-spawn + description: "Player starts at chamber center" + condition_type: player_near + x: 50 + y: 86 + radius: 2.0 + + - id: pause-player-facing-north + description: "Player faces north at spawn" + condition_type: player_facing + direction: North + + - id: pause-npc-present + description: "Pause target NPC is visible" + condition_type: entity_present + entity_id: 29 + + - id: pause-talk-verb + description: "Talk verb available on NPC (should work while paused)" + condition_type: expected_interaction_verb + entity_id: 29 + verb: Talk diff --git a/docs/DEVOPS.md b/docs/DEVOPS.md index 162c07568..e63c955bd 100644 --- a/docs/DEVOPS.md +++ b/docs/DEVOPS.md @@ -88,6 +88,24 @@ Regenerate both after any protocol change. Commit the updated fixtures alongside - **`gdscript_generated_fixtures_deserialize` fails:** Fixtures in `server/tests/fixtures/gdscript/` are stale or corrupted. Re-run `make fixtures-client` and commit the updated files. - **Fixture staleness in `make pre-pr`:** Protocol changed but fixtures were not regenerated. Run `make fixtures && make fixtures-client`, then commit both `client/tests/fixtures/` and `server/tests/fixtures/gdscript/`. +### Golden File Management + +```bash +make golden-diff # Show diff if golden file output has changed +make golden-update # Regenerate golden file and stage for commit +``` + +The golden file (`server/tests/golden/proof_room_tick_10.json`) is a committed snapshot of ObserverSnapshot output after a deterministic 10-tick replay. It catches unintentional changes to simulation output. + +**Workflow after intentional simulation changes:** + +1. Run `make golden-diff` to see what changed +2. Review the diff — confirm changes are expected +3. Run `make golden-update` to regenerate and stage the new golden file +4. Commit the updated golden file alongside your simulation change + +`golden-diff` exits 1 if the golden file has changed (useful in scripts). `golden-update` regenerates the file and runs `git add` but does not commit — the developer reviews and commits manually. + ### Lint ```bash @@ -149,6 +167,19 @@ make validate-content # Validate content YAML against JSON schemas make check-fact-ids # Check fact_id references against knowledge catalogs ``` +### Gauntlet Checklists + +```bash +make checklist-validate # Validate checklist YAML against schema (standalone) +make checklist-generate # Validate + print per-room condition summary +``` + +Checklists live at `content/gauntlet/rooms/{room_id}/checklist.yaml` (per-room) and `content/gauntlet/cross_room_checks.yaml` (cross-room). Each condition is evaluable from an `ObserverSnapshot`. + +7 condition types: `player_near`, `player_facing`, `entity_present`, `entity_absent`, `expected_monologue`, `expected_dialogue`, `expected_interaction_verb`. + +Schema: `content/_schema/checklist.schema.json`. The checklist format feeds into #503 (client auto-checklist progress tracking). + `check-fact-ids` operates in two modes: - **Advisory** — when knowledge catalogs (`content/global/knowledge/*.yaml`) have no fact definitions yet: lists referenced fact_ids and exits cleanly. - **Enforcing** — when catalogs are populated: fails on any `fact_id` reference that doesn't match a canonical definition. diff --git a/tooling/validate-checklist b/tooling/validate-checklist new file mode 100755 index 000000000..30a0699fc --- /dev/null +++ b/tooling/validate-checklist @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Validate Gauntlet checklist YAML files against the checklist JSON schema. + +Usage: + validate-checklist Validate all checklists + print summary + validate-checklist --check Schema validation only (for pre-PR chain) + +Exit code 0 = all valid, 1 = validation errors found. +""" + +import json +import sys +from pathlib import Path + +import jsonschema +import yaml + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_PATH = ROOT / "content" / "_schema" / "checklist.schema.json" +GAUNTLET_DIR = ROOT / "content" / "gauntlet" + + +def load_schema(): + with open(SCHEMA_PATH) as f: + return json.load(f) + + +def find_checklists(): + """Find all checklist YAML files under content/gauntlet/.""" + files = [] + if not GAUNTLET_DIR.exists(): + return files + for path in sorted(GAUNTLET_DIR.rglob("checklist.yaml")): + files.append(path) + cross = GAUNTLET_DIR / "cross_room_checks.yaml" + if cross.exists(): + files.append(cross) + return files + + +def _rel(path: Path) -> str: + try: + return str(path.relative_to(ROOT)) + except ValueError: + return str(path) + + +def validate_schema(files, schema): + """Pass 1: JSON Schema validation. Returns (validated, errors).""" + errors = 0 + validated = 0 + for path in files: + rel = _rel(path) + try: + with open(path) as f: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + print(f"YAML ERROR: {rel}: {e}") + errors += 1 + continue + + if data is None: + print(f"EMPTY: {rel}") + errors += 1 + continue + + try: + jsonschema.validate(instance=data, schema=schema) + validated += 1 + except jsonschema.ValidationError as e: + print(f"INVALID: {rel}") + print(f" Error: {e.message}") + if e.absolute_path: + print(f" Path: {'.'.join(str(p) for p in e.absolute_path)}") + errors += 1 + + return validated, errors + + +def check_id_uniqueness(files): + """Pass 2: Condition ID uniqueness within and across files.""" + errors = 0 + global_ids: dict[str, Path] = {} + + for path in files: + rel = _rel(path) + try: + with open(path) as f: + data = yaml.safe_load(f) + except (yaml.YAMLError, OSError): + continue + if not data or not isinstance(data, dict): + continue + + local_seen: set[str] = set() + for cond in data.get("conditions", []): + cid = cond.get("id") + if not cid: + continue + if cid in local_seen: + print(f'ID ERROR: duplicate condition id "{cid}" in {rel}') + errors += 1 + local_seen.add(cid) + + if cid in global_ids and global_ids[cid] != path: + print(f'ID ERROR: condition id "{cid}" used in multiple files') + print(f" First: {_rel(global_ids[cid])}") + print(f" Also: {rel}") + errors += 1 + elif cid not in global_ids: + global_ids[cid] = path + + return errors + + +def summarize(files): + """Print per-room condition counts and type breakdown.""" + total = 0 + type_counts: dict[str, int] = {} + + for path in files: + try: + with open(path) as f: + data = yaml.safe_load(f) + except (yaml.YAMLError, OSError): + continue + if not data or not isinstance(data, dict): + continue + + conditions = data.get("conditions", []) + count = len(conditions) + total += count + room = data.get("room_id", "cross_room") + print(f" {room}: {count} conditions") + + for cond in conditions: + ct = cond.get("condition_type", "unknown") + type_counts[ct] = type_counts.get(ct, 0) + 1 + + print(f"\n Total: {total} conditions across {len(files)} files") + if type_counts: + print(" By type:") + for ct in sorted(type_counts): + print(f" {ct}: {type_counts[ct]}") + + +def main(): + check_only = "--check" in sys.argv + + if not GAUNTLET_DIR.exists(): + print(f"No gauntlet directory at {_rel(GAUNTLET_DIR)}") + print("Checklist validation skipped (no content yet).") + return 0 + + files = find_checklists() + if not files: + print("No checklist files found under content/gauntlet/.") + print("Checklist validation skipped.") + return 0 + + schema = load_schema() + + # Pass 1: Schema validation + validated, schema_errors = validate_schema(files, schema) + print(f"Pass 1 (schema): {validated} valid, {schema_errors} errors") + + if schema_errors > 0: + print(f"\nSchema validation failed ({schema_errors} errors) — skipping ID checks") + return 1 + + # Pass 2: Condition ID uniqueness + id_errors = check_id_uniqueness(files) + if id_errors > 0: + print(f"Pass 2 (IDs): {id_errors} errors") + + total_errors = schema_errors + id_errors + print(f"\nChecklist validation: {validated} valid, {total_errors} errors") + + if total_errors > 0: + return 1 + + if not check_only: + print("\nChecklist summary:") + summarize(files) + + return 0 + + +if __name__ == "__main__": + sys.exit(main())