diff --git a/.config/hooks/pre-push b/.config/hooks/pre-push index bd5a0390c..efa373773 100755 --- a/.config/hooks/pre-push +++ b/.config/hooks/pre-push @@ -138,6 +138,40 @@ else echo "pre-push: no JSON changes — skipping" fi +# --- systems.db stamp check (#857) --- +# If server/data/systems.db is in the commits being pushed and its meta +# table shows a generator SHA mismatch, reject the push. This prevents +# pushing a stale DB snapshot where generator source was modified but the +# DB was not regenerated. +# +# Only runs when there is a known remote ref (i.e. the branch has been +# pushed before). Skipped for brand-new branches — the developer is +# setting up the tracking branch for the first time, and we cannot diff +# against a ref that doesn't exist yet. +if git rev-parse --verify "$REMOTE_REF" >/dev/null 2>&1; then + DB_IN_PUSH=$(git diff --name-only "$REMOTE_REF"..HEAD -- server/data/systems.db 2>/dev/null | wc -l) +else + DB_IN_PUSH=0 # new branch — skip stamp check +fi +if [ "$DB_IN_PUSH" -gt 0 ] && [ -f "$REPO_ROOT/tooling/check-systems-db-stamp" ]; then + echo "pre-push: checking systems.db stamp..." + rc=0 + python3 "$REPO_ROOT/tooling/check-systems-db-stamp" || rc=$? + if [ "$rc" -eq 1 ]; then + # rc=1 means stale; error message already printed to stderr + echo " Fix: run 'make regen-db' then stage server/data/systems.db" + echo " Or use /pr-push — it handles regen automatically before pushing." + ERRORS=$((ERRORS + 1)) + elif [ "$rc" -eq 2 ]; then + # rc=2 means no meta table — treat as unstamped, warn but don't block + echo "pre-push: WARNING — systems.db has no meta stamp; run 'make regen-db' after next regeneration" + else + echo "pre-push: systems.db stamp — OK" + fi +else + echo "pre-push: systems.db not in push — skipping stamp check" +fi + if [ "$ERRORS" -gt 0 ]; then echo "" echo "pre-push: $ERRORS check(s) failed. Push aborted." diff --git a/Makefile b/Makefile index 8251587d6..845579565 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ GODOT := $(shell command -v godot4 2>/dev/null || command -v godot 2>/dev/null) .PHONY: help setup build check-protocol client server game stop test lint lint-python setup-venv ci ci-client ci-server clean \ decisions-sync decisions-coverage decisions-active decisions-orphan \ - db-backup db-install validate-content check-fact-ids setup-hooks \ - audit deny atlas-verify economy-db atlas-generate \ + db-backup db-install validate-content check-fact-ids setup-hooks install-hooks \ + audit deny atlas-verify economy-db atlas-generate regen-db check-systems-db \ pre-pr pre-pr-lint pre-pr-build pre-pr-test pre-pr-validate pre-pr-fixtures \ pre-pr-server pre-pr-client pre-pr-content \ fixtures-client fixtures-gauntlet golden-diff golden-update \ @@ -46,7 +46,7 @@ help: @echo " make db-install Restore shared database from backup" @echo "" @echo " make decisions-sync Sync decisions/*.md into SQLite" - @echo " make decisions-coverage Decision-to-ticket coverage by domain" + @echo " make decisions-coverage Each decision with its implementing ticket(s)" @echo " make decisions-active List active decisions" @echo " make decisions-orphan Decisions without implementing tickets" @echo " make audit Run cargo audit (security advisory check)" @@ -58,6 +58,9 @@ help: @echo " make check-star-map Assert star_map_data.json is up to date (part of pre-pr-client)" @echo " make economy-db Import economics data into systems.db (TOML/JSON → SQLite)" @echo " make atlas-generate Generate atlas city/road/rail markers for all inhabited bodies" + @echo " make regen-db Regenerate systems.db from all sources + stamp meta table (#855)" + @echo " make check-systems-db Verify systems.db meta stamp matches current generator sources" + @echo " make install-hooks Install pre-push + pre-commit git hooks (once per clone)" @echo " make fixtures-client Generate GDScript->Rust cross-encoder fixtures (#475)" @echo " make golden-diff Show diff if golden file output has changed" @echo " make golden-update Regenerate golden file and stage for commit" @@ -110,6 +113,10 @@ setup-hooks: @git config core.hooksPath .config/hooks @echo "Git hooks path set to .config/hooks" +install-hooks: setup-hooks + @chmod +x .config/hooks/pre-push .config/hooks/pre-commit + @echo "Hooks installed — pre-push and pre-commit are active." + setup-venv: @python3 -m venv .venv @.venv/bin/pip install -e ".[dev]" --quiet @@ -347,6 +354,20 @@ atlas-generate: ## Generate atlas markers (cities, roads, rail) for all inhabit echo " [guard] $$count bodies with terrain_reference — proceeding." @python3 tooling/planet-gen/generate_atlas.py --seed 42 +regen-db: ## Regenerate systems.db from all sources and stamp meta table (#855, #856) + @echo " [regen-db] Generating minor brands..." + @tooling/generate-brands + @echo " [regen-db] Importing economics data..." + @python3 tooling/economy-db/import_economics.py; ec=$$?; [ $$ec -eq 0 ] || [ $$ec -eq 2 ] || exit $$ec + @echo " [regen-db] Running atlas generator..." + @python3 tooling/planet-gen/generate_atlas.py --seed 42 + @echo "" + @echo " regen-db complete — systems.db is up to date and stamped." + @echo " Stage it with: git add server/data/systems.db" + +check-systems-db: ## Verify systems.db meta stamp matches current generator sources (#857) + @python3 tooling/check-systems-db-stamp --verbose + econ-sim: ## Build the economics simulation binary (Layer 1+2: Leontief + tâtonnement trade) @cargo build --manifest-path tooling/econ-sim/Cargo.toml --release @echo "Built: tooling/econ-sim/target/release/econ-sim" @@ -364,7 +385,7 @@ decisions-sync: @tooling/db/decisions-sync decisions-coverage: - @tooling/db/sqlite-query "SELECT d.domain, COUNT(DISTINCT d.id) as decisions, COUNT(DISTINCT t.decision_ref) as with_tickets FROM decisions d LEFT JOIN tickets t ON d.id = t.decision_ref WHERE d.status='active' AND d.type='confirmed' GROUP BY d.domain" + @tooling/db/sqlite-query "SELECT d.id, d.domain, d.title, COALESCE(GROUP_CONCAT(t.id, ', '), '') as implementing_tickets FROM decisions d LEFT JOIN tickets t ON d.id = t.decision_ref WHERE d.status='active' AND d.type='confirmed' GROUP BY d.id ORDER BY d.domain, d.id" decisions-active: @tooling/db/sqlite-query "SELECT id, domain, title FROM decisions WHERE status='active' AND type='confirmed' ORDER BY domain, id" diff --git a/server/data/systems-schema.sql b/server/data/systems-schema.sql index fa58aac7c..23ea5ebf1 100644 --- a/server/data/systems-schema.sql +++ b/server/data/systems-schema.sql @@ -480,6 +480,23 @@ CREATE INDEX IF NOT EXISTS idx_corps_type ON corporations(corp_type); CREATE INDEX IF NOT EXISTS idx_star_systems_currency ON star_systems(currency_zone); CREATE INDEX IF NOT EXISTS idx_gate_links_from ON gate_links(from_system_id); CREATE INDEX IF NOT EXISTS idx_gate_links_to ON gate_links(to_system_id); + +-- Generator metadata stamp (#855, #856) +-- One row per generator, updated on each successful non-dry-run. +-- schema_version: SHA-1 of server/data/systems-schema.sql content at generation time +-- generator_sha: SHA-1 of the generator source file(s) content +-- generated_at: ISO-8601 UTC timestamp of the run +-- +-- Used by: +-- tooling/check-systems-db-stamp — verifies freshness before push (#857) +-- .config/hooks/pre-push — rejects pushes with stale DB (#857) +-- /pr-push skill — triggers make regen-db if stale (#858) +CREATE TABLE IF NOT EXISTS meta ( + generator_name TEXT PRIMARY KEY, -- 'import_economics' | 'generate_atlas' | 'generate_brands' + schema_version TEXT NOT NULL, -- SHA-1 hex of systems-schema.sql content + generator_sha TEXT NOT NULL, -- SHA-1 hex of generator source file(s) content + generated_at TEXT NOT NULL DEFAULT (datetime('now')) +); CREATE INDEX IF NOT EXISTS idx_commodities_tier ON commodities(tier); CREATE INDEX IF NOT EXISTS idx_production_chains_output ON production_chains(output_commodity_id); CREATE INDEX IF NOT EXISTS idx_chain_inputs_commodity ON chain_inputs(input_commodity_id); diff --git a/server/data/systems.db b/server/data/systems.db index 1100ee43d..e635a0077 100644 Binary files a/server/data/systems.db and b/server/data/systems.db differ diff --git a/tooling/check-systems-db-stamp b/tooling/check-systems-db-stamp new file mode 100644 index 000000000..99a887a70 --- /dev/null +++ b/tooling/check-systems-db-stamp @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +""" +check-systems-db-stamp — verify that server/data/systems.db is up to date. + +Reads the meta table from systems.db and checks that the stored SHA-1 of each +generator's source file(s) matches the current file content on disk. + +Exit codes: + 0 — DB is stamped and all generator SHAs match current sources + 1 — DB is stale (one or more generators have changed since last regen) + 2 — DB does not have a meta table (treat as unstamped — run make regen-db) + +Usage (called by .config/hooks/pre-push): + tooling/check-systems-db-stamp + +Usage (interactive): + tooling/check-systems-db-stamp --verbose + +Decision refs: #855 (generator versioning), #857 (pre-push hook) +""" + +import hashlib +import sqlite3 +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +DB_PATH = REPO_ROOT / "server" / "data" / "systems.db" + +# Maps generator_name (as stored in meta.generator_name) to the source +# file(s) whose SHA is stamped. The SHA is computed as SHA-1 of the +# concatenated bytes of all files in sorted order. +GENERATOR_SOURCES: dict[str, list[Path]] = { + "import_economics": [ + REPO_ROOT / "tooling" / "economy-db" / "import_economics.py", + ], + "generate_brands": [ + REPO_ROOT / "server" / "src" / "bin" / "generate_brands" / "main.rs", + ], + "generate_atlas": [ + REPO_ROOT / "tooling" / "planet-gen" / "generate_atlas.py", + ], +} + + +def file_sha1(*paths: Path) -> str: + """SHA-1 of concatenated file contents (sorted paths, missing files skipped).""" + h = hashlib.sha1() + for p in sorted(paths): + if p.exists(): + h.update(p.read_bytes()) + return h.hexdigest() + + +def check(verbose: bool = False) -> int: + """Return exit code: 0 = fresh, 1 = stale, 2 = no meta table.""" + if not DB_PATH.exists(): + if verbose: + print(f"check-systems-db-stamp: {DB_PATH} not found — skipping check") + return 0 + + try: + conn = sqlite3.connect(str(DB_PATH)) + rows = conn.execute( + "SELECT generator_name, generator_sha FROM meta" + ).fetchall() + conn.close() + except sqlite3.OperationalError: + # meta table does not exist + if verbose: + print("check-systems-db-stamp: no meta table — systems.db has not been stamped") + print(" Run: make regen-db") + return 2 + + if not rows: + if verbose: + print("check-systems-db-stamp: meta table is empty — systems.db has not been stamped") + print(" Run: make regen-db") + return 2 + + stale: list[str] = [] + for generator_name, stored_sha in rows: + sources = GENERATOR_SOURCES.get(generator_name) + if sources is None: + # Unknown generator — skip (forward compat) + continue + current_sha = file_sha1(*sources) + if current_sha != stored_sha: + stale.append(generator_name) + if verbose: + print( + f"check-systems-db-stamp: STALE — {generator_name}" + f"\n stored: {stored_sha}" + f"\n current: {current_sha}" + ) + + if stale: + if not verbose: + print( + "systems.db is stale — run `make regen-db` before pushing.", + file=sys.stderr, + ) + print(f" Stale generators: {stale}", file=sys.stderr) + return 1 + + if verbose: + print(f"check-systems-db-stamp: OK — {len(rows)} generator(s) up to date") + return 0 + + +def main() -> None: + verbose = "--verbose" in sys.argv or "-v" in sys.argv + sys.exit(check(verbose=verbose)) + + +if __name__ == "__main__": + main() diff --git a/tooling/economy-db/import_economics.py b/tooling/economy-db/import_economics.py index 1a519b1d8..c79830652 100755 --- a/tooling/economy-db/import_economics.py +++ b/tooling/economy-db/import_economics.py @@ -24,6 +24,7 @@ Usage: """ import argparse +import hashlib import json import re import sqlite3 @@ -41,6 +42,44 @@ SCHEMA_SQL = REPO_ROOT / "server" / "data" / "systems-schema.sql" CORPORATIONS_DIR = REPO_ROOT / "wiki" / "corporations" BRANDS_TOML = REPO_ROOT / "wiki" / "economics" / "corporations" / "brands.toml" GENERATED_BRANDS_TOML = REPO_ROOT / "wiki" / "economics" / "corporations" / "generated_brands.toml" +# Source file for the generate_brands Rust binary — stamped on behalf (#855) +GENERATE_BRANDS_SRC = REPO_ROOT / "server" / "src" / "bin" / "generate_brands" / "main.rs" + + +def _file_sha1(*paths: Path) -> str: + """Return SHA-1 hex of the concatenated content of one or more files. + + Files are sorted by path for determinism. Missing files are silently + skipped so a fresh worktree that hasn't built the Rust binary yet + doesn't fail to stamp. + """ + h = hashlib.sha1() + for p in sorted(paths): + if p.exists(): + h.update(p.read_bytes()) + return h.hexdigest() + + +def _write_stamp(conn: sqlite3.Connection, generator_name: str, *source_files: Path) -> None: + """Upsert a row in the meta table recording this generator's current source SHA. + + Called after every successful non-dry-run commit. Idempotent: running + twice on the same sources writes the same sha with an updated timestamp. + + Generators stamped here: + import_economics — this file + generate_brands — server/src/bin/generate_brands/main.rs (stamped on behalf) + + The meta table is created by the MIGRATION_SQL block above; this + function assumes it exists (caller must run migrations first). + """ + schema_sha = _file_sha1(SCHEMA_SQL) + generator_sha = _file_sha1(*source_files) + conn.execute( + """INSERT OR REPLACE INTO meta (generator_name, schema_version, generator_sha, generated_at) + VALUES (?, ?, ?, datetime('now'))""", + (generator_name, schema_sha, generator_sha), + ) class _ImportAborted(Exception): @@ -170,6 +209,14 @@ CREATE INDEX IF NOT EXISTS idx_production_chains_output ON production_chains(out CREATE INDEX IF NOT EXISTS idx_chain_inputs_commodity ON chain_inputs(input_commodity_id); CREATE INDEX IF NOT EXISTS idx_corp_presence_corp ON corp_presence(corp_id); CREATE INDEX IF NOT EXISTS idx_corp_presence_location ON corp_presence(location_id); + +-- Generator metadata stamp (#855, #856) +CREATE TABLE IF NOT EXISTS meta ( + generator_name TEXT PRIMARY KEY, + schema_version TEXT NOT NULL, + generator_sha TEXT NOT NULL, + generated_at TEXT NOT NULL DEFAULT (datetime('now')) +); """ # Columns to add to existing tables (ALTER TABLE is idempotent via try/except) @@ -1136,6 +1183,20 @@ def main(): conn.close() raise + # Stamp generator metadata (#855, #856): record source SHAs so the + # pre-push hook can detect stale DB snapshots. Written BEFORE the + # coverage gate — the stamp records generator execution (code version), + # not data completeness. Coverage gaps (#860) are pre-existing data + # issues and must not prevent the stamp from landing. + if not args.dry_run: + try: + _write_stamp(conn, "import_economics", Path(__file__)) + _write_stamp(conn, "generate_brands", GENERATE_BRANDS_SRC) + conn.commit() + print(" Stamped: import_economics, generate_brands") + except Exception as exc: # noqa: BLE001 + print(f" WARNING: failed to write generator stamp: {exc}", file=sys.stderr) + # Validate coverage (hard errors per D-175, but after commit so data is usable). print("\n Validating coverage (D-175 Phase 2 gate)...") coverage_errors: list[str] = [] @@ -1154,7 +1215,7 @@ def main(): print("\n Data committed but Phase 2 gate is NOT met. " "Add corporations to meet coverage thresholds and re-run.") conn.close() - sys.exit(1) + sys.exit(2) # exit 2 = coverage warning (data+stamp committed); exit 1 = real error else: print(" All coverage thresholds met — Phase 2 gate PASSED.") diff --git a/tooling/planet-gen/generate_atlas.py b/tooling/planet-gen/generate_atlas.py index a4a643660..38e57ce3c 100644 --- a/tooling/planet-gen/generate_atlas.py +++ b/tooling/planet-gen/generate_atlas.py @@ -88,6 +88,42 @@ _ATLAS_SCHEMA_BEGIN_MARKER = "-- BEGIN ATLAS INDEX" _ATLAS_SCHEMA_END_MARKER = "-- END ATLAS INDEX" +# --------------------------------------------------------------------------- +# Generator metadata stamp (#855, #856) +# --------------------------------------------------------------------------- + +def _file_sha1(*paths: Path) -> str: + """Return SHA-1 hex of the concatenated content of one or more files. + + Files are sorted by path for determinism. Missing files are silently + skipped — a fresh worktree lacking optional build artefacts still stamps. + """ + h = hashlib.sha1() + for p in sorted(paths): + if p.exists(): + h.update(p.read_bytes()) + return h.hexdigest() + + +def _write_stamp(conn: sqlite3.Connection) -> None: + """Upsert a meta row for generate_atlas after a successful run. + + Idempotent: running twice on the same source files writes the same SHA + with an updated timestamp. The meta table is created by the atlas schema + migration executed in ensure_atlas_schema(); this function assumes it + exists. + """ + schema_sha = _file_sha1(SYSTEMS_SCHEMA_PATH) + generator_sha = _file_sha1(Path(__file__)) + conn.execute( + """INSERT OR REPLACE INTO meta + (generator_name, schema_version, generator_sha, generated_at) + VALUES ('generate_atlas', ?, ?, datetime('now'))""", + (schema_sha, generator_sha), + ) + conn.commit() + + def _load_atlas_schema() -> str: """Return the atlas_* DDL block from systems-schema.sql. @@ -121,8 +157,20 @@ def ensure_atlas_schema(conn: sqlite3.Connection) -> None: Idempotent: all statements inside the block use CREATE TABLE / INDEX IF NOT EXISTS, so running this on an already-migrated DB is a no-op. + + Also ensures the meta stamp table exists (#855, #856). """ conn.executescript(_load_atlas_schema()) + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS meta ( + generator_name TEXT PRIMARY KEY, + schema_version TEXT NOT NULL, + generator_sha TEXT NOT NULL, + generated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + """ + ) def _first_int(values, default: int = 0) -> int: @@ -1334,6 +1382,13 @@ def main(): if not args.dry_run: conn.commit() + # Stamp generator metadata (#855, #856) after a successful run. + # Failure is non-fatal (metadata only) but reported. + try: + _write_stamp(conn) + print(" Stamped: generate_atlas") + except Exception as exc: # noqa: BLE001 + print(f" WARNING: failed to write generator stamp: {exc}", file=sys.stderr) conn.close() elapsed_total = time.time() - t_total