#!/usr/bin/env python3 """ Import economics data into systems.db. Reads TOML/JSON source files and populates the economics tables: - gate_links from docs/design/star-map.json (335 edges, bidirectional) - commodities from wiki/economics/commodities.toml (36 types) - production_chains + chain_inputs from wiki/economics/production_chains.toml - currency_zone on star_systems (default TRACTUS_PRIMARY) - gate_energy_connected on star_systems (D-186: false for MARK_PRIMARY zones) - corporations from wiki/corporations/*.md (sync + insert new records) - corp_presence from wiki/corporations/*.md (headquarters location data) Validation (hard errors, non-zero exit on any failure): - Wiki corporation names must match DB proper_name records (D-182 sync constraint) - Chain completeness: every intermediate commodity has at least one production chain - Commodity coverage: 3+ corporations per major commodity type (D-175) - System coverage: 1+ corporation per inhabited system with population > 100K (D-175) Usage: python3 tooling/economy-db/import_economics.py python3 tooling/economy-db/import_economics.py --dry-run python3 tooling/economy-db/import_economics.py --db path/to/systems.db """ import argparse import json import re import sqlite3 import sys import tomllib from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent.parent DB_PATH = REPO_ROOT / "server" / "data" / "systems.db" STAR_MAP = REPO_ROOT / "docs" / "design" / "star-map.json" COMMODITIES_TOML = REPO_ROOT / "wiki" / "economics" / "commodities.toml" CHAINS_TOML = REPO_ROOT / "wiki" / "economics" / "production_chains.toml" SCHEMA_SQL = REPO_ROOT / "server" / "data" / "systems-schema.sql" CORPORATIONS_DIR = REPO_ROOT / "wiki" / "corporations" # --------------------------------------------------------------------------- # Schema migration — add new tables and columns to existing DB # --------------------------------------------------------------------------- MIGRATION_SQL = """ -- Economics tables (idempotent — safe to re-run) CREATE TABLE IF NOT EXISTS gate_links ( from_system_id TEXT NOT NULL REFERENCES star_systems(system_id), to_system_id TEXT NOT NULL REFERENCES star_systems(system_id), PRIMARY KEY (from_system_id, to_system_id) ); CREATE TABLE IF NOT EXISTS commodities ( commodity_id TEXT PRIMARY KEY, name TEXT NOT NULL, tier TEXT NOT NULL, elasticity TEXT NOT NULL, base_price REAL NOT NULL, bulk_class TEXT, unit TEXT, production_ubiquity TEXT, demand_model TEXT, commission_certifiable INTEGER DEFAULT 0, compact_contested INTEGER DEFAULT 0, shadow_viable INTEGER DEFAULT 0, panic_threshold_weeks INTEGER DEFAULT 0, description TEXT, updated_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS production_chains ( chain_id TEXT PRIMARY KEY, output_commodity_id TEXT NOT NULL REFERENCES commodities(commodity_id), output_quantity REAL NOT NULL DEFAULT 1.0, location_bound INTEGER DEFAULT 0, description TEXT, updated_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS chain_inputs ( chain_id TEXT NOT NULL REFERENCES production_chains(chain_id), input_commodity_id TEXT NOT NULL REFERENCES commodities(commodity_id), quantity REAL NOT NULL, PRIMARY KEY (chain_id, input_commodity_id) ); CREATE TABLE IF NOT EXISTS corp_presence ( corp_id TEXT NOT NULL REFERENCES corporations(corp_id), location_id TEXT NOT NULL, location_type TEXT NOT NULL, primary_operation TEXT, updated_at TEXT DEFAULT (datetime('now')), PRIMARY KEY (corp_id, location_id) ); -- New indexes CREATE INDEX IF NOT EXISTS idx_star_systems_currency ON star_systems(currency_zone); CREATE INDEX IF NOT EXISTS idx_gate_links_from ON gate_links(from_system_id); CREATE INDEX IF NOT EXISTS idx_gate_links_to ON gate_links(to_system_id); CREATE INDEX IF NOT EXISTS idx_commodities_tier ON commodities(tier); CREATE INDEX IF NOT EXISTS idx_production_chains_output ON production_chains(output_commodity_id); CREATE INDEX IF NOT EXISTS idx_chain_inputs_commodity ON chain_inputs(input_commodity_id); CREATE INDEX IF NOT EXISTS idx_corp_presence_corp ON corp_presence(corp_id); CREATE INDEX IF NOT EXISTS idx_corp_presence_location ON corp_presence(location_id); """ # Columns to add to existing tables (ALTER TABLE is idempotent via try/except) COLUMN_MIGRATIONS = [ ("star_systems", "currency_zone", "TEXT DEFAULT 'TRACTUS_PRIMARY'"), ("star_systems", "gate_energy_connected", "INTEGER DEFAULT 1"), ("corporations", "behavioral_archetype", "TEXT"), ("corporations", "supply_chain_role", "TEXT"), ("corporations", "shadow_economy_access", "INTEGER DEFAULT 0"), ] def _add_column(conn: sqlite3.Connection, table: str, col: str, col_type: str): """Add a column if it doesn't exist. SQLite has no IF NOT EXISTS for ALTER.""" try: conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} {col_type}") except sqlite3.OperationalError as e: if "duplicate column" in str(e).lower(): pass # already exists else: raise # --------------------------------------------------------------------------- # Gate links # --------------------------------------------------------------------------- def import_gate_links(conn: sqlite3.Connection, dry_run: bool) -> int: with open(STAR_MAP) as f: data = json.load(f) edges = data["edges"] system_ids = {r[0] for r in conn.execute("SELECT system_id FROM star_systems").fetchall()} rows = [] skipped = [] for a, b in edges: if a not in system_ids: skipped.append(a) continue if b not in system_ids: skipped.append(b) continue rows.append((a, b)) rows.append((b, a)) if skipped: unique_skipped = sorted(set(skipped)) print(f" warning: {len(unique_skipped)} system(s) in star-map.json not in DB: {unique_skipped[:5]}...") if not dry_run: conn.executemany( "INSERT OR IGNORE INTO gate_links (from_system_id, to_system_id) VALUES (?, ?)", rows, ) return len(rows) # --------------------------------------------------------------------------- # Commodities # --------------------------------------------------------------------------- def import_commodities(conn: sqlite3.Connection, dry_run: bool) -> int: with open(COMMODITIES_TOML, "rb") as f: data = tomllib.load(f) rows = [] for cid, c in data.items(): rows.append(( cid, c["name"], c["tier"], c["elasticity"], c["base_price"], c.get("bulk_class"), c.get("unit"), c.get("production_ubiquity"), c.get("demand_model"), int(c.get("commission_certifiable", False)), int(c.get("compact_contested", False)), int(c.get("shadow_viable", False)), c.get("panic_threshold_weeks", 0), c.get("description"), )) if not dry_run: conn.executemany( """INSERT INTO commodities ( commodity_id, name, tier, elasticity, base_price, bulk_class, unit, production_ubiquity, demand_model, commission_certifiable, compact_contested, shadow_viable, panic_threshold_weeks, description ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", rows, ) return len(rows) # --------------------------------------------------------------------------- # Production chains + inputs # --------------------------------------------------------------------------- def import_chains(conn: sqlite3.Connection, dry_run: bool) -> tuple[int, int]: with open(CHAINS_TOML, "rb") as f: data = tomllib.load(f) chain_rows = [] input_rows = [] for chain_id, c in data.items(): chain_rows.append(( chain_id, c["output"], c.get("output_quantity", 1.0), int(c.get("location_bound", False)), c.get("description"), )) for inp in c.get("inputs", []): input_rows.append(( chain_id, inp["commodity"], inp["quantity"], )) if not dry_run: conn.executemany( """INSERT INTO production_chains ( chain_id, output_commodity_id, output_quantity, location_bound, description ) VALUES (?, ?, ?, ?, ?)""", chain_rows, ) conn.executemany( """INSERT INTO chain_inputs ( chain_id, input_commodity_id, quantity ) VALUES (?, ?, ?)""", input_rows, ) return len(chain_rows), len(input_rows) # --------------------------------------------------------------------------- # Currency zones # --------------------------------------------------------------------------- def set_currency_zones(conn: sqlite3.Connection, dry_run: bool) -> dict: """Set currency_zone on star_systems. Default TRACTUS_PRIMARY, Sol = MIXED.""" if dry_run: return {"TRACTUS_PRIMARY": "all", "MIXED": "GJ 0"} # Default everything to TRACTUS_PRIMARY conn.execute("UPDATE star_systems SET currency_zone = 'TRACTUS_PRIMARY' WHERE currency_zone IS NULL") # Sol system is MIXED (Earth legacy currency presence) conn.execute("UPDATE star_systems SET currency_zone = 'MIXED' WHERE system_id = 'GJ 0'") # Future: Compact systems → MARK_PRIMARY (requires authored Compact membership data) counts = {} for row in conn.execute("SELECT currency_zone, COUNT(*) FROM star_systems GROUP BY currency_zone"): counts[row[0]] = row[1] return counts # --------------------------------------------------------------------------- # Gate energy connectivity (D-186) # --------------------------------------------------------------------------- def set_gate_energy(conn: sqlite3.Connection, dry_run: bool) -> dict: """Set gate_energy_connected on star_systems based on currency_zone. MARK_PRIMARY zones default to false (Compact refused Gate Corp dependency). All other zones default to true. """ if dry_run: return {"on_grid": "non-MARK_PRIMARY", "off_grid": "MARK_PRIMARY"} # Default: all systems on-grid conn.execute("UPDATE star_systems SET gate_energy_connected = 1 WHERE gate_energy_connected IS NULL") # MARK_PRIMARY zones are off-grid (Compact energy sovereignty) conn.execute("UPDATE star_systems SET gate_energy_connected = 0 WHERE currency_zone = 'MARK_PRIMARY'") counts = {} for row in conn.execute( "SELECT gate_energy_connected, COUNT(*) FROM star_systems GROUP BY gate_energy_connected" ): label = "on_grid" if row[0] == 1 else "off_grid" counts[label] = row[1] return counts # --------------------------------------------------------------------------- # Corporation wiki parsing # --------------------------------------------------------------------------- def _parse_corp_frontmatter(path: Path) -> dict | None: """Parse YAML frontmatter from a wiki corporation markdown file.""" text = path.read_text() lines = text.split("\n") if not lines or lines[0].strip() != "---": return None end_idx = None for i, line in enumerate(lines[1:], 1): if line.strip() == "---": end_idx = i break if end_idx is None: return None fm: dict = {} for line in lines[1:end_idx]: if ":" not in line: continue key, _, val = line.partition(":") key = key.strip() val = val.strip() if val.startswith("[") and val.endswith("]"): items = [x.strip().strip('"').strip("'") for x in val[1:-1].split(",")] fm[key] = [item for item in items if item] else: fm[key] = val.strip('"').strip("'") return fm def load_wiki_corps() -> list[dict]: """Load all wiki corporation files. Returns list of parsed corp records.""" corps = [] for md_file in sorted(CORPORATIONS_DIR.glob("*.md")): if md_file.name == "index.md": continue fm = _parse_corp_frontmatter(md_file) if not fm or not fm.get("slug") or not fm.get("title"): continue hq = fm.get("headquarters", "") m = re.search(r"\(([^)]+)\)", hq) system_id = m.group(1) if m else None corps.append({ "corp_id": fm["slug"], "proper_name": fm["title"], "system_id": system_id, "tags": fm.get("tags", []), "scope": fm.get("scope", ""), }) return corps # --------------------------------------------------------------------------- # Corporation sync (D-182: wiki is source of truth) # --------------------------------------------------------------------------- def sync_corporations( conn: sqlite3.Connection, wiki_corps: list[dict], dry_run: bool ) -> list[str]: """Sync wiki corps to DB. Hard error on proper_name divergence (D-182). Returns list of error strings. Inserts corps that exist in wiki but not DB. Corps that exist only in DB (legacy records) are left untouched. headquarters_system is only written if the system_id exists in star_systems (to avoid FK violations when atlas hasn't yet registered the system). """ errors: list[str] = [] existing = { r[0]: r[1] for r in conn.execute("SELECT corp_id, proper_name FROM corporations").fetchall() } valid_systems = { r[0] for r in conn.execute("SELECT system_id FROM star_systems").fetchall() } to_insert = [] for corp in wiki_corps: corp_id = corp["corp_id"] proper_name = corp["proper_name"] if corp_id in existing: if existing[corp_id] != proper_name: errors.append( f"name divergence: corp_id='{corp_id}' " f"wiki='{proper_name}' db='{existing[corp_id]}'" ) else: system_id = corp.get("system_id") hq_system = system_id if system_id and system_id in valid_systems else None if system_id and system_id not in valid_systems: print(f" warning: {corp_id} HQ system '{system_id}' not in DB, " f"headquarters_system set to NULL") to_insert.append(( corp_id, proper_name, "corporation", corp.get("scope") or None, hq_system, )) if not dry_run and not errors: conn.executemany( """INSERT OR IGNORE INTO corporations (corp_id, proper_name, corp_type, scope, headquarters_system) VALUES (?, ?, ?, ?, ?)""", to_insert, ) return errors # --------------------------------------------------------------------------- # Corp presence population # --------------------------------------------------------------------------- def import_corp_presence( conn: sqlite3.Connection, wiki_corps: list[dict], commodity_ids: set[str], dry_run: bool, ) -> int: """Populate corp_presence from wiki headquarters data. Each corporation gets one presence row at its headquarters system. primary_operation is set to the first commodity tag that matches a known commodity ID, or None if no commodity tags are present. """ valid_systems = { r[0] for r in conn.execute("SELECT system_id FROM star_systems").fetchall() } rows = [] skipped = [] for corp in wiki_corps: system_id = corp.get("system_id") if not system_id: skipped.append(f"{corp['corp_id']} (no headquarters system parsed)") continue if system_id not in valid_systems: skipped.append(f"{corp['corp_id']} (system '{system_id}' not in DB)") continue primary_op = next( (tag for tag in corp.get("tags", []) if tag in commodity_ids), None ) rows.append((corp["corp_id"], system_id, "system", primary_op)) if skipped: for s in skipped: print(f" warning: skipped corp_presence for {s}") if not dry_run: conn.execute("DELETE FROM corp_presence") conn.executemany( """INSERT OR IGNORE INTO corp_presence (corp_id, location_id, location_type, primary_operation) VALUES (?, ?, ?, ?)""", rows, ) return len(rows) # --------------------------------------------------------------------------- # Validation # --------------------------------------------------------------------------- def validate(conn: sqlite3.Connection) -> list[str]: """Validate structural integrity of imported data. Checks FK integrity, chain commodity references, and chain completeness. These are hard blockers — broken data must not be committed. Coverage validation (commodity/system thresholds) is separate and runs after commit via _validate_commodity_coverage() and _validate_system_coverage(). """ errors = [] # FK integrity fk_issues = conn.execute("PRAGMA foreign_key_check").fetchall() if fk_issues: for issue in fk_issues[:10]: errors.append(f"FK violation: table={issue[0]} rowid={issue[1]} " f"parent={issue[2]} fkid={issue[3]}") # Chain inputs reference valid commodities orphan_inputs = conn.execute(""" SELECT ci.chain_id, ci.input_commodity_id FROM chain_inputs ci LEFT JOIN commodities c ON ci.input_commodity_id = c.commodity_id WHERE c.commodity_id IS NULL """).fetchall() for chain_id, cid in orphan_inputs: errors.append(f"chain_inputs: chain '{chain_id}' references unknown commodity '{cid}'") # Chain outputs reference valid commodities orphan_outputs = conn.execute(""" SELECT pc.chain_id, pc.output_commodity_id FROM production_chains pc LEFT JOIN commodities c ON pc.output_commodity_id = c.commodity_id WHERE c.commodity_id IS NULL """).fetchall() for chain_id, cid in orphan_outputs: errors.append(f"production_chains: chain '{chain_id}' outputs unknown commodity '{cid}'") # Chain completeness: every intermediate commodity must have at least one producer missing_chains = conn.execute(""" SELECT c.commodity_id, c.name FROM commodities c WHERE c.tier = 'intermediate' AND c.commodity_id NOT IN (SELECT output_commodity_id FROM production_chains) ORDER BY c.commodity_id """).fetchall() for cid, name in missing_chains: errors.append(f"chain completeness: no production chain produces intermediate '{cid}' ({name})") return errors def _validate_commodity_coverage( conn: sqlite3.Connection, wiki_corps: list[dict], commodity_ids: set[str] ) -> list[str]: """3+ corporations per major commodity type (raw + intermediate). D-175.""" errors: list[str] = [] major = [ r[0] for r in conn.execute( "SELECT commodity_id FROM commodities " "WHERE tier IN ('raw', 'intermediate') ORDER BY commodity_id" ).fetchall() ] # Build commodity → corp set from wiki tags filtered to known commodity IDs coverage: dict[str, set[str]] = {cid: set() for cid in major} for corp in wiki_corps: for tag in corp.get("tags", []): if tag in coverage: coverage[tag].add(corp["corp_id"]) for cid in major: n = len(coverage[cid]) if n < 3: corp_list = sorted(coverage[cid]) if coverage[cid] else ["none"] errors.append( f"commodity coverage: '{cid}' has {n}/3 corp(s) — {corp_list}" ) return errors def _validate_system_coverage( conn: sqlite3.Connection, wiki_corps: list[dict] ) -> list[str]: """1+ corporation per inhabited system with population > 100K. D-175. Uses wiki_corps headquarters data (not DB corp_presence) so this check is accurate in both dry-run and real-run modes. """ covered = {c["system_id"] for c in wiki_corps if c.get("system_id")} populated = conn.execute(""" SELECT se.system_id, ss.proper_name, se.population FROM system_economy se JOIN star_systems ss ON se.system_id = ss.system_id WHERE se.population > 100000 ORDER BY se.system_id """).fetchall() return [ f"system coverage: no corp presence in '{sid}' ({name}, pop={pop:,})" for sid, name, pop in populated if sid not in covered ] # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- def main(): parser = argparse.ArgumentParser(description="Import economics data into systems.db") parser.add_argument("--db", default=str(DB_PATH), help="Path to systems.db") parser.add_argument("--dry-run", action="store_true", help="Validate without writing") args = parser.parse_args() db_path = Path(args.db) if not db_path.exists(): print(f"error: {db_path} not found", file=sys.stderr) sys.exit(1) print(f"\n Economics Import Pipeline") print(f" DB: {db_path}") if args.dry_run: print(f" Mode: DRY RUN") print() # Load wiki corps before opening DB — allows early exit on parse failures print(" Loading wiki corporations...") wiki_corps = load_wiki_corps() print(f" {len(wiki_corps)} corporation files parsed") conn = sqlite3.connect(str(db_path)) conn.execute("PRAGMA foreign_keys=ON") # 1. Migrate schema print(" [1/8] Schema migration...") for table, col, col_type in COLUMN_MIGRATIONS: _add_column(conn, table, col, col_type) conn.executescript(MIGRATION_SQL) print(" tables and columns ready") # Clear economics tables in FK-safe order (children before parents) # corp_presence cleared here; corporations table is append-only (never cleared) if not args.dry_run: conn.execute("DELETE FROM corp_presence") conn.execute("DELETE FROM chain_inputs") conn.execute("DELETE FROM production_chains") conn.execute("DELETE FROM commodities") conn.execute("DELETE FROM gate_links") # 2. Gate links print(" [2/8] Importing gate links...") n_links = import_gate_links(conn, args.dry_run) print(f" {n_links} rows (bidirectional)") # 3. Commodities print(" [3/8] Importing commodities...") n_commodities = import_commodities(conn, args.dry_run) print(f" {n_commodities} commodities") # 4. Production chains print(" [4/8] Importing production chains...") n_chains, n_inputs = import_chains(conn, args.dry_run) print(f" {n_chains} chains, {n_inputs} inputs") # 5. Currency zones print(" [5/8] Setting currency zones...") zones = set_currency_zones(conn, args.dry_run) for zone, count in sorted(zones.items()): print(f" {zone}: {count}") # 6. Gate energy connectivity (D-186) — must run after currency zones print(" [6/8] Setting gate energy connectivity...") energy = set_gate_energy(conn, args.dry_run) for label, count in sorted(energy.items()): print(f" {label}: {count}") # 7. Sync corporations from wiki (D-182: hard error on name divergence) print(" [7/8] Syncing corporations...") corp_errors = sync_corporations(conn, wiki_corps, args.dry_run) if corp_errors: print(f" ERRORS ({len(corp_errors)}) — name divergence detected (D-182):") for e in corp_errors: print(f" - {e}") print(" Fix: update wiki title or DB proper_name to match, then re-run.") conn.close() sys.exit(1) n_db_corps = conn.execute("SELECT COUNT(*) FROM corporations").fetchone()[0] print(f" {n_db_corps} corporations in DB ({len(wiki_corps)} from wiki)") # 8. Corp presence from wiki headquarters data print(" [8/8] Importing corp presence...") commodity_ids = { r[0] for r in conn.execute("SELECT commodity_id FROM commodities").fetchall() } n_presence = import_corp_presence(conn, wiki_corps, commodity_ids, args.dry_run) print(f" {n_presence} corp_presence rows") # Validate structural integrity (FK, chain refs, chain completeness). # These errors indicate broken imported data — do NOT commit. print("\n Validating structural integrity...") struct_errors = validate(conn) if struct_errors: print(f" STRUCTURAL ERRORS ({len(struct_errors)}) — rolling back:") for e in struct_errors: print(f" - {e}") conn.close() sys.exit(1) else: print(" FK integrity and chain completeness OK") # Commit all imported data (corps, presence, etc.) before coverage check. # Coverage validation is a Phase 2 gate (D-175) — data should be persisted # so tools can query it and report gaps clearly. if not args.dry_run: conn.commit() print(" Data committed.") else: print(" Dry run — no changes written.") # Validate coverage (hard errors per D-175, but after commit so data is usable). print("\n Validating coverage (D-175 Phase 2 gate)...") coverage_errors: list[str] = [] commodity_ids_for_coverage = { r[0] for r in conn.execute("SELECT commodity_id FROM commodities").fetchall() } coverage_errors.extend( _validate_commodity_coverage(conn, wiki_corps, commodity_ids_for_coverage) ) coverage_errors.extend(_validate_system_coverage(conn, wiki_corps)) if coverage_errors: print(f" COVERAGE ERRORS ({len(coverage_errors)}) — Phase 2 gate not met:") for e in coverage_errors: print(f" - {e}") print("\n Data committed but Phase 2 gate is NOT met. " "Add corporations to meet coverage thresholds and re-run.") conn.close() sys.exit(1) else: print(" All coverage thresholds met — Phase 2 gate PASSED.") conn.close() print(f"\n Done: {n_links} gate_links, {n_commodities} commodities, " f"{n_chains} chains, {n_inputs} inputs, {n_presence} corp_presence\n") if __name__ == "__main__": main()