"""Logic for the `ledger` domain — the economics import into systems.db. Transport-agnostic (D-263). Reads TOML/JSON source files and populates the economics tables: - gate_links from docs/design/star-map.json (335 edges, bidirectional) - commodities from wiki/economics/commodities.toml (36 types) - production_chains + chain_inputs from wiki/economics/production_chains.toml - currency_zone on star_systems (default TRACTUS_PRIMARY) - gate_energy_connected on star_systems (D-186: false for MARK_PRIMARY zones) - corporations from wiki/corporations/*.md (sync + insert new records) - corp_specialization/hq_placement/headquarters_body/headquarters_city_id on corporations (D-242: HQ-placement key + baked CityTenant link, from corp_hq_placement.toml) - corp_presence from wiki/corporations/*.md (headquarters location data) - atlas_city_names Standalone-HQ settlement rows (D-242, T-1074) Validation (hard errors, non-zero exit on any failure): - Wiki corporation names must match DB proper_name records (D-182 sync constraint) - Chain completeness: every intermediate commodity has at least one production chain - Commodity coverage: 3+ corporations per major commodity type (D-175) - System coverage: 1+ corporation per inhabited system with population > 100K (D-175) This is the sole generator of `server/data/systems.db` and was `tooling/economy-db/import_economics.py` until T-1289. The steps live in the `economy_import` package; the stamped source set is defined in `tooling/generator_sources.py`. **Exit codes are a contract, kept exactly.** 0 = imported and every coverage threshold met. 2 = imported AND stamped, but the D-175 coverage gate is not met — `make regen-db` and the tooling gate both tolerate it, because the data is usable and the gap is content, not code. 1 = nothing written. **Two error types, deliberately.** `economy_import.errors.ImportAborted` is internal control flow — a step raises it to request rollback after printing what went wrong. It never escapes this module: `run()` rolls back and converts it to a `ReachError` carrying the remedy. That is the explicit reconciliation DOMAINS.md asked for, rather than a second error type with different semantics reaching the caller. """ from __future__ import annotations import sqlite3 from pathlib import Path from tooling.core import console from tooling.core.errors import ReachError from tooling.domains.ledger.economy_import import ( atlas, bodies, brands, corporations, db, economy, migration, specialization, stamp, traits, validators, ) from tooling.domains.ledger.economy_import.errors import ImportAborted from tooling.domains.ledger.economy_import.paths import DB_PATH from tooling.generator_sources import IMPORT_ECONOMICS_SOURCES __all__ = ["DB_PATH", "run"] # Named once, so the progress fraction and the count cannot disagree. STEPS = 24 ABORTED_FIX = ( "fix the errors listed above in their source (wiki/economics/*.toml, " "wiki/corporations/*.md), then re-run: reach ledger import --dry-run" ) class _Steps: """Numbered progress through the import, one event per step.""" def __init__(self) -> None: self.n = 0 def __call__(self, message: str) -> None: self.n += 1 console.event(message, phase=f"{self.n}/{STEPS}", progress=self.n / STEPS) def run(db_path: Path, *, dry_run: bool = False, strict_specialization: bool = False) -> None: """Import every economics source into `db_path`, in one transaction. Raises ReachError with exit code 1 when nothing was written, and with exit code 2 when the import committed but the coverage gate is not met. """ if not db_path.exists(): raise ReachError( f"{db_path} not found", fix="pass --db with an existing systems.db, or restore it: git restore server/data/systems.db", ) console.event(f"Economics import into {db_path}" + (" — DRY RUN" if dry_run else "")) # Load wiki corps before opening the DB — allows early exit on parse failures. console.event("Loading wiki corporations...") wiki_corps = corporations.load_wiki_corps() console.event(f"{len(wiki_corps)} corporation files parsed") # Regenerate generated_brands.toml via the Rust binary before the import # reads it — single pipeline, single stamp (review T2/H3). Skipped on a # dry run to avoid a disk side effect during validation. if not dry_run: brands.regenerate_brands() conn = db.connect(db_path) try: counts = _import(conn, wiki_corps, dry_run, strict_specialization) except ImportAborted: conn.rollback() conn.close() raise ReachError( "economics import aborted — rolled back, nothing written", fix=ABORTED_FIX ) from None except BaseException: # Anything else (KeyboardInterrupt, MemoryError, a DB error, a bug) # rolls back too, so the DB is never left half-imported. Re-raised so # the traceback survives. conn.rollback() conn.close() raise # Stamp generator metadata (#855, #856) so the pre-push hook can detect a # stale snapshot. Written BEFORE the coverage gate: the stamp records which # code produced the DB, not whether the data is complete (#860). if not dry_run: try: stamp.write_stamp(conn, "import_economics", *IMPORT_ECONOMICS_SOURCES) conn.commit() console.event("Stamped: import_economics (covers brand pipeline Rust sources)") except Exception as exc: # noqa: BLE001 console.event(f"failed to write generator stamp: {exc}", level="warn") coverage_errors = _coverage(conn, wiki_corps) conn.close() summary = ( f"{counts['links']} gate_links, {counts['commodities']} commodities, " f"{counts['chains']} chains, {counts['inputs']} inputs, " f"{counts['presence']} corp_presence, {counts['brands']} brand_products, " f"{counts['brand_inputs']} brand_inputs, {counts['fiscal']} system_fiscal" ) if coverage_errors: for e in coverage_errors: console.event(e, level="error") raise ReachError( f"imported ({summary}) but the D-175 Phase 2 coverage gate is not met — " f"{len(coverage_errors)} gap(s) above. " + ("Nothing was written (dry run)." if dry_run else "Data and stamp are committed."), fix="add corporations to wiki/corporations/ until the thresholds are met, then re-run", exit_code=2, ) written = "validated, nothing written (dry run)" if dry_run else "imported and stamped" console.verdict(f"ledger import: {written} — {summary}") def _import( conn: sqlite3.Connection, wiki_corps, dry_run: bool, strict_specialization: bool ) -> dict[str, int]: """The clear-then-reimport cycle, as one explicit transaction. Any crash, validation error or interrupt between the first DELETE and the commit rolls everything back (the caller owns that): the DB never ends up half-cleared. On success it commits exactly once, right after structural validation passes. On a dry run the transaction is left open so the coverage check can still SELECT the imported rows; closing discards it. """ step = _Steps() conn.execute("BEGIN") # Schema migration — idempotent, inside the tx so a crash leaves no # half-applied ALTER TABLE. step("Schema migration...") migration.apply_schema_migrations(conn) # Atlas index tables: canonical DDL + empty geometry (D-223, #951). atlas.ensure_atlas_index_schema(conn, dry_run) # Clear economics tables in FK-safe order (children before parents). if not dry_run: db.clear_economics_tables(conn) step("Importing gate links...") n_links = economy.import_gate_links(conn, dry_run) console.event(f"{n_links} rows (bidirectional)") step("Importing commodities...") n_commodities = economy.import_commodities(conn, dry_run) console.event(f"{n_commodities} commodities") step("Importing production chains...") n_chains, n_inputs = economy.import_chains(conn, dry_run) console.event(f"{n_chains} chains, {n_inputs} inputs") # D-237 authored layer — after commodities (FK), before currency zones. # UPSERTs onto pre-existing system_economy / system_factions rows; # unauthored systems stay NULL. step("Importing system specialization (D-237)...") spec = specialization.import_system_specialization(conn, dry_run, strict=strict_specialization) console.event( f"vocab {spec['vocab']} | economic {spec['economic']} | " f"cultural {spec['cultural']} | faction {spec['faction']}" ) if spec["missing_economy_row"]: console.event( f"{len(spec['missing_economy_row'])} authored system(s) lack a " f"system_economy row (values dropped): {spec['missing_economy_row']}", level="warn", ) if spec["missing_faction_row"]: console.event( f"{len(spec['missing_faction_row'])} authored system(s) lack a " f"system_factions row (faction dropped): {spec['missing_faction_row']}", level="warn", ) step("Setting currency zones...") for zone, count in sorted(economy.set_currency_zones(conn, dry_run).items()): console.event(f"{zone}: {count}") # D-186 — must run after currency zones. step("Setting gate energy connectivity...") for label, count in sorted(economy.set_gate_energy(conn, dry_run).items()): console.event(f"{label}: {count}") # D-182: a name divergence is a hard error. step("Syncing corporations...") corp_errors = corporations.sync_corporations(conn, wiki_corps, dry_run) if corp_errors: console.event(f"name divergence detected (D-182) — {len(corp_errors)} error(s):", level="error") for e in corp_errors: console.event(e, level="error") console.event("update the wiki title or the DB proper_name to match, then re-run", level="error") raise ImportAborted() n_db_corps = conn.execute("SELECT COUNT(*) FROM corporations").fetchone()[0] console.event(f"{n_db_corps} corporations in DB ({len(wiki_corps)} from wiki)") # D-242 Phase A — reset + derive every run (PR #177 H1/T2). After corp # sync (rows must exist), before corp_presence (reads headquarters_body # back), and before the city-presence tiebreak further down, which reads # atlas_city_names BEFORE this run's rebuild — the previous run's settled # state, by design. step("Importing corp specialization + HQ placement (D-242)...") corp_spec = corporations.populate_corp_specialization(conn, wiki_corps, dry_run) console.event( f"{corp_spec['specialized']}/{corp_spec['total']} corps specialized, " f"{corp_spec['hq_resolved']} headquarters_body resolved (recomputed every run; " f"{corp_spec['hq_overridden']} authored overrides)" ) step("Importing corp presence...") commodity_ids = {r[0] for r in conn.execute("SELECT commodity_id FROM commodities").fetchall()} n_presence = corporations.import_corp_presence(conn, wiki_corps, commodity_ids, dry_run) console.event(f"{n_presence} corp_presence rows") # D-189, #827. step("Importing brand products and inputs...") n_brands, n_brand_inputs = brands.import_brands(conn, dry_run) console.event(f"{n_brands} brand_products, {n_brand_inputs} brand_inputs") # D-189 §6. step("Populating system_fiscal...") n_fiscal = economy.import_system_fiscal(conn, dry_run) console.event(f"{n_fiscal} system_fiscal rows") # D-204, #910. step("Populating body_radius_km fallback...") console.event(f"{bodies.populate_body_radius_km(conn, dry_run)} bodies updated") # D-207, #908. step("Populating atlas_city_names from wiki content...") console.event(f"{atlas.populate_atlas_city_names(conn, dry_run)} city name rows") # D-223, T-1169 — mirrors the city pool's shape over a different # names-only markers.json key set; independent of the settlement pool. step("Populating atlas_feature_names from wiki content...") console.event(f"{atlas.populate_atlas_feature_names(conn, dry_run)} feature name rows") # D-242 Phase B (T-1074). SUPERSEDES the retired corp-HQ cross-reference # (D-207, #909), which inserted one city row per corp HQ with no # UNIQUE(body_id, name) — ten co-named Groombridge rows on GJ380c. Must # follow the city pool and the corp-specialization step. step("Emitting Standalone-HQ settlements + CityTenant links (D-242)...") hq = corporations.populate_standalone_hq_settlements(conn, dry_run) console.event( f"{hq['standalone_inserted']} Standalone-HQ settlements, " f"{hq['tenant_linked']} CityTenant links ({hq['tenant_unmatched']} unmatched)" ) # D-242, T-1075 — over the CORRECTED pool, so Standalone-HQ settlements # are part of the rank-size spread rather than bolted on after. step("Baking settlement population + settlement_class (D-242)...") pop = atlas.populate_settlement_population_class(conn, dry_run) console.event( f"{pop['cities_populated']} cities populated across {pop['bodies_spread']} bodies, " f"{pop['name_locked_applied']} NameLocked pins applied" ) # D-232, #993 — catalog first, then sparse per-body hero bias (FK). step("Baking trait_templates catalog (D-232)...") console.event(f"{traits.populate_trait_templates(conn, dry_run)} trait templates") step("Baking atlas_body_trait_bias hero pins (D-232)...") console.event(f"{traits.populate_atlas_body_trait_bias(conn, dry_run)} body trait-bias rows") # T-1024, D-239 §2. step("Populating axial_tilt_deg from body-def frontmatter...") console.event(f"{bodies.populate_axial_tilt_deg(conn, dry_run)} bodies updated with axial_tilt_deg") # D-247, T-1085 — frontmatter override + two-gate default. step("Populating biosphere_class (D-247)...") console.event(f"{bodies.populate_biosphere_class(conn, dry_run)} bodies updated with biosphere_class") # T-988, D-235 — both follow trait_templates (V-TT-06 / V-TT-07 validate # against it). step("Baking architecture_zone_bias table (D-235)...") console.event(f"{traits.populate_architecture_zone_bias(conn, dry_run)} zone-bias rows") step("Baking color_register_bands table (D-235)...") console.event(f"{traits.populate_color_register_bands(conn, dry_run)} color register bands") # Structural integrity (FK, chain refs, chain completeness). These mean # the imported data is broken — do NOT commit. step("Validating structural integrity...") struct_errors = validators.validate(conn) + brands.validate_brands(conn) if struct_errors: console.event(f"STRUCTURAL ERRORS ({len(struct_errors)}) — rolling back:", level="error") for e in struct_errors: console.event(e, level="error") raise ImportAborted() console.event("FK integrity, chain completeness, and brand layer (V-B01–V-B06) OK") # Commit before the coverage check: coverage is a Phase 2 gate (D-175), # and the data should be queryable so the gaps can be reported clearly. if not dry_run: conn.commit() console.event("Data committed.") else: console.event("Dry run — no changes written.") if step.n != STEPS: # A step added without bumping STEPS would make every progress # fraction wrong without failing anything. Cheap to refuse here. raise AssertionError(f"STEPS is {STEPS} but the import ran {step.n} steps") return { "links": n_links, "commodities": n_commodities, "chains": n_chains, "inputs": n_inputs, "presence": n_presence, "brands": n_brands, "brand_inputs": n_brand_inputs, "fiscal": n_fiscal, } def _coverage(conn: sqlite3.Connection, wiki_corps) -> list[str]: """The D-175 Phase 2 gate. Runs after the commit, so the data is usable.""" console.event("Validating coverage (D-175 Phase 2 gate)...") commodity_ids = {r[0] for r in conn.execute("SELECT commodity_id FROM commodities").fetchall()} errors = validators.validate_commodity_coverage(conn, wiki_corps, commodity_ids) errors += validators.validate_system_coverage(conn, wiki_corps) if not errors: console.event("All coverage thresholds met — Phase 2 gate PASSED.") return errors