settled-reach/tooling/domains/ledger/service.py

"""Logic for the `ledger` domain — the economics import into systems.db. Transport-agnostic (D-263).

Reads TOML/JSON source files and populates the economics tables:
  - gate_links               from docs/design/star-map.json (335 edges, bidirectional)
  - commodities              from wiki/economics/commodities.toml (36 types)
  - production_chains + chain_inputs from wiki/economics/production_chains.toml
  - currency_zone            on star_systems (default TRACTUS_PRIMARY)
  - gate_energy_connected    on star_systems (D-186: false for MARK_PRIMARY zones)
  - corporations             from wiki/corporations/*.md (sync + insert new records)
  - corp_specialization/hq_placement/headquarters_body/headquarters_city_id
                             on corporations (D-242: HQ-placement key + baked
                             CityTenant link, from corp_hq_placement.toml)
  - corp_presence            from wiki/corporations/*.md (headquarters location data)
  - atlas_city_names         Standalone-HQ settlement rows (D-242, T-1074)

Validation (hard errors, non-zero exit on any failure):
  - Wiki corporation names must match DB proper_name records (D-182 sync constraint)
  - Chain completeness: every intermediate commodity has at least one production chain
  - Commodity coverage: 3+ corporations per major commodity type (D-175)
  - System coverage: 1+ corporation per inhabited system with population > 100K (D-175)

This is the sole generator of `server/data/systems.db` and was
`tooling/economy-db/import_economics.py` until T-1289. The steps live in the
`economy_import` package; the stamped source set is defined in
`tooling/generator_sources.py`.

**Exit codes are a contract, kept exactly.** 0 = imported and every coverage
threshold met. 2 = imported AND stamped, but the D-175 coverage gate is not met
— `make regen-db` and the tooling gate both tolerate it, because the data is
usable and the gap is content, not code. 1 = nothing written.

**Two error types, deliberately.** `economy_import.errors.ImportAborted` is
internal control flow — a step raises it to request rollback after printing
what went wrong. It never escapes this module: `run()` rolls back and converts
it to a `ReachError` carrying the remedy. That is the explicit reconciliation
DOMAINS.md asked for, rather than a second error type with different semantics
reaching the caller.
"""

from __future__ import annotations

import sqlite3
from pathlib import Path

from tooling.core import console
from tooling.core.errors import ReachError
from tooling.domains.ledger.economy_import import (
    atlas,
    bodies,
    brands,
    corporations,
    db,
    economy,
    migration,
    specialization,
    stamp,
    traits,
    validators,
)
from tooling.domains.ledger.economy_import.errors import ImportAborted
from tooling.domains.ledger.economy_import.paths import DB_PATH
from tooling.generator_sources import IMPORT_ECONOMICS_SOURCES

__all__ = ["DB_PATH", "run"]

# Named once, so the progress fraction and the count cannot disagree.
STEPS = 24

ABORTED_FIX = (
    "fix the errors listed above in their source (wiki/economics/*.toml, "
    "wiki/corporations/*.md), then re-run: reach ledger import --dry-run"
)


class _Steps:
    """Numbered progress through the import, one event per step."""

    def __init__(self) -> None:
        self.n = 0

    def __call__(self, message: str) -> None:
        self.n += 1
        console.event(message, phase=f"{self.n}/{STEPS}", progress=self.n / STEPS)


def run(db_path: Path, *, dry_run: bool = False, strict_specialization: bool = False) -> None:
    """Import every economics source into `db_path`, in one transaction.

    Raises ReachError with exit code 1 when nothing was written, and with exit
    code 2 when the import committed but the coverage gate is not met.
    """
    if not db_path.exists():
        raise ReachError(
            f"{db_path} not found",
            fix="pass --db with an existing systems.db, or restore it: git restore server/data/systems.db",
        )

    console.event(f"Economics import into {db_path}" + (" — DRY RUN" if dry_run else ""))

    # Load wiki corps before opening the DB — allows early exit on parse failures.
    console.event("Loading wiki corporations...")
    wiki_corps = corporations.load_wiki_corps()
    console.event(f"{len(wiki_corps)} corporation files parsed")

    # Regenerate generated_brands.toml via the Rust binary before the import
    # reads it — single pipeline, single stamp (review T2/H3). Skipped on a
    # dry run to avoid a disk side effect during validation.
    if not dry_run:
        brands.regenerate_brands()

    conn = db.connect(db_path)
    try:
        counts = _import(conn, wiki_corps, dry_run, strict_specialization)
    except ImportAborted:
        conn.rollback()
        conn.close()
        raise ReachError(
            "economics import aborted — rolled back, nothing written", fix=ABORTED_FIX
        ) from None
    except BaseException:
        # Anything else (KeyboardInterrupt, MemoryError, a DB error, a bug)
        # rolls back too, so the DB is never left half-imported. Re-raised so
        # the traceback survives.
        conn.rollback()
        conn.close()
        raise

    # Stamp generator metadata (#855, #856) so the pre-push hook can detect a
    # stale snapshot. Written BEFORE the coverage gate: the stamp records which
    # code produced the DB, not whether the data is complete (#860).
    if not dry_run:
        try:
            stamp.write_stamp(conn, "import_economics", *IMPORT_ECONOMICS_SOURCES)
            conn.commit()
            console.event("Stamped: import_economics (covers brand pipeline Rust sources)")
        except Exception as exc:  # noqa: BLE001
            console.event(f"failed to write generator stamp: {exc}", level="warn")

    coverage_errors = _coverage(conn, wiki_corps)
    conn.close()

    summary = (
        f"{counts['links']} gate_links, {counts['commodities']} commodities, "
        f"{counts['chains']} chains, {counts['inputs']} inputs, "
        f"{counts['presence']} corp_presence, {counts['brands']} brand_products, "
        f"{counts['brand_inputs']} brand_inputs, {counts['fiscal']} system_fiscal"
    )
    if coverage_errors:
        for e in coverage_errors:
            console.event(e, level="error")
        raise ReachError(
            f"imported ({summary}) but the D-175 Phase 2 coverage gate is not met — "
            f"{len(coverage_errors)} gap(s) above. "
            + ("Nothing was written (dry run)." if dry_run else "Data and stamp are committed."),
            fix="add corporations to wiki/corporations/ until the thresholds are met, then re-run",
            exit_code=2,
        )

    written = "validated, nothing written (dry run)" if dry_run else "imported and stamped"
    console.verdict(f"ledger import: {written} — {summary}")


def _import(
    conn: sqlite3.Connection, wiki_corps, dry_run: bool, strict_specialization: bool
) -> dict[str, int]:
    """The clear-then-reimport cycle, as one explicit transaction.

    Any crash, validation error or interrupt between the first DELETE and the
    commit rolls everything back (the caller owns that): the DB never ends up
    half-cleared. On success it commits exactly once, right after structural
    validation passes. On a dry run the transaction is left open so the
    coverage check can still SELECT the imported rows; closing discards it.
    """
    step = _Steps()
    conn.execute("BEGIN")

    # Schema migration — idempotent, inside the tx so a crash leaves no
    # half-applied ALTER TABLE.
    step("Schema migration...")
    migration.apply_schema_migrations(conn)
    # Atlas index tables: canonical DDL + empty geometry (D-223, #951).
    atlas.ensure_atlas_index_schema(conn, dry_run)

    # Clear economics tables in FK-safe order (children before parents).
    if not dry_run:
        db.clear_economics_tables(conn)

    step("Importing gate links...")
    n_links = economy.import_gate_links(conn, dry_run)
    console.event(f"{n_links} rows (bidirectional)")

    step("Importing commodities...")
    n_commodities = economy.import_commodities(conn, dry_run)
    console.event(f"{n_commodities} commodities")

    step("Importing production chains...")
    n_chains, n_inputs = economy.import_chains(conn, dry_run)
    console.event(f"{n_chains} chains, {n_inputs} inputs")

    # D-237 authored layer — after commodities (FK), before currency zones.
    # UPSERTs onto pre-existing system_economy / system_factions rows;
    # unauthored systems stay NULL.
    step("Importing system specialization (D-237)...")
    spec = specialization.import_system_specialization(conn, dry_run, strict=strict_specialization)
    console.event(
        f"vocab {spec['vocab']} | economic {spec['economic']} | "
        f"cultural {spec['cultural']} | faction {spec['faction']}"
    )
    if spec["missing_economy_row"]:
        console.event(
            f"{len(spec['missing_economy_row'])} authored system(s) lack a "
            f"system_economy row (values dropped): {spec['missing_economy_row']}",
            level="warn",
        )
    if spec["missing_faction_row"]:
        console.event(
            f"{len(spec['missing_faction_row'])} authored system(s) lack a "
            f"system_factions row (faction dropped): {spec['missing_faction_row']}",
            level="warn",
        )

    step("Setting currency zones...")
    for zone, count in sorted(economy.set_currency_zones(conn, dry_run).items()):
        console.event(f"{zone}: {count}")

    # D-186 — must run after currency zones.
    step("Setting gate energy connectivity...")
    for label, count in sorted(economy.set_gate_energy(conn, dry_run).items()):
        console.event(f"{label}: {count}")

    # D-182: a name divergence is a hard error.
    step("Syncing corporations...")
    corp_errors = corporations.sync_corporations(conn, wiki_corps, dry_run)
    if corp_errors:
        console.event(f"name divergence detected (D-182) — {len(corp_errors)} error(s):", level="error")
        for e in corp_errors:
            console.event(e, level="error")
        console.event("update the wiki title or the DB proper_name to match, then re-run", level="error")
        raise ImportAborted()
    n_db_corps = conn.execute("SELECT COUNT(*) FROM corporations").fetchone()[0]
    console.event(f"{n_db_corps} corporations in DB ({len(wiki_corps)} from wiki)")

    # D-242 Phase A — reset + derive every run (PR #177 H1/T2). After corp
    # sync (rows must exist), before corp_presence (reads headquarters_body
    # back), and before the city-presence tiebreak further down, which reads
    # atlas_city_names BEFORE this run's rebuild — the previous run's settled
    # state, by design.
    step("Importing corp specialization + HQ placement (D-242)...")
    corp_spec = corporations.populate_corp_specialization(conn, wiki_corps, dry_run)
    console.event(
        f"{corp_spec['specialized']}/{corp_spec['total']} corps specialized, "
        f"{corp_spec['hq_resolved']} headquarters_body resolved (recomputed every run; "
        f"{corp_spec['hq_overridden']} authored overrides)"
    )

    step("Importing corp presence...")
    commodity_ids = {r[0] for r in conn.execute("SELECT commodity_id FROM commodities").fetchall()}
    n_presence = corporations.import_corp_presence(conn, wiki_corps, commodity_ids, dry_run)
    console.event(f"{n_presence} corp_presence rows")

    # D-189, #827.
    step("Importing brand products and inputs...")
    n_brands, n_brand_inputs = brands.import_brands(conn, dry_run)
    console.event(f"{n_brands} brand_products, {n_brand_inputs} brand_inputs")

    # D-189 §6.
    step("Populating system_fiscal...")
    n_fiscal = economy.import_system_fiscal(conn, dry_run)
    console.event(f"{n_fiscal} system_fiscal rows")

    # D-204, #910.
    step("Populating body_radius_km fallback...")
    console.event(f"{bodies.populate_body_radius_km(conn, dry_run)} bodies updated")

    # D-207, #908.
    step("Populating atlas_city_names from wiki content...")
    console.event(f"{atlas.populate_atlas_city_names(conn, dry_run)} city name rows")

    # D-223, T-1169 — mirrors the city pool's shape over a different
    # names-only markers.json key set; independent of the settlement pool.
    step("Populating atlas_feature_names from wiki content...")
    console.event(f"{atlas.populate_atlas_feature_names(conn, dry_run)} feature name rows")

    # D-242 Phase B (T-1074). SUPERSEDES the retired corp-HQ cross-reference
    # (D-207, #909), which inserted one city row per corp HQ with no
    # UNIQUE(body_id, name) — ten co-named Groombridge rows on GJ380c. Must
    # follow the city pool and the corp-specialization step.
    step("Emitting Standalone-HQ settlements + CityTenant links (D-242)...")
    hq = corporations.populate_standalone_hq_settlements(conn, dry_run)
    console.event(
        f"{hq['standalone_inserted']} Standalone-HQ settlements, "
        f"{hq['tenant_linked']} CityTenant links ({hq['tenant_unmatched']} unmatched)"
    )

    # D-242, T-1075 — over the CORRECTED pool, so Standalone-HQ settlements
    # are part of the rank-size spread rather than bolted on after.
    step("Baking settlement population + settlement_class (D-242)...")
    pop = atlas.populate_settlement_population_class(conn, dry_run)
    console.event(
        f"{pop['cities_populated']} cities populated across {pop['bodies_spread']} bodies, "
        f"{pop['name_locked_applied']} NameLocked pins applied"
    )

    # D-232, #993 — catalog first, then sparse per-body hero bias (FK).
    step("Baking trait_templates catalog (D-232)...")
    console.event(f"{traits.populate_trait_templates(conn, dry_run)} trait templates")
    step("Baking atlas_body_trait_bias hero pins (D-232)...")
    console.event(f"{traits.populate_atlas_body_trait_bias(conn, dry_run)} body trait-bias rows")

    # T-1024, D-239 §2.
    step("Populating axial_tilt_deg from body-def frontmatter...")
    console.event(f"{bodies.populate_axial_tilt_deg(conn, dry_run)} bodies updated with axial_tilt_deg")

    # D-247, T-1085 — frontmatter override + two-gate default.
    step("Populating biosphere_class (D-247)...")
    console.event(f"{bodies.populate_biosphere_class(conn, dry_run)} bodies updated with biosphere_class")

    # T-988, D-235 — both follow trait_templates (V-TT-06 / V-TT-07 validate
    # against it).
    step("Baking architecture_zone_bias table (D-235)...")
    console.event(f"{traits.populate_architecture_zone_bias(conn, dry_run)} zone-bias rows")
    step("Baking color_register_bands table (D-235)...")
    console.event(f"{traits.populate_color_register_bands(conn, dry_run)} color register bands")

    # Structural integrity (FK, chain refs, chain completeness). These mean
    # the imported data is broken — do NOT commit.
    step("Validating structural integrity...")
    struct_errors = validators.validate(conn) + brands.validate_brands(conn)
    if struct_errors:
        console.event(f"STRUCTURAL ERRORS ({len(struct_errors)}) — rolling back:", level="error")
        for e in struct_errors:
            console.event(e, level="error")
        raise ImportAborted()
    console.event("FK integrity, chain completeness, and brand layer (V-B01–V-B06) OK")

    # Commit before the coverage check: coverage is a Phase 2 gate (D-175),
    # and the data should be queryable so the gaps can be reported clearly.
    if not dry_run:
        conn.commit()
        console.event("Data committed.")
    else:
        console.event("Dry run — no changes written.")

    if step.n != STEPS:
        # A step added without bumping STEPS would make every progress
        # fraction wrong without failing anything. Cheap to refuse here.
        raise AssertionError(f"STEPS is {STEPS} but the import ran {step.n} steps")

    return {
        "links": n_links,
        "commodities": n_commodities,
        "chains": n_chains,
        "inputs": n_inputs,
        "presence": n_presence,
        "brands": n_brands,
        "brand_inputs": n_brand_inputs,
        "fiscal": n_fiscal,
    }


def _coverage(conn: sqlite3.Connection, wiki_corps) -> list[str]:
    """The D-175 Phase 2 gate. Runs after the commit, so the data is usable."""
    console.event("Validating coverage (D-175 Phase 2 gate)...")
    commodity_ids = {r[0] for r in conn.execute("SELECT commodity_id FROM commodities").fetchall()}
    errors = validators.validate_commodity_coverage(conn, wiki_corps, commodity_ids)
    errors += validators.validate_system_coverage(conn, wiki_corps)
    if not errors:
        console.event("All coverage thresholds met — Phase 2 gate PASSED.")
    return errors