settled-reach/tooling/economy-db/economy_import/atlas.py

"""Atlas index ownership (D-223, #951): canonical atlas_* schema and the
names-only city pool.

The corp-HQ city cross-reference (D-207, `populate_atlas_city_names_corps`)
was REMOVED here (D-242, T-1074) — it inserted one atlas_city_names row per
corp HQ with no UNIQUE(body_id, name), producing duplicate co-named "cities"
(e.g. 10 Groombridge rows on GJ380c). The corp<->settlement relationship now
lives with the corp (economy_import/corporations.py:populate_corp_specialization),
not in the city pool. `most_populated_body_in_system` below is the one piece
of that removed function's logic still needed (by the D-242 replacement).

`populate_settlement_population_class` (D-242, T-1075) bakes the per-settlement
population spread + settlement_class onto atlas_city_names — see its own
docstring for the rank-size derivation and D-196 class-assignment rule."""

import glob
import json
import sqlite3
import tomllib

from .paths import SCHEMA_SQL, SETTLEMENT_NAME_LOCKED_TOML, WIKI_STAR_SYSTEMS

# Atlas geometry index tables (D-191). These hold computed positions — city
# centres, road/river/rail polylines, ocean/mountain extents. Under D-223 the
# Python atlas geometry generator was retired (#951); the deterministic
# server-side cascade (Phase 4) is the sole producer of this geometry. We keep
# the tables (the Atlas viewer #960 and the cascade read them) but empty them on
# every regen so the committed DB carries no stale prototype geometry — the
# empty tables are the gap the server cascade fills.
_ATLAS_GEOMETRY_TABLES: tuple[str, ...] = (
    "atlas_cities",
    "atlas_roads",
    "atlas_railroads",
    "atlas_pois",
    "atlas_rivers",
    "atlas_oceans",
    "atlas_mountain_ranges",
    "atlas_body_grids",
)

_ATLAS_INDEX_BEGIN_MARKER = "-- BEGIN ATLAS INDEX"
_ATLAS_INDEX_END_MARKER = "-- END ATLAS INDEX"


def ensure_atlas_index_schema(conn: sqlite3.Connection, dry_run: bool) -> None:
    """Apply the canonical atlas_* DDL and empty the geometry tables (D-223, #951).

    systems-schema.sql is the single source of truth for the atlas index tables
    (the BEGIN/END ATLAS INDEX block). The retired generate_atlas.py used to
    apply this block; import_economics now owns it, since it is the only
    regen-db generator that touches systems.db's atlas tables. The block is all
    CREATE ... IF NOT EXISTS, so applying it on the committed DB is a no-op and
    on a fresh DB it creates the geometry tables.

    After ensuring the schema, the geometry tables are cleared: their geometry
    now comes from the server cascade, not from authored markers (D-223).
    """
    text = SCHEMA_SQL.read_text()
    try:
        start = text.index(_ATLAS_INDEX_BEGIN_MARKER)
        end = text.index(_ATLAS_INDEX_END_MARKER, start)
    except ValueError as e:
        raise RuntimeError(
            f"systems-schema.sql is missing the {_ATLAS_INDEX_BEGIN_MARKER}/"
            f"{_ATLAS_INDEX_END_MARKER} block — has the schema been restructured?"
        ) from e
    conn.executescript(text[start:end])
    if not dry_run:
        for table in _ATLAS_GEOMETRY_TABLES:
            conn.execute(f"DELETE FROM {table}")


def populate_atlas_city_names(conn: sqlite3.Connection, dry_run: bool) -> int:
    """Populate atlas_city_names from the names-only markers.json pool (D-223, #951).

    Scans wiki/star-systems/*/bodies/*/markers.json for the flavoured city
    name pool at `names.cities` and inserts one atlas_city_names row per name:
      - body_id       : directory name (e.g. GJ0e)
      - name          : pooled city name
      - kind          : 'city' — capital is chosen at placement (#955)
      - economic_role : inherited from bodies.economic_role; fallback 'mixed'
      - population    : 0 — baked at import by populate_settlement_population_class (D-242, T-1075)
      - corp_id       : NULL — SUPERSEDED (D-242): the corp-HQ cross-reference
                        that used to populate this was removed; always NULL now
      - reserved      : 0

    markers.json is a names-only flavoured pool (D-223): it carries no geometry
    or population. The deterministic server cascade attaches these names to
    computed settlements and assigns population/kind/position at placement time;
    this importer just loads the pool.

    Deterministic rebuild: clears atlas_city_names first (the FK cascade clears
    atlas_city_positions), so re-runs are idempotent — there is no UNIQUE on
    (body_id, name), so without the clear a re-run would accumulate duplicates.
    Skips body directories not found in the bodies table (missing FK).
    """
    # Build body_id -> economic_role map
    body_roles: dict[str, str] = {}
    for body_id, role in conn.execute(
        "SELECT body_id, economic_role FROM bodies"
    ).fetchall():
        body_roles[body_id] = role or "mixed"

    valid_body_ids: set[str] = set(body_roles.keys())

    # Sol (system 'GJ 0') is permanently exempt from the normal generators
    # (D-223, #951): its bodies use real Earth/Mars/Luna geography via
    # sol_import.py and keep geometry-bearing markers.json as preserved config.
    # Sol names come from its own (future) scripted integration, not the names
    # pool — skip Sol bodies here regardless of their markers format.
    sol_body_ids: set[str] = {
        r[0] for r in conn.execute(
            "SELECT body_id FROM bodies WHERE system_id = 'GJ 0'"
        ).fetchall()
    }

    rows: list[tuple] = []
    skipped_bodies: list[str] = []

    pattern = str(WIKI_STAR_SYSTEMS / "*" / "bodies" / "*" / "markers.json")
    for markers_path in sorted(glob.glob(pattern)):
        body_id = markers_path.split("/bodies/")[1].split("/")[0]
        if body_id not in valid_body_ids or body_id in sol_body_ids:
            if body_id not in valid_body_ids:
                skipped_bodies.append(body_id)
            continue

        with open(markers_path) as fh:
            data = json.load(fh)

        names_pool = (data.get("names") or {}).get("cities") or []
        economic_role = body_roles[body_id]
        for raw_name in names_pool:
            name = (raw_name or "").strip()
            if not name:
                continue
            # kind defaults to 'city'; population 0 until placement (#955).
            rows.append((body_id, name, "city", economic_role, 0))

    if skipped_bodies:
        unique = sorted(set(skipped_bodies))
        print(f"        warning: {len(unique)} body dirs not in DB — skipped: {unique[:5]}")

    if not dry_run:
        # D-242: corporations.headquarters_city_id FK-references
        # atlas_city_names(id). corporations is append-only (never cleared —
        # db.clear_economics_tables doesn't touch it), so a PRIOR run's
        # resolved CityTenant links survive as live FK references into the
        # very rows this DELETE is about to remove — clear them first or the
        # DELETE hits a FOREIGN KEY constraint failure on every run after the
        # first (caught live in the T-1074/T-1075 regen-db integration test).
        # Re-populated by corporations.populate_standalone_hq_settlements
        # later in this same import run.
        conn.execute("UPDATE corporations SET headquarters_city_id = NULL")
        conn.execute("DELETE FROM atlas_city_names")
        if rows:
            conn.executemany(
                """INSERT INTO atlas_city_names
                   (body_id, name, kind, economic_role, population)
                   VALUES (?, ?, ?, ?, ?)""",
                rows,
            )

    return len(rows)


# markers.json `names` key -> atlas_feature_names.feature_type value. Only
# rivers/mountain_ranges are populated here (T-1169 scope — the two pools
# server/src/atlas/layer1.rs's `attach_feature_names` actually consumes,
# river_names/mountain_names). `oceans`/`passes`/`regions` pools are NOT
# wired to any production consumer yet (no `attach_feature_names`-equivalent
# reads them) — populating rows for them here would be dead data with
# nothing to keep it honest, the same D-223 discipline that keeps
# atlas_city_names to names an actual placement pipeline consumes.
_FEATURE_NAME_POOL_KEYS: dict[str, str] = {
    "rivers": "river",
    "mountain_ranges": "mountain",
}


def populate_atlas_feature_names(conn: sqlite3.Connection, dry_run: bool) -> int:
    """Populate atlas_feature_names from the names-only markers.json pools
    (D-223, T-1169) — mirrors `populate_atlas_city_names` exactly, over the
    `rivers`/`mountain_ranges` marker keys instead of `cities`:
      - body_id      : directory name (e.g. GJ0e)
      - name         : pooled feature name
      - feature_type : 'river' | 'mountain' (see `_FEATURE_NAME_POOL_KEYS`)
      - priority     : 0 — no authored ranking signal in markers.json; the
                       server-side name attachment (`layer1::attach_feature_names`)
                       ranks by computed mouth/peak strength at assignment time,
                       not by a priority stored on the pool row.

    markers.json is a names-only flavoured pool (D-223): it carries no
    geometry or position. `attach_feature_names` (Phase 4 cascade) attaches
    these names to computed river mouths / alpine peaks at generation time;
    this importer just loads the pool, exactly as `populate_atlas_city_names`
    loads the city pool for the settlement placement pipeline.

    Deterministic rebuild: clears atlas_feature_names first, so re-runs are
    idempotent — there is no UNIQUE(body_id, name, feature_type), so without
    the clear a re-run would accumulate duplicates. Skips body directories not
    found in the bodies table (missing FK), and Sol bodies (D-223 permanent
    exemption — same as `populate_atlas_city_names`, Sol keeps authored
    geometry-bearing markers.json via sol_import.py, not the names pool).
    """
    valid_body_ids: set[str] = {
        r[0] for r in conn.execute("SELECT body_id FROM bodies").fetchall()
    }
    sol_body_ids: set[str] = {
        r[0] for r in conn.execute(
            "SELECT body_id FROM bodies WHERE system_id = 'GJ 0'"
        ).fetchall()
    }

    rows: list[tuple] = []
    skipped_bodies: list[str] = []

    pattern = str(WIKI_STAR_SYSTEMS / "*" / "bodies" / "*" / "markers.json")
    for markers_path in sorted(glob.glob(pattern)):
        body_id = markers_path.split("/bodies/")[1].split("/")[0]
        if body_id not in valid_body_ids or body_id in sol_body_ids:
            if body_id not in valid_body_ids:
                skipped_bodies.append(body_id)
            continue

        with open(markers_path) as fh:
            data = json.load(fh)

        names = data.get("names") or {}
        for pool_key, feature_type in _FEATURE_NAME_POOL_KEYS.items():
            for raw_name in names.get(pool_key) or []:
                name = (raw_name or "").strip()
                if not name:
                    continue
                rows.append((body_id, name, feature_type, 0))

    if skipped_bodies:
        unique = sorted(set(skipped_bodies))
        print(f"        warning: {len(unique)} body dirs not in DB — skipped: {unique[:5]}")

    if not dry_run:
        conn.execute("DELETE FROM atlas_feature_names")
        if rows:
            conn.executemany(
                """INSERT INTO atlas_feature_names
                   (body_id, name, feature_type, priority)
                   VALUES (?, ?, ?, ?)""",
                rows,
            )

    return len(rows)


# body_type rank for the most_populated_body_in_system LAST-RESORT tiebreak —
# lower sorts first. This is NOT "belts can't host settlements" (they can and
# DO in this world: GJ845-belt hosts Orkney Ceramics, GJ268-belt hosts Jeju
# Lattice, GJ222A-belt hosts two Standalone-HQ settlements — PR #177 review
# H2 corrected the earlier false premise). It only engages when a system's
# bodies tie on BOTH population and city-presence — i.e. a fully cityless,
# fully unpopulated system, where NOTHING in the data distinguishes the
# siblings (found live: GJ 702B — 12 bodies, all population=0, zero city rows
# anywhere). There, a bare body_id sort lands on the belt purely because '-'
# (the naming convention for belts/oort clouds, e.g. "GJ702B-belt") sorts
# before letters; among equally blank siblings a planet is the saner default
# host for a future settlement than orbital rubble.
_BODY_TYPE_SETTLEMENT_PREFERENCE: dict[str, int] = {
    "planet": 0,
    "moon": 1,
    "gas_giant": 2,
    "asteroid_belt": 3,
    "oort_cloud": 3,
}


def most_populated_body_in_system(
    conn: sqlite3.Connection, system_id: str, city_bearing_bodies: set[str]
) -> str | None:
    """Return the body_id in `system_id` that should host a corp HQ, or None
    if the system has no bodies.

    D-242, T-1074: descended from the heuristic the retired
    `populate_atlas_city_names_corps` (D-207, #909) used to attach a corp-HQ
    row to a body; used by `populate_corp_specialization`
    (economy_import/corporations.py) to resolve `corporations.headquarters_body`
    on every run (PR #177 H1 — recompute, never treat prior DB output as
    authored).

    Deterministic tiebreak, in order (D-010 #4; PR #177 H2):
    (1) population descending — an authored population signal always wins;
    (2) city-presence — a body in `city_bearing_bodies` (has atlas_city_names
        rows) beats a cityless sibling. This is what keeps settlements where
        they already are: a belt that hosts settlements stays the system's HQ
        anchor (GJ845-belt/GJ268-belt tenancies), while a genuinely cityless
        system falls through to (3);
    (3) body_type rank (planet < moon < gas_giant < belt/oort — see
        `_BODY_TYPE_SETTLEMENT_PREFERENCE`) — last resort among fully blank
        siblings (GJ 702B → planet GJ702Bb);
    (4) body_id ascending — final deterministic tiebreak.

    `city_bearing_bodies` is the caller's snapshot of `SELECT DISTINCT body_id
    FROM atlas_city_names` taken BEFORE this run's pool rebuild (step 7b runs
    before step 12's clear) — i.e. the PREVIOUS run's settled state. That is a
    deliberate hysteresis: established settlements anchor future HQ placement
    (settlement continuity across regens), and the fixpoint is stable — a
    run's output presence distribution reproduces itself, so consecutive
    regens compute identical values. See populate_corp_specialization's
    docstring for the one-run-lag / fresh-DB caveats.
    """
    rows = conn.execute(
        "SELECT body_id, body_type, COALESCE(population, 0) FROM bodies WHERE system_id = ?",
        (system_id,),
    ).fetchall()
    if not rows:
        return None
    rows.sort(
        key=lambda r: (
            -r[2],  # population descending
            0 if r[0] in city_bearing_bodies else 1,  # city-bearing body wins
            _BODY_TYPE_SETTLEMENT_PREFERENCE.get(r[1], 3),  # body_type rank
            r[0],  # body_id ascending
        )
    )
    return rows[0][0]


# ---------------------------------------------------------------------------
# D-242 population/settlement_class bake (T-1075)
# ---------------------------------------------------------------------------

# Zipf/rank-size curve constants (T-1075: "name + document the curve
# constants — tuning is a code change by design"). weight(rank) =
# ZIPF_WEIGHT_SCALE // rank, i.e. the classic Zipf's-law exponent s=1.0
# (rank-2 city gets half of rank-1's weight, rank-3 a third, ...) expressed
# as an integer basis-weight — no floats anywhere in the derivation (D-010).
# ZIPF_WEIGHT_SCALE only needs to be large enough that //rank doesn't
# truncate the tail ranks to 0 before normalization; 100 pooled cities on the
# densest body (Groombridge, ~16 rows pre-T-1074-cleanup) is far under this.
ZIPF_WEIGHT_SCALE = 1_000_000


def _zipf_population_spread(body_population: int, n_cities: int) -> list[int]:
    """Split `body_population` across `n_cities` settlements via a Zipf/rank-size
    curve (T-1075). Returns a list of `n_cities` integer populations, index 0 =
    rank 1 (the largest), summing EXACTLY to `body_population` (any integer-
    division remainder is added to rank 1, so the total never drifts — the
    standard largest-remainder-to-the-leader technique, not silently dropped).

    `n_cities == 0` returns `[]`. `body_population <= 0` returns all zeros
    (an authored-population gap; not this function's job to invent one, per
    D-242's "only position stays seed-derived" — POPULATION is baked from
    `bodies.population`, and an unauthored body simply bakes zero).

    Which of a body's `atlas_city_names` rows *is* "rank 1" (gets the largest
    share) is the caller's business (see `populate_settlement_population_class`
    — rank is assigned by ascending `id`, i.e. insertion order, D-010
    determinism; there is no population signal to rank by BEFORE this
    function runs, which is the whole reason it exists).
    """
    if n_cities <= 0:
        return []
    if body_population <= 0:
        return [0] * n_cities

    weights = [ZIPF_WEIGHT_SCALE // rank for rank in range(1, n_cities + 1)]
    total_weight = sum(weights)
    shares = [body_population * w // total_weight for w in weights]
    remainder = body_population - sum(shares)
    shares[0] += remainder  # give the leftover to rank 1 (D-010: deterministic)
    return shares


def populate_settlement_population_class(conn: sqlite3.Connection, dry_run: bool) -> dict:
    """Bake per-settlement population + settlement_class onto atlas_city_names
    (D-242, T-1075).

    MUST run after BOTH D-242/T-1074 steps (`populate_atlas_city_names` and
    `corporations.populate_standalone_hq_settlements`) — "the spread runs over
    the CORRECTED pool," i.e. Standalone-HQ settlement rows are included in
    the per-body city count/rank BEFORE the spread runs, not bolted on after.

    Two passes:

    (1) Population — for every body with at least one atlas_city_names row,
        split `bodies.population` (COALESCE to 0 for unauthored bodies) across
        its cities via `_zipf_population_spread`. Rank 1 (the largest share)
        is the row with the LOWEST atlas_city_names.id on that body (D-010:
        deterministic — insertion order is the only signal available; there
        is no pre-existing population to rank by, which is exactly the gap
        this function closes). A body with population=0/NULL bakes every one
        of its cities to population=0 (not an error — 6/273 populated bodies
        currently have zero pooled cities and 9/276 city-bearing bodies
        currently have zero authored population; both directions are
        legitimate content gaps, not import failures).

    (2) settlement_class — every pooled city defaults to 'PopulationBudget'
        (D-196; the >=50k-active/<5k-ghost thresholds are read from this baked
        population at GENERATION time by a later, not-yet-built consumer —
        out of this bake's scope, confirmed: no such consumer exists in
        server/src/atlas yet). The authored NameLocked override list
        (settlement_name_locked.toml, `[[hero]]` stanzas of body_id + name)
        then overrides specific rows to 'NameLocked' — matched by
        (body_id, name), NOT the volatile numeric id (see that TOML's header).
        A stanza with no matching row is a non-fatal warning (a future
        markers.json rename must not hard-fail regen-db).

    Returns a coverage dict: {"bodies_spread", "cities_populated",
    "name_locked_applied", "name_locked_unmatched"}.
    """
    # --- Pass 1: population spread -----------------------------------------
    body_population: dict[str, int] = {
        r[0]: (r[1] or 0) for r in conn.execute("SELECT body_id, population FROM bodies").fetchall()
    }
    # body_id -> [(city_id, ...)] ordered by id ascending (rank 1 = lowest id).
    city_rows_by_body: dict[str, list[int]] = {}
    for city_id, body_id in conn.execute(
        "SELECT id, body_id FROM atlas_city_names ORDER BY body_id, id"
    ).fetchall():
        city_rows_by_body.setdefault(body_id, []).append(city_id)

    population_updates: list[tuple[int, int]] = []  # (population, city_id)
    for body_id, city_ids in city_rows_by_body.items():
        pop = body_population.get(body_id, 0)
        spread = _zipf_population_spread(pop, len(city_ids))
        for city_id, city_pop in zip(city_ids, spread):
            population_updates.append((city_pop, city_id))

    # --- Pass 2: settlement_class default + NameLocked override ------------
    with open(SETTLEMENT_NAME_LOCKED_TOML, "rb") as f:
        hero_data = tomllib.load(f)
    hero_entries: list[dict] = hero_data.get("hero", [])

    if not dry_run:
        if population_updates:
            conn.executemany(
                "UPDATE atlas_city_names SET population = ? WHERE id = ?",
                population_updates,
            )
        # Default every row to PopulationBudget (D-196). Idempotent: a re-run
        # always resets to the default before re-applying overrides below, so
        # a stanza REMOVED from settlement_name_locked.toml correctly reverts
        # its row to PopulationBudget rather than leaving a stale NameLocked.
        conn.execute("UPDATE atlas_city_names SET settlement_class = 'PopulationBudget'")

    # NameLocked matching is a read-only SELECT count (safe under dry_run too
    # — a --dry-run run must still be able to REPORT an unmatched hero stanza,
    # not just skip detecting it; the WRITE is what's gated on dry_run).
    name_locked_applied = 0
    name_locked_unmatched: list[str] = []
    for entry in hero_entries:
        body_id = entry.get("body_id")
        name = entry.get("name")
        if not body_id or not name:
            continue
        match_count = conn.execute(
            "SELECT COUNT(*) FROM atlas_city_names WHERE body_id = ? AND name = ?",
            (body_id, name),
        ).fetchone()[0]
        if match_count == 0:
            name_locked_unmatched.append(f"{body_id}/{name}")
            continue
        if not dry_run:
            conn.execute(
                "UPDATE atlas_city_names SET settlement_class = 'NameLocked' "
                "WHERE body_id = ? AND name = ?",
                (body_id, name),
            )
        name_locked_applied += match_count

    if name_locked_unmatched:
        print(
            f"        warning: {len(name_locked_unmatched)} NameLocked hero stanza(s) "
            f"matched no atlas_city_names row: {name_locked_unmatched}"
        )

    return {
        "bodies_spread": len(city_rows_by_body),
        "cities_populated": len(population_updates),
        "name_locked_applied": name_locked_applied,
        "name_locked_unmatched": len(name_locked_unmatched),
    }