feat(db): import_economics owns atlas index — names pool, empty geometry (D-223 #951)

Rework the regen-db atlas path for the names-only marker pool:

- populate_atlas_city_names now reads the names.cities pool instead of
  the retired geometry-bearing cities[] records; population/kind are
  deferred to placement (#955). Adds a deterministic clear-then-insert
  (no UNIQUE on (body_id, name)) that fixes a latent duplicate-
  accumulation bug — atlas_city_names dropped from an inflated 3276 to a
  clean 329 pooled names + 134 corp-HQ rows.
- ensure_atlas_index_schema applies the canonical ATLAS INDEX block from
  systems-schema.sql and empties the 8 geometry tables every regen; the
  Phase 4 server cascade fills them (they start empty — the revealed gap).
- Sol (system 'GJ 0') is permanently exempt from the normal generators:
  skipped in both the name-pool importer and the corp-HQ cross-ref.
- MIGRATION_SQL drops the retired generate_atlas meta stamp row so the
  fail-closed stamp checker accepts older committed DBs.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-22 23:28:53 +02:00
co-authored by Claude Opus 4.7
parent 642be3ac41
commit b0bfbfc7dd
2 changed files with 116 additions and 29 deletions
+110 -25
View File
@@ -290,6 +290,13 @@ CREATE TABLE IF NOT EXISTS meta (
-- path (fail-closed per T6) compatible with older DBs that still have the row.
DELETE FROM meta WHERE generator_name = 'generate_brands';
-- Drop the retired 'generate_atlas' stamp row if it exists (#951, D-223).
-- The atlas geometry generator was retired; import_economics now owns the
-- atlas index, so generate_atlas no longer merits its own meta row. Without
-- this DELETE, check-systems-db-stamp's fail-closed "unknown generator" path
-- (T6) would reject any committed DB that still carries the old row.
DELETE FROM meta WHERE generator_name = 'generate_atlas';
-- Heightmap BLOB storage (D-202, #901)
CREATE TABLE IF NOT EXISTS atlas_body_heightmaps (
body_id TEXT PRIMARY KEY REFERENCES bodies(body_id) ON DELETE CASCADE,
@@ -1165,20 +1172,77 @@ def populate_body_radius_km(conn: sqlite3.Connection, dry_run: bool) -> int:
return len(updates)
# Atlas geometry index tables (D-191). These hold computed positions — city
# centres, road/river/rail polylines, ocean/mountain extents. Under D-223 the
# Python atlas geometry generator was retired (#951); the deterministic
# server-side cascade (Phase 4) is the sole producer of this geometry. We keep
# the tables (the Atlas viewer #960 and the cascade read them) but empty them on
# every regen so the committed DB carries no stale prototype geometry — the
# empty tables are the gap the server cascade fills.
_ATLAS_GEOMETRY_TABLES = (
"atlas_cities",
"atlas_roads",
"atlas_railroads",
"atlas_pois",
"atlas_rivers",
"atlas_oceans",
"atlas_mountain_ranges",
"atlas_body_grids",
)
_ATLAS_INDEX_BEGIN_MARKER = "-- BEGIN ATLAS INDEX"
_ATLAS_INDEX_END_MARKER = "-- END ATLAS INDEX"
def ensure_atlas_index_schema(conn: sqlite3.Connection, dry_run: bool) -> None:
"""Apply the canonical atlas_* DDL and empty the geometry tables (D-223, #951).
systems-schema.sql is the single source of truth for the atlas index tables
(the BEGIN/END ATLAS INDEX block). The retired generate_atlas.py used to
apply this block; import_economics now owns it, since it is the only
regen-db generator that touches systems.db's atlas tables. The block is all
CREATE ... IF NOT EXISTS, so applying it on the committed DB is a no-op and
on a fresh DB it creates the geometry tables.
After ensuring the schema, the geometry tables are cleared: their geometry
now comes from the server cascade, not from authored markers (D-223).
"""
text = SCHEMA_SQL.read_text()
try:
start = text.index(_ATLAS_INDEX_BEGIN_MARKER)
end = text.index(_ATLAS_INDEX_END_MARKER, start)
except ValueError as e:
raise RuntimeError(
f"systems-schema.sql is missing the {_ATLAS_INDEX_BEGIN_MARKER}/"
f"{_ATLAS_INDEX_END_MARKER} block — has the schema been restructured?"
) from e
conn.executescript(text[start:end])
if not dry_run:
for table in _ATLAS_GEOMETRY_TABLES:
conn.execute(f"DELETE FROM {table}")
def populate_atlas_city_names(conn: sqlite3.Connection, dry_run: bool) -> int:
"""Populate atlas_city_names from wiki markers.json city entries (D-207, #908).
"""Populate atlas_city_names from the names-only markers.json pool (D-223, #951).
Scans wiki/star-systems/*/bodies/*/markers.json for 'cities' arrays.
Each entry yields one atlas_city_names row:
- body_id : directory name (e.g. GJ0e)
- name : city name from markers.json
- kind : 'capital' or 'city' (default 'city')
Scans wiki/star-systems/*/bodies/*/markers.json for the flavoured city
name pool at `names.cities` and inserts one atlas_city_names row per name:
- body_id : directory name (e.g. GJ0e)
- name : pooled city name
- kind : 'city' — capital is chosen at placement (#955)
- economic_role : inherited from bodies.economic_role; fallback 'mixed'
- population : from markers.json (integer)
- corp_id : NULL — populated by populate_atlas_city_names_corps (#909)
- reserved : 0
- population : 0 — assigned by the server cascade at placement (#955)
- corp_id : NULL — populated by populate_atlas_city_names_corps (#909)
- reserved : 0
Uses INSERT OR REPLACE so re-runs are idempotent per (body_id, name).
markers.json is a names-only flavoured pool (D-223): it carries no geometry
or population. The deterministic server cascade attaches these names to
computed settlements and assigns population/kind/position at placement time;
this importer just loads the pool.
Deterministic rebuild: clears atlas_city_names first (the FK cascade clears
atlas_city_positions), so re-runs are idempotent — there is no UNIQUE on
(body_id, name), so without the clear a re-run would accumulate duplicates.
Skips body directories not found in the bodies table (missing FK).
"""
# Build body_id -> economic_role map
@@ -1190,39 +1254,53 @@ def populate_atlas_city_names(conn: sqlite3.Connection, dry_run: bool) -> int:
valid_body_ids: set[str] = set(body_roles.keys())
# Sol (system 'GJ 0') is permanently exempt from the normal generators
# (D-223, #951): its bodies use real Earth/Mars/Luna geography via
# sol_import.py and keep geometry-bearing markers.json as preserved config.
# Sol names come from its own (future) scripted integration, not the names
# pool — skip Sol bodies here regardless of their markers format.
sol_body_ids: set[str] = {
r[0] for r in conn.execute(
"SELECT body_id FROM bodies WHERE system_id = 'GJ 0'"
).fetchall()
}
rows: list[tuple] = []
skipped_bodies: list[str] = []
pattern = str(WIKI_STAR_SYSTEMS / "*" / "bodies" / "*" / "markers.json")
for markers_path in sorted(glob.glob(pattern)):
body_id = markers_path.split("/bodies/")[1].split("/")[0]
if body_id not in valid_body_ids:
skipped_bodies.append(body_id)
if body_id not in valid_body_ids or body_id in sol_body_ids:
if body_id not in valid_body_ids:
skipped_bodies.append(body_id)
continue
with open(markers_path) as fh:
data = json.load(fh)
for city in data.get("cities", []):
name = city.get("name", "").strip()
names_pool = (data.get("names") or {}).get("cities") or []
economic_role = body_roles[body_id]
for raw_name in names_pool:
name = (raw_name or "").strip()
if not name:
continue
kind = city.get("kind", "city")
population = int(city.get("population", 0))
economic_role = body_roles[body_id]
rows.append((body_id, name, kind, economic_role, population))
# kind defaults to 'city'; population 0 until placement (#955).
rows.append((body_id, name, "city", economic_role, 0))
if skipped_bodies:
unique = sorted(set(skipped_bodies))
print(f" warning: {len(unique)} body dirs not in DB — skipped: {unique[:5]}")
if not dry_run and rows:
conn.executemany(
"""INSERT OR REPLACE INTO atlas_city_names
(body_id, name, kind, economic_role, population)
VALUES (?, ?, ?, ?, ?)""",
rows,
)
if not dry_run:
conn.execute("DELETE FROM atlas_city_names")
if rows:
conn.executemany(
"""INSERT INTO atlas_city_names
(body_id, name, kind, economic_role, population)
VALUES (?, ?, ?, ?, ?)""",
rows,
)
return len(rows)
@@ -1305,6 +1383,11 @@ def populate_atlas_city_names_corps(conn: sqlite3.Connection, dry_run: bool) ->
if match_id is not None:
updated.append((corp_id, match_id))
else:
# Sol (system 'GJ 0') is exempt from the normal generators (D-223,
# #951) — do not synthesize a reserved corp-HQ row on a Sol body;
# Sol's atlas data comes from its own scripted integration.
if headquarters_system == "GJ 0":
continue
# Insert a reserved row on the most-populated body in the system
candidates = sys_bodies.get(headquarters_system, [])
if not candidates:
@@ -1499,6 +1582,8 @@ def main():
for table, col, col_type in COLUMN_MIGRATIONS:
_add_column(conn, table, col, col_type)
conn.executescript(MIGRATION_SQL)
# Atlas index tables: apply canonical DDL + empty geometry (D-223, #951)
ensure_atlas_index_schema(conn, args.dry_run)
print(" tables and columns ready")
# Clear economics tables in FK-safe order (children before parents)