feat(db): import_economics owns atlas index — names pool, empty geometry (D-223 #951)
Rework the regen-db atlas path for the names-only marker pool: - populate_atlas_city_names now reads the names.cities pool instead of the retired geometry-bearing cities[] records; population/kind are deferred to placement (#955). Adds a deterministic clear-then-insert (no UNIQUE on (body_id, name)) that fixes a latent duplicate- accumulation bug — atlas_city_names dropped from an inflated 3276 to a clean 329 pooled names + 134 corp-HQ rows. - ensure_atlas_index_schema applies the canonical ATLAS INDEX block from systems-schema.sql and empties the 8 geometry tables every regen; the Phase 4 server cascade fills them (they start empty — the revealed gap). - Sol (system 'GJ 0') is permanently exempt from the normal generators: skipped in both the name-pool importer and the corp-HQ cross-ref. - MIGRATION_SQL drops the retired generate_atlas meta stamp row so the fail-closed stamp checker accepts older committed DBs. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -183,7 +183,7 @@ CREATE TABLE IF NOT EXISTS bodies (
|
||||
-- Rendering
|
||||
-- terrain_reference: repo-root-relative path to the body's heightmap PNG.
|
||||
-- Convention (enforced by populate_terrain_reference.py and assumed by
|
||||
-- generate_atlas.py and the Godot client's atlas scene loader):
|
||||
-- the atlas importers and the Godot client's atlas scene loader):
|
||||
-- wiki/star-systems/<system_slug>/bodies/<body_id>/heightmap.png
|
||||
-- where <system_slug> = system_id with spaces replaced by hyphens
|
||||
-- (e.g. "GJ 244A" → "GJ-244A"). NULL means no heightmap has been
|
||||
@@ -359,9 +359,11 @@ CREATE TABLE IF NOT EXISTS corp_lifecycle_events (
|
||||
-- these tables exist so the atlas implant app and development queries don't
|
||||
-- have to scan hundreds of JSON files. Polyline geometry stays in the files —
|
||||
-- the DB only stores scalar/filterable fields + `point_count` as a rough length
|
||||
-- proxy. Populated and refreshed by tooling/planet-gen/generate_atlas.py, which
|
||||
-- proxy. The geometry tables are populated by the server-side generation
|
||||
-- cascade (Phase 4, D-223) and start empty; tooling/economy-db/import_economics.py
|
||||
-- extracts this entire block (between BEGIN/END ATLAS INDEX markers) from this
|
||||
-- file at runtime so the DDL lives in exactly one place.
|
||||
-- file at runtime and empties the geometry tables on regen, so the DDL lives in
|
||||
-- exactly one place.
|
||||
|
||||
-- Per-body grid dimensions (one row per body that has a markers.json).
|
||||
-- Lets any query interpret the pixel coordinates below without touching disk.
|
||||
@@ -566,7 +568,7 @@ CREATE INDEX IF NOT EXISTS idx_gate_links_to ON gate_links(to_system_id);
|
||||
-- .config/hooks/pre-push — rejects pushes with stale DB (#857)
|
||||
-- /pr-push skill — triggers make regen-db if stale (#858)
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
generator_name TEXT PRIMARY KEY, -- 'import_economics' | 'generate_atlas'
|
||||
generator_name TEXT PRIMARY KEY, -- 'import_economics' (generate_atlas retired #951)
|
||||
schema_version TEXT NOT NULL, -- monotonic semver string (e.g. "1.0.0") — see #888
|
||||
schema_sha TEXT, -- SHA-1 hex of systems-schema.sql content (tamper detection)
|
||||
generator_sha TEXT NOT NULL, -- SHA-1 hex of generator source file(s) content
|
||||
|
||||
@@ -290,6 +290,13 @@ CREATE TABLE IF NOT EXISTS meta (
|
||||
-- path (fail-closed per T6) compatible with older DBs that still have the row.
|
||||
DELETE FROM meta WHERE generator_name = 'generate_brands';
|
||||
|
||||
-- Drop the retired 'generate_atlas' stamp row if it exists (#951, D-223).
|
||||
-- The atlas geometry generator was retired; import_economics now owns the
|
||||
-- atlas index, so generate_atlas no longer merits its own meta row. Without
|
||||
-- this DELETE, check-systems-db-stamp's fail-closed "unknown generator" path
|
||||
-- (T6) would reject any committed DB that still carries the old row.
|
||||
DELETE FROM meta WHERE generator_name = 'generate_atlas';
|
||||
|
||||
-- Heightmap BLOB storage (D-202, #901)
|
||||
CREATE TABLE IF NOT EXISTS atlas_body_heightmaps (
|
||||
body_id TEXT PRIMARY KEY REFERENCES bodies(body_id) ON DELETE CASCADE,
|
||||
@@ -1165,20 +1172,77 @@ def populate_body_radius_km(conn: sqlite3.Connection, dry_run: bool) -> int:
|
||||
return len(updates)
|
||||
|
||||
|
||||
# Atlas geometry index tables (D-191). These hold computed positions — city
|
||||
# centres, road/river/rail polylines, ocean/mountain extents. Under D-223 the
|
||||
# Python atlas geometry generator was retired (#951); the deterministic
|
||||
# server-side cascade (Phase 4) is the sole producer of this geometry. We keep
|
||||
# the tables (the Atlas viewer #960 and the cascade read them) but empty them on
|
||||
# every regen so the committed DB carries no stale prototype geometry — the
|
||||
# empty tables are the gap the server cascade fills.
|
||||
_ATLAS_GEOMETRY_TABLES = (
|
||||
"atlas_cities",
|
||||
"atlas_roads",
|
||||
"atlas_railroads",
|
||||
"atlas_pois",
|
||||
"atlas_rivers",
|
||||
"atlas_oceans",
|
||||
"atlas_mountain_ranges",
|
||||
"atlas_body_grids",
|
||||
)
|
||||
|
||||
_ATLAS_INDEX_BEGIN_MARKER = "-- BEGIN ATLAS INDEX"
|
||||
_ATLAS_INDEX_END_MARKER = "-- END ATLAS INDEX"
|
||||
|
||||
|
||||
def ensure_atlas_index_schema(conn: sqlite3.Connection, dry_run: bool) -> None:
|
||||
"""Apply the canonical atlas_* DDL and empty the geometry tables (D-223, #951).
|
||||
|
||||
systems-schema.sql is the single source of truth for the atlas index tables
|
||||
(the BEGIN/END ATLAS INDEX block). The retired generate_atlas.py used to
|
||||
apply this block; import_economics now owns it, since it is the only
|
||||
regen-db generator that touches systems.db's atlas tables. The block is all
|
||||
CREATE ... IF NOT EXISTS, so applying it on the committed DB is a no-op and
|
||||
on a fresh DB it creates the geometry tables.
|
||||
|
||||
After ensuring the schema, the geometry tables are cleared: their geometry
|
||||
now comes from the server cascade, not from authored markers (D-223).
|
||||
"""
|
||||
text = SCHEMA_SQL.read_text()
|
||||
try:
|
||||
start = text.index(_ATLAS_INDEX_BEGIN_MARKER)
|
||||
end = text.index(_ATLAS_INDEX_END_MARKER, start)
|
||||
except ValueError as e:
|
||||
raise RuntimeError(
|
||||
f"systems-schema.sql is missing the {_ATLAS_INDEX_BEGIN_MARKER}/"
|
||||
f"{_ATLAS_INDEX_END_MARKER} block — has the schema been restructured?"
|
||||
) from e
|
||||
conn.executescript(text[start:end])
|
||||
if not dry_run:
|
||||
for table in _ATLAS_GEOMETRY_TABLES:
|
||||
conn.execute(f"DELETE FROM {table}")
|
||||
|
||||
|
||||
def populate_atlas_city_names(conn: sqlite3.Connection, dry_run: bool) -> int:
|
||||
"""Populate atlas_city_names from wiki markers.json city entries (D-207, #908).
|
||||
"""Populate atlas_city_names from the names-only markers.json pool (D-223, #951).
|
||||
|
||||
Scans wiki/star-systems/*/bodies/*/markers.json for 'cities' arrays.
|
||||
Each entry yields one atlas_city_names row:
|
||||
- body_id : directory name (e.g. GJ0e)
|
||||
- name : city name from markers.json
|
||||
- kind : 'capital' or 'city' (default 'city')
|
||||
Scans wiki/star-systems/*/bodies/*/markers.json for the flavoured city
|
||||
name pool at `names.cities` and inserts one atlas_city_names row per name:
|
||||
- body_id : directory name (e.g. GJ0e)
|
||||
- name : pooled city name
|
||||
- kind : 'city' — capital is chosen at placement (#955)
|
||||
- economic_role : inherited from bodies.economic_role; fallback 'mixed'
|
||||
- population : from markers.json (integer)
|
||||
- corp_id : NULL — populated by populate_atlas_city_names_corps (#909)
|
||||
- reserved : 0
|
||||
- population : 0 — assigned by the server cascade at placement (#955)
|
||||
- corp_id : NULL — populated by populate_atlas_city_names_corps (#909)
|
||||
- reserved : 0
|
||||
|
||||
Uses INSERT OR REPLACE so re-runs are idempotent per (body_id, name).
|
||||
markers.json is a names-only flavoured pool (D-223): it carries no geometry
|
||||
or population. The deterministic server cascade attaches these names to
|
||||
computed settlements and assigns population/kind/position at placement time;
|
||||
this importer just loads the pool.
|
||||
|
||||
Deterministic rebuild: clears atlas_city_names first (the FK cascade clears
|
||||
atlas_city_positions), so re-runs are idempotent — there is no UNIQUE on
|
||||
(body_id, name), so without the clear a re-run would accumulate duplicates.
|
||||
Skips body directories not found in the bodies table (missing FK).
|
||||
"""
|
||||
# Build body_id -> economic_role map
|
||||
@@ -1190,39 +1254,53 @@ def populate_atlas_city_names(conn: sqlite3.Connection, dry_run: bool) -> int:
|
||||
|
||||
valid_body_ids: set[str] = set(body_roles.keys())
|
||||
|
||||
# Sol (system 'GJ 0') is permanently exempt from the normal generators
|
||||
# (D-223, #951): its bodies use real Earth/Mars/Luna geography via
|
||||
# sol_import.py and keep geometry-bearing markers.json as preserved config.
|
||||
# Sol names come from its own (future) scripted integration, not the names
|
||||
# pool — skip Sol bodies here regardless of their markers format.
|
||||
sol_body_ids: set[str] = {
|
||||
r[0] for r in conn.execute(
|
||||
"SELECT body_id FROM bodies WHERE system_id = 'GJ 0'"
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
rows: list[tuple] = []
|
||||
skipped_bodies: list[str] = []
|
||||
|
||||
pattern = str(WIKI_STAR_SYSTEMS / "*" / "bodies" / "*" / "markers.json")
|
||||
for markers_path in sorted(glob.glob(pattern)):
|
||||
body_id = markers_path.split("/bodies/")[1].split("/")[0]
|
||||
if body_id not in valid_body_ids:
|
||||
skipped_bodies.append(body_id)
|
||||
if body_id not in valid_body_ids or body_id in sol_body_ids:
|
||||
if body_id not in valid_body_ids:
|
||||
skipped_bodies.append(body_id)
|
||||
continue
|
||||
|
||||
with open(markers_path) as fh:
|
||||
data = json.load(fh)
|
||||
|
||||
for city in data.get("cities", []):
|
||||
name = city.get("name", "").strip()
|
||||
names_pool = (data.get("names") or {}).get("cities") or []
|
||||
economic_role = body_roles[body_id]
|
||||
for raw_name in names_pool:
|
||||
name = (raw_name or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
kind = city.get("kind", "city")
|
||||
population = int(city.get("population", 0))
|
||||
economic_role = body_roles[body_id]
|
||||
rows.append((body_id, name, kind, economic_role, population))
|
||||
# kind defaults to 'city'; population 0 until placement (#955).
|
||||
rows.append((body_id, name, "city", economic_role, 0))
|
||||
|
||||
if skipped_bodies:
|
||||
unique = sorted(set(skipped_bodies))
|
||||
print(f" warning: {len(unique)} body dirs not in DB — skipped: {unique[:5]}")
|
||||
|
||||
if not dry_run and rows:
|
||||
conn.executemany(
|
||||
"""INSERT OR REPLACE INTO atlas_city_names
|
||||
(body_id, name, kind, economic_role, population)
|
||||
VALUES (?, ?, ?, ?, ?)""",
|
||||
rows,
|
||||
)
|
||||
if not dry_run:
|
||||
conn.execute("DELETE FROM atlas_city_names")
|
||||
if rows:
|
||||
conn.executemany(
|
||||
"""INSERT INTO atlas_city_names
|
||||
(body_id, name, kind, economic_role, population)
|
||||
VALUES (?, ?, ?, ?, ?)""",
|
||||
rows,
|
||||
)
|
||||
|
||||
return len(rows)
|
||||
|
||||
@@ -1305,6 +1383,11 @@ def populate_atlas_city_names_corps(conn: sqlite3.Connection, dry_run: bool) ->
|
||||
if match_id is not None:
|
||||
updated.append((corp_id, match_id))
|
||||
else:
|
||||
# Sol (system 'GJ 0') is exempt from the normal generators (D-223,
|
||||
# #951) — do not synthesize a reserved corp-HQ row on a Sol body;
|
||||
# Sol's atlas data comes from its own scripted integration.
|
||||
if headquarters_system == "GJ 0":
|
||||
continue
|
||||
# Insert a reserved row on the most-populated body in the system
|
||||
candidates = sys_bodies.get(headquarters_system, [])
|
||||
if not candidates:
|
||||
@@ -1499,6 +1582,8 @@ def main():
|
||||
for table, col, col_type in COLUMN_MIGRATIONS:
|
||||
_add_column(conn, table, col, col_type)
|
||||
conn.executescript(MIGRATION_SQL)
|
||||
# Atlas index tables: apply canonical DDL + empty geometry (D-223, #951)
|
||||
ensure_atlas_index_schema(conn, args.dry_run)
|
||||
print(" tables and columns ready")
|
||||
|
||||
# Clear economics tables in FK-safe order (children before parents)
|
||||
|
||||
Reference in New Issue
Block a user