feat(db): import_economics owns atlas index — names pool, empty geometry (D-223 #951)
Rework the regen-db atlas path for the names-only marker pool: - populate_atlas_city_names now reads the names.cities pool instead of the retired geometry-bearing cities[] records; population/kind are deferred to placement (#955). Adds a deterministic clear-then-insert (no UNIQUE on (body_id, name)) that fixes a latent duplicate- accumulation bug — atlas_city_names dropped from an inflated 3276 to a clean 329 pooled names + 134 corp-HQ rows. - ensure_atlas_index_schema applies the canonical ATLAS INDEX block from systems-schema.sql and empties the 8 geometry tables every regen; the Phase 4 server cascade fills them (they start empty — the revealed gap). - Sol (system 'GJ 0') is permanently exempt from the normal generators: skipped in both the name-pool importer and the corp-HQ cross-ref. - MIGRATION_SQL drops the retired generate_atlas meta stamp row so the fail-closed stamp checker accepts older committed DBs. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -183,7 +183,7 @@ CREATE TABLE IF NOT EXISTS bodies (
|
|||||||
-- Rendering
|
-- Rendering
|
||||||
-- terrain_reference: repo-root-relative path to the body's heightmap PNG.
|
-- terrain_reference: repo-root-relative path to the body's heightmap PNG.
|
||||||
-- Convention (enforced by populate_terrain_reference.py and assumed by
|
-- Convention (enforced by populate_terrain_reference.py and assumed by
|
||||||
-- generate_atlas.py and the Godot client's atlas scene loader):
|
-- the atlas importers and the Godot client's atlas scene loader):
|
||||||
-- wiki/star-systems/<system_slug>/bodies/<body_id>/heightmap.png
|
-- wiki/star-systems/<system_slug>/bodies/<body_id>/heightmap.png
|
||||||
-- where <system_slug> = system_id with spaces replaced by hyphens
|
-- where <system_slug> = system_id with spaces replaced by hyphens
|
||||||
-- (e.g. "GJ 244A" → "GJ-244A"). NULL means no heightmap has been
|
-- (e.g. "GJ 244A" → "GJ-244A"). NULL means no heightmap has been
|
||||||
@@ -359,9 +359,11 @@ CREATE TABLE IF NOT EXISTS corp_lifecycle_events (
|
|||||||
-- these tables exist so the atlas implant app and development queries don't
|
-- these tables exist so the atlas implant app and development queries don't
|
||||||
-- have to scan hundreds of JSON files. Polyline geometry stays in the files —
|
-- have to scan hundreds of JSON files. Polyline geometry stays in the files —
|
||||||
-- the DB only stores scalar/filterable fields + `point_count` as a rough length
|
-- the DB only stores scalar/filterable fields + `point_count` as a rough length
|
||||||
-- proxy. Populated and refreshed by tooling/planet-gen/generate_atlas.py, which
|
-- proxy. The geometry tables are populated by the server-side generation
|
||||||
|
-- cascade (Phase 4, D-223) and start empty; tooling/economy-db/import_economics.py
|
||||||
-- extracts this entire block (between BEGIN/END ATLAS INDEX markers) from this
|
-- extracts this entire block (between BEGIN/END ATLAS INDEX markers) from this
|
||||||
-- file at runtime so the DDL lives in exactly one place.
|
-- file at runtime and empties the geometry tables on regen, so the DDL lives in
|
||||||
|
-- exactly one place.
|
||||||
|
|
||||||
-- Per-body grid dimensions (one row per body that has a markers.json).
|
-- Per-body grid dimensions (one row per body that has a markers.json).
|
||||||
-- Lets any query interpret the pixel coordinates below without touching disk.
|
-- Lets any query interpret the pixel coordinates below without touching disk.
|
||||||
@@ -566,7 +568,7 @@ CREATE INDEX IF NOT EXISTS idx_gate_links_to ON gate_links(to_system_id);
|
|||||||
-- .config/hooks/pre-push — rejects pushes with stale DB (#857)
|
-- .config/hooks/pre-push — rejects pushes with stale DB (#857)
|
||||||
-- /pr-push skill — triggers make regen-db if stale (#858)
|
-- /pr-push skill — triggers make regen-db if stale (#858)
|
||||||
CREATE TABLE IF NOT EXISTS meta (
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
generator_name TEXT PRIMARY KEY, -- 'import_economics' | 'generate_atlas'
|
generator_name TEXT PRIMARY KEY, -- 'import_economics' (generate_atlas retired #951)
|
||||||
schema_version TEXT NOT NULL, -- monotonic semver string (e.g. "1.0.0") — see #888
|
schema_version TEXT NOT NULL, -- monotonic semver string (e.g. "1.0.0") — see #888
|
||||||
schema_sha TEXT, -- SHA-1 hex of systems-schema.sql content (tamper detection)
|
schema_sha TEXT, -- SHA-1 hex of systems-schema.sql content (tamper detection)
|
||||||
generator_sha TEXT NOT NULL, -- SHA-1 hex of generator source file(s) content
|
generator_sha TEXT NOT NULL, -- SHA-1 hex of generator source file(s) content
|
||||||
|
|||||||
@@ -290,6 +290,13 @@ CREATE TABLE IF NOT EXISTS meta (
|
|||||||
-- path (fail-closed per T6) compatible with older DBs that still have the row.
|
-- path (fail-closed per T6) compatible with older DBs that still have the row.
|
||||||
DELETE FROM meta WHERE generator_name = 'generate_brands';
|
DELETE FROM meta WHERE generator_name = 'generate_brands';
|
||||||
|
|
||||||
|
-- Drop the retired 'generate_atlas' stamp row if it exists (#951, D-223).
|
||||||
|
-- The atlas geometry generator was retired; import_economics now owns the
|
||||||
|
-- atlas index, so generate_atlas no longer merits its own meta row. Without
|
||||||
|
-- this DELETE, check-systems-db-stamp's fail-closed "unknown generator" path
|
||||||
|
-- (T6) would reject any committed DB that still carries the old row.
|
||||||
|
DELETE FROM meta WHERE generator_name = 'generate_atlas';
|
||||||
|
|
||||||
-- Heightmap BLOB storage (D-202, #901)
|
-- Heightmap BLOB storage (D-202, #901)
|
||||||
CREATE TABLE IF NOT EXISTS atlas_body_heightmaps (
|
CREATE TABLE IF NOT EXISTS atlas_body_heightmaps (
|
||||||
body_id TEXT PRIMARY KEY REFERENCES bodies(body_id) ON DELETE CASCADE,
|
body_id TEXT PRIMARY KEY REFERENCES bodies(body_id) ON DELETE CASCADE,
|
||||||
@@ -1165,20 +1172,77 @@ def populate_body_radius_km(conn: sqlite3.Connection, dry_run: bool) -> int:
|
|||||||
return len(updates)
|
return len(updates)
|
||||||
|
|
||||||
|
|
||||||
|
# Atlas geometry index tables (D-191). These hold computed positions — city
|
||||||
|
# centres, road/river/rail polylines, ocean/mountain extents. Under D-223 the
|
||||||
|
# Python atlas geometry generator was retired (#951); the deterministic
|
||||||
|
# server-side cascade (Phase 4) is the sole producer of this geometry. We keep
|
||||||
|
# the tables (the Atlas viewer #960 and the cascade read them) but empty them on
|
||||||
|
# every regen so the committed DB carries no stale prototype geometry — the
|
||||||
|
# empty tables are the gap the server cascade fills.
|
||||||
|
_ATLAS_GEOMETRY_TABLES = (
|
||||||
|
"atlas_cities",
|
||||||
|
"atlas_roads",
|
||||||
|
"atlas_railroads",
|
||||||
|
"atlas_pois",
|
||||||
|
"atlas_rivers",
|
||||||
|
"atlas_oceans",
|
||||||
|
"atlas_mountain_ranges",
|
||||||
|
"atlas_body_grids",
|
||||||
|
)
|
||||||
|
|
||||||
|
_ATLAS_INDEX_BEGIN_MARKER = "-- BEGIN ATLAS INDEX"
|
||||||
|
_ATLAS_INDEX_END_MARKER = "-- END ATLAS INDEX"
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_atlas_index_schema(conn: sqlite3.Connection, dry_run: bool) -> None:
|
||||||
|
"""Apply the canonical atlas_* DDL and empty the geometry tables (D-223, #951).
|
||||||
|
|
||||||
|
systems-schema.sql is the single source of truth for the atlas index tables
|
||||||
|
(the BEGIN/END ATLAS INDEX block). The retired generate_atlas.py used to
|
||||||
|
apply this block; import_economics now owns it, since it is the only
|
||||||
|
regen-db generator that touches systems.db's atlas tables. The block is all
|
||||||
|
CREATE ... IF NOT EXISTS, so applying it on the committed DB is a no-op and
|
||||||
|
on a fresh DB it creates the geometry tables.
|
||||||
|
|
||||||
|
After ensuring the schema, the geometry tables are cleared: their geometry
|
||||||
|
now comes from the server cascade, not from authored markers (D-223).
|
||||||
|
"""
|
||||||
|
text = SCHEMA_SQL.read_text()
|
||||||
|
try:
|
||||||
|
start = text.index(_ATLAS_INDEX_BEGIN_MARKER)
|
||||||
|
end = text.index(_ATLAS_INDEX_END_MARKER, start)
|
||||||
|
except ValueError as e:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"systems-schema.sql is missing the {_ATLAS_INDEX_BEGIN_MARKER}/"
|
||||||
|
f"{_ATLAS_INDEX_END_MARKER} block — has the schema been restructured?"
|
||||||
|
) from e
|
||||||
|
conn.executescript(text[start:end])
|
||||||
|
if not dry_run:
|
||||||
|
for table in _ATLAS_GEOMETRY_TABLES:
|
||||||
|
conn.execute(f"DELETE FROM {table}")
|
||||||
|
|
||||||
|
|
||||||
def populate_atlas_city_names(conn: sqlite3.Connection, dry_run: bool) -> int:
|
def populate_atlas_city_names(conn: sqlite3.Connection, dry_run: bool) -> int:
|
||||||
"""Populate atlas_city_names from wiki markers.json city entries (D-207, #908).
|
"""Populate atlas_city_names from the names-only markers.json pool (D-223, #951).
|
||||||
|
|
||||||
Scans wiki/star-systems/*/bodies/*/markers.json for 'cities' arrays.
|
Scans wiki/star-systems/*/bodies/*/markers.json for the flavoured city
|
||||||
Each entry yields one atlas_city_names row:
|
name pool at `names.cities` and inserts one atlas_city_names row per name:
|
||||||
- body_id : directory name (e.g. GJ0e)
|
- body_id : directory name (e.g. GJ0e)
|
||||||
- name : city name from markers.json
|
- name : pooled city name
|
||||||
- kind : 'capital' or 'city' (default 'city')
|
- kind : 'city' — capital is chosen at placement (#955)
|
||||||
- economic_role : inherited from bodies.economic_role; fallback 'mixed'
|
- economic_role : inherited from bodies.economic_role; fallback 'mixed'
|
||||||
- population : from markers.json (integer)
|
- population : 0 — assigned by the server cascade at placement (#955)
|
||||||
- corp_id : NULL — populated by populate_atlas_city_names_corps (#909)
|
- corp_id : NULL — populated by populate_atlas_city_names_corps (#909)
|
||||||
- reserved : 0
|
- reserved : 0
|
||||||
|
|
||||||
Uses INSERT OR REPLACE so re-runs are idempotent per (body_id, name).
|
markers.json is a names-only flavoured pool (D-223): it carries no geometry
|
||||||
|
or population. The deterministic server cascade attaches these names to
|
||||||
|
computed settlements and assigns population/kind/position at placement time;
|
||||||
|
this importer just loads the pool.
|
||||||
|
|
||||||
|
Deterministic rebuild: clears atlas_city_names first (the FK cascade clears
|
||||||
|
atlas_city_positions), so re-runs are idempotent — there is no UNIQUE on
|
||||||
|
(body_id, name), so without the clear a re-run would accumulate duplicates.
|
||||||
Skips body directories not found in the bodies table (missing FK).
|
Skips body directories not found in the bodies table (missing FK).
|
||||||
"""
|
"""
|
||||||
# Build body_id -> economic_role map
|
# Build body_id -> economic_role map
|
||||||
@@ -1190,39 +1254,53 @@ def populate_atlas_city_names(conn: sqlite3.Connection, dry_run: bool) -> int:
|
|||||||
|
|
||||||
valid_body_ids: set[str] = set(body_roles.keys())
|
valid_body_ids: set[str] = set(body_roles.keys())
|
||||||
|
|
||||||
|
# Sol (system 'GJ 0') is permanently exempt from the normal generators
|
||||||
|
# (D-223, #951): its bodies use real Earth/Mars/Luna geography via
|
||||||
|
# sol_import.py and keep geometry-bearing markers.json as preserved config.
|
||||||
|
# Sol names come from its own (future) scripted integration, not the names
|
||||||
|
# pool — skip Sol bodies here regardless of their markers format.
|
||||||
|
sol_body_ids: set[str] = {
|
||||||
|
r[0] for r in conn.execute(
|
||||||
|
"SELECT body_id FROM bodies WHERE system_id = 'GJ 0'"
|
||||||
|
).fetchall()
|
||||||
|
}
|
||||||
|
|
||||||
rows: list[tuple] = []
|
rows: list[tuple] = []
|
||||||
skipped_bodies: list[str] = []
|
skipped_bodies: list[str] = []
|
||||||
|
|
||||||
pattern = str(WIKI_STAR_SYSTEMS / "*" / "bodies" / "*" / "markers.json")
|
pattern = str(WIKI_STAR_SYSTEMS / "*" / "bodies" / "*" / "markers.json")
|
||||||
for markers_path in sorted(glob.glob(pattern)):
|
for markers_path in sorted(glob.glob(pattern)):
|
||||||
body_id = markers_path.split("/bodies/")[1].split("/")[0]
|
body_id = markers_path.split("/bodies/")[1].split("/")[0]
|
||||||
if body_id not in valid_body_ids:
|
if body_id not in valid_body_ids or body_id in sol_body_ids:
|
||||||
skipped_bodies.append(body_id)
|
if body_id not in valid_body_ids:
|
||||||
|
skipped_bodies.append(body_id)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
with open(markers_path) as fh:
|
with open(markers_path) as fh:
|
||||||
data = json.load(fh)
|
data = json.load(fh)
|
||||||
|
|
||||||
for city in data.get("cities", []):
|
names_pool = (data.get("names") or {}).get("cities") or []
|
||||||
name = city.get("name", "").strip()
|
economic_role = body_roles[body_id]
|
||||||
|
for raw_name in names_pool:
|
||||||
|
name = (raw_name or "").strip()
|
||||||
if not name:
|
if not name:
|
||||||
continue
|
continue
|
||||||
kind = city.get("kind", "city")
|
# kind defaults to 'city'; population 0 until placement (#955).
|
||||||
population = int(city.get("population", 0))
|
rows.append((body_id, name, "city", economic_role, 0))
|
||||||
economic_role = body_roles[body_id]
|
|
||||||
rows.append((body_id, name, kind, economic_role, population))
|
|
||||||
|
|
||||||
if skipped_bodies:
|
if skipped_bodies:
|
||||||
unique = sorted(set(skipped_bodies))
|
unique = sorted(set(skipped_bodies))
|
||||||
print(f" warning: {len(unique)} body dirs not in DB — skipped: {unique[:5]}")
|
print(f" warning: {len(unique)} body dirs not in DB — skipped: {unique[:5]}")
|
||||||
|
|
||||||
if not dry_run and rows:
|
if not dry_run:
|
||||||
conn.executemany(
|
conn.execute("DELETE FROM atlas_city_names")
|
||||||
"""INSERT OR REPLACE INTO atlas_city_names
|
if rows:
|
||||||
(body_id, name, kind, economic_role, population)
|
conn.executemany(
|
||||||
VALUES (?, ?, ?, ?, ?)""",
|
"""INSERT INTO atlas_city_names
|
||||||
rows,
|
(body_id, name, kind, economic_role, population)
|
||||||
)
|
VALUES (?, ?, ?, ?, ?)""",
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
|
||||||
return len(rows)
|
return len(rows)
|
||||||
|
|
||||||
@@ -1305,6 +1383,11 @@ def populate_atlas_city_names_corps(conn: sqlite3.Connection, dry_run: bool) ->
|
|||||||
if match_id is not None:
|
if match_id is not None:
|
||||||
updated.append((corp_id, match_id))
|
updated.append((corp_id, match_id))
|
||||||
else:
|
else:
|
||||||
|
# Sol (system 'GJ 0') is exempt from the normal generators (D-223,
|
||||||
|
# #951) — do not synthesize a reserved corp-HQ row on a Sol body;
|
||||||
|
# Sol's atlas data comes from its own scripted integration.
|
||||||
|
if headquarters_system == "GJ 0":
|
||||||
|
continue
|
||||||
# Insert a reserved row on the most-populated body in the system
|
# Insert a reserved row on the most-populated body in the system
|
||||||
candidates = sys_bodies.get(headquarters_system, [])
|
candidates = sys_bodies.get(headquarters_system, [])
|
||||||
if not candidates:
|
if not candidates:
|
||||||
@@ -1499,6 +1582,8 @@ def main():
|
|||||||
for table, col, col_type in COLUMN_MIGRATIONS:
|
for table, col, col_type in COLUMN_MIGRATIONS:
|
||||||
_add_column(conn, table, col, col_type)
|
_add_column(conn, table, col, col_type)
|
||||||
conn.executescript(MIGRATION_SQL)
|
conn.executescript(MIGRATION_SQL)
|
||||||
|
# Atlas index tables: apply canonical DDL + empty geometry (D-223, #951)
|
||||||
|
ensure_atlas_index_schema(conn, args.dry_run)
|
||||||
print(" tables and columns ready")
|
print(" tables and columns ready")
|
||||||
|
|
||||||
# Clear economics tables in FK-safe order (children before parents)
|
# Clear economics tables in FK-safe order (children before parents)
|
||||||
|
|||||||
Reference in New Issue
Block a user