feat(tooling): rewrite import_heightmaps as the 16-bit heightmap + relief bake (#963)
Replaces the dead atlas_body_heightmaps DB-BLOB importer (Tyre review B2 — it asserted 256×512 and wrote a dropped table) with the canonical asset bake: - one-time rename of the legacy color heightmap.png → reliefmap.png for ALL bodies (incl. Sol — display-file rename only, no re-sim); - for each non-Sol inhabited body: simulate() at 1024×512 → clean reliefmap.png (render_heightmap, no painted features) + 16-bit grayscale heightmap.png with sea_level in a tEXt chunk; - no systems.db writes; uses parse_system for proper body defs; run via uv. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -1,218 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
import_heightmaps.py — Import terrain elevation grids into atlas_body_heightmaps.
|
||||
import_heightmaps.py — bake per-body canonical elevation assets (D-202 amended, #963).
|
||||
|
||||
For each inhabited body with a terrain_reference, simulates the terrain via
|
||||
planet_simulation.simulate() and stores the float32 LE elevation BLOB plus
|
||||
sea_level metadata in atlas_body_heightmaps (#906, D-202).
|
||||
This is the build-time elevation producer. For each NON-Sol inhabited body it
|
||||
runs planet_simulation.simulate() at the canonical 1024×512 grid and writes two
|
||||
files into the body's wiki directory:
|
||||
|
||||
The BLOB format matches the Rust loader spec (D-202):
|
||||
- float32 little-endian, row-major
|
||||
- width × height values, each in [0.0, 1.0]
|
||||
- width = GRID_W (512), height = GRID_H (256)
|
||||
reliefmap.png — clean color hypsometric render (display / Atlas), 1024×512
|
||||
heightmap.png — 16-bit GRAYSCALE elevation, the canonical cascade input,
|
||||
with sea_level stored in a PNG tEXt chunk
|
||||
|
||||
Incremental: bodies that already have a row in atlas_body_heightmaps are
|
||||
skipped unless --force is passed.
|
||||
It also performs the one-time rename of the legacy color `heightmap.png`
|
||||
(which was a relief render) → `reliefmap.png` across ALL bodies, including Sol
|
||||
(GJ-0) — Sol's terrain is real-geography (sol_import) and is NOT re-simulated,
|
||||
only its display file is renamed.
|
||||
|
||||
Usage:
|
||||
tooling/planet-gen/import_heightmaps.py
|
||||
tooling/planet-gen/import_heightmaps.py --body GJ380c
|
||||
tooling/planet-gen/import_heightmaps.py --force
|
||||
tooling/planet-gen/import_heightmaps.py --dry-run
|
||||
No systems.db writes: the PNG is the store (the atlas_body_heightmaps BLOB
|
||||
table is dropped). Run with uv (numpy/scipy/Pillow):
|
||||
|
||||
Exit codes:
|
||||
0 completed (possibly with skipped or errored bodies)
|
||||
1 fatal error (missing DB, schema error)
|
||||
uv run python tooling/planet-gen/import_heightmaps.py # full bake
|
||||
uv run python tooling/planet-gen/import_heightmaps.py --limit 3 # smoke test
|
||||
uv run python tooling/planet-gen/import_heightmaps.py --dry-run
|
||||
|
||||
Exit codes: 0 = completed (possibly with per-body errors), 1 = fatal.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
TOOLING_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = (TOOLING_DIR / ".." / "..").resolve()
|
||||
sys.path.insert(0, str(TOOLING_DIR))
|
||||
|
||||
_venv_python = REPO_ROOT / ".venv" / "bin" / "python"
|
||||
if _venv_python.exists() and Path(sys.executable).resolve() != _venv_python.resolve():
|
||||
import os
|
||||
os.execv(str(_venv_python), [str(_venv_python)] + sys.argv)
|
||||
from body_definition_parser import parse_system # noqa: E402
|
||||
from planet_simulation import simulate # noqa: E402
|
||||
from render_heightmap import render_heightmap # noqa: E402
|
||||
|
||||
import numpy as np
|
||||
import sqlite3
|
||||
|
||||
from atlas_common import (
|
||||
GRID_W,
|
||||
GRID_H,
|
||||
DB_PATH,
|
||||
ensure_atlas_schema,
|
||||
load_body_def,
|
||||
query_inhabited_bodies,
|
||||
)
|
||||
from planet_simulation import simulate
|
||||
DB_PATH = REPO_ROOT / "server" / "data" / "systems.db"
|
||||
WIKI_SYSTEMS = REPO_ROOT / "wiki" / "star-systems"
|
||||
|
||||
|
||||
def _elevation_to_blob(elevation: np.ndarray) -> bytes:
|
||||
"""Convert a float32 elevation grid to a little-endian BLOB."""
|
||||
arr = elevation.astype("<f4") # float32 LE, explicit
|
||||
assert arr.shape == (GRID_H, GRID_W), (
|
||||
f"elevation shape {arr.shape} does not match expected ({GRID_H}, {GRID_W})"
|
||||
)
|
||||
return arr.tobytes()
|
||||
def rename_legacy_heightmaps(dry_run: bool) -> int:
|
||||
"""One-time: legacy color `heightmap.png` (a relief render) → `reliefmap.png`,
|
||||
for every body. Bodies the bake re-simulates get their `reliefmap.png`
|
||||
overwritten with a fresh 1024 render afterwards. Idempotent: skips bodies
|
||||
that already have `reliefmap.png`."""
|
||||
n = 0
|
||||
for hm in sorted(glob.glob(str(WIKI_SYSTEMS / "*" / "bodies" / "*" / "heightmap.png"))):
|
||||
relief = Path(hm).with_name("reliefmap.png")
|
||||
if relief.exists():
|
||||
continue # already migrated (or freshly baked)
|
||||
if not dry_run:
|
||||
Path(hm).rename(relief)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def import_body(
|
||||
body_info: dict,
|
||||
conn: sqlite3.Connection,
|
||||
force: bool,
|
||||
dry_run: bool,
|
||||
verbose: bool,
|
||||
) -> dict:
|
||||
"""Import heightmap BLOB for one body.
|
||||
|
||||
Returns a dict with:
|
||||
status: 'imported' | 'skipped' | 'gas_giant' | 'error'
|
||||
message: detail (on error or skip)
|
||||
"""
|
||||
body_id = body_info["body_id"]
|
||||
terrain_ref = body_info["terrain_reference"]
|
||||
|
||||
# Incremental check — skip if already imported
|
||||
if not force:
|
||||
existing = conn.execute(
|
||||
"SELECT 1 FROM atlas_body_heightmaps WHERE body_id = ?", (body_id,)
|
||||
).fetchone()
|
||||
if existing:
|
||||
return {"status": "skipped", "message": "already imported"}
|
||||
|
||||
body_dir = REPO_ROOT / Path(terrain_ref).parent
|
||||
if not body_dir.exists():
|
||||
return {"status": "error", "message": f"body_dir not found: {body_dir}"}
|
||||
|
||||
bd = load_body_def(body_dir)
|
||||
if not bd:
|
||||
return {"status": "error", "message": f"no body definition found in {body_dir}"}
|
||||
|
||||
def bake_body(bd: dict, body_dir: Path, dry_run: bool) -> dict:
|
||||
"""Simulate one body at 1024×512 and write reliefmap.png + 16-bit heightmap.png."""
|
||||
try:
|
||||
terrain = simulate(bd)
|
||||
except Exception as exc:
|
||||
return {"status": "error", "message": f"simulate() failed: {exc}"}
|
||||
|
||||
if not terrain:
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"status": "error", "message": f"simulate failed: {exc}"}
|
||||
if not terrain or "elevation" not in terrain:
|
||||
return {"status": "gas_giant"}
|
||||
|
||||
elevation = terrain.get("elevation")
|
||||
if elevation is None:
|
||||
return {"status": "error", "message": "terrain dict missing 'elevation' key"}
|
||||
|
||||
elevation = np.clip(np.asarray(terrain["elevation"], dtype=np.float32), 0.0, 1.0)
|
||||
sea_level = float(terrain.get("sea_level", 0.0))
|
||||
blob = _elevation_to_blob(elevation)
|
||||
|
||||
if verbose:
|
||||
land_pct = float(np.mean(elevation >= sea_level)) * 100
|
||||
print(f" {body_id}: {GRID_W}x{GRID_H} grid, sea_level={sea_level:.3f}, "
|
||||
f"land={land_pct:.1f}%, blob={len(blob)} bytes")
|
||||
|
||||
if not dry_run:
|
||||
conn.execute(
|
||||
"""INSERT INTO atlas_body_heightmaps
|
||||
(body_id, width, height, data, sea_level, imported_at)
|
||||
VALUES (?, ?, ?, ?, ?, datetime('now'))
|
||||
ON CONFLICT(body_id) DO UPDATE SET
|
||||
width = excluded.width,
|
||||
height = excluded.height,
|
||||
data = excluded.data,
|
||||
sea_level = excluded.sea_level,
|
||||
imported_at = excluded.imported_at""",
|
||||
(body_id, GRID_W, GRID_H, blob, sea_level),
|
||||
)
|
||||
# Clean color relief (no painted features — cascade/Atlas overlay those).
|
||||
relief = render_heightmap(bd, terrain, render_mode="cartographic", chrome=False)
|
||||
relief.save(body_dir / "reliefmap.png")
|
||||
# Canonical 16-bit grayscale elevation; sea_level rides in a tEXt chunk.
|
||||
u16 = (elevation * 65535.0).astype(np.uint16)
|
||||
meta = PngInfo()
|
||||
meta.add_text("sea_level", f"{sea_level:.6f}")
|
||||
Image.fromarray(u16).save(body_dir / "heightmap.png", pnginfo=meta)
|
||||
|
||||
return {"status": "imported"}
|
||||
return {"status": "baked", "shape": elevation.shape, "sea_level": sea_level}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Import terrain elevation BLOBs into atlas_body_heightmaps (#906)"
|
||||
)
|
||||
parser.add_argument("--db", default=str(DB_PATH), help="Path to systems.db")
|
||||
parser.add_argument("--body", help="Process only this body_id")
|
||||
parser.add_argument("--force", action="store_true",
|
||||
help="Re-import even if a row already exists")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="Simulate without writing to DB")
|
||||
parser.add_argument("--verbose", action="store_true",
|
||||
help="Print per-body detail")
|
||||
args = parser.parse_args()
|
||||
ap = argparse.ArgumentParser(description="Bake canonical heightmap/reliefmap assets (#963)")
|
||||
ap.add_argument("--db", default=str(DB_PATH))
|
||||
ap.add_argument("--body", help="Bake only this body_id")
|
||||
ap.add_argument("--limit", type=int, help="Bake at most N bodies (smoke test)")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--skip-rename", action="store_true", help="Skip the legacy rename pass")
|
||||
args = ap.parse_args()
|
||||
|
||||
db_path = Path(args.db)
|
||||
if not db_path.exists():
|
||||
print(f"error: {db_path} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\n Heightmap BLOB Import (#906)")
|
||||
print(f" DB: {db_path}")
|
||||
print("\n Heightmap bake (#963, D-202 amended)")
|
||||
if args.dry_run:
|
||||
print(f" Mode: DRY RUN (no DB writes)")
|
||||
if args.force:
|
||||
print(f" Force: enabled (will overwrite existing rows)")
|
||||
print()
|
||||
print(" Mode: DRY RUN (no files written)")
|
||||
|
||||
# 1. Universal rename of the legacy color heightmap.png → reliefmap.png.
|
||||
if not args.skip_rename:
|
||||
n_renamed = rename_legacy_heightmaps(args.dry_run)
|
||||
print(f" Renamed legacy heightmap.png → reliefmap.png: {n_renamed} bodies")
|
||||
|
||||
# 2. Bake non-Sol inhabited bodies.
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
ensure_atlas_schema(conn)
|
||||
|
||||
bodies = query_inhabited_bodies(conn)
|
||||
if args.body:
|
||||
bodies = [b for b in bodies if b["body_id"] == args.body]
|
||||
if not bodies:
|
||||
print(f"error: body '{args.body}' not found or has no terrain_reference",
|
||||
file=sys.stderr)
|
||||
conn.close()
|
||||
sys.exit(1)
|
||||
|
||||
print(f" {len(bodies)} inhabited bodies with terrain_reference\n")
|
||||
|
||||
t_total = time.time()
|
||||
n_imported = 0
|
||||
n_skipped = 0
|
||||
n_gas = 0
|
||||
n_errors = 0
|
||||
|
||||
for i, body_info in enumerate(bodies):
|
||||
body_id = body_info["body_id"]
|
||||
t0 = time.time()
|
||||
|
||||
result = import_body(body_info, conn, args.force, args.dry_run, args.verbose)
|
||||
elapsed = time.time() - t0
|
||||
status = result["status"]
|
||||
|
||||
if status == "imported":
|
||||
n_imported += 1
|
||||
print(f" [{i+1}/{len(bodies)}] {body_id:20s} imported ({elapsed:.1f}s)")
|
||||
elif status == "skipped":
|
||||
n_skipped += 1
|
||||
if args.verbose:
|
||||
print(f" [{i+1}/{len(bodies)}] {body_id:20s} skipped (already imported)")
|
||||
elif status == "gas_giant":
|
||||
n_gas += 1
|
||||
if args.verbose:
|
||||
print(f" [{i+1}/{len(bodies)}] {body_id:20s} gas giant — no surface")
|
||||
elif status == "error":
|
||||
n_errors += 1
|
||||
print(f" [{i+1}/{len(bodies)}] {body_id:20s} ERROR: {result.get('message', '')}")
|
||||
|
||||
if not args.dry_run:
|
||||
conn.commit()
|
||||
|
||||
rows = conn.execute(
|
||||
"SELECT body_id, terrain_reference FROM bodies "
|
||||
"WHERE inhabited=1 AND terrain_reference IS NOT NULL AND population>0 "
|
||||
"AND system_id != 'GJ 0' ORDER BY body_id"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
if args.body:
|
||||
rows = [r for r in rows if r[0] == args.body]
|
||||
if args.limit:
|
||||
rows = rows[: args.limit]
|
||||
|
||||
elapsed_total = time.time() - t_total
|
||||
print(f"\n Done in {elapsed_total:.1f}s")
|
||||
print(f" imported={n_imported} skipped={n_skipped} "
|
||||
f"gas_giant={n_gas} errors={n_errors}")
|
||||
print(f" Baking {len(rows)} non-Sol inhabited bodies\n")
|
||||
|
||||
if n_errors > 0:
|
||||
print(f"\n {n_errors} error(s) — check output above", file=sys.stderr)
|
||||
parsed: dict[str, list] = {}
|
||||
t0 = time.time()
|
||||
n_baked = n_gas = n_err = 0
|
||||
for i, (body_id, tref) in enumerate(rows):
|
||||
body_dir = REPO_ROOT / Path(tref).parent
|
||||
sys_index = str(REPO_ROOT / Path(tref.split("/bodies/")[0]) / "index.md")
|
||||
try:
|
||||
defs = parsed.get(sys_index) or parse_system(sys_index)
|
||||
parsed[sys_index] = defs
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR parse_system: {exc}")
|
||||
n_err += 1
|
||||
continue
|
||||
bd = next((d for d in defs if d.get("id") == body_id), None)
|
||||
if bd is None:
|
||||
print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR: not in system defs")
|
||||
n_err += 1
|
||||
continue
|
||||
ts = time.time()
|
||||
res = bake_body(bd, body_dir, args.dry_run)
|
||||
dt = time.time() - ts
|
||||
if res["status"] == "baked":
|
||||
n_baked += 1
|
||||
if (i + 1) % 25 == 0 or args.limit:
|
||||
print(f" [{i+1}/{len(rows)}] {body_id:18s} baked {res['shape']} "
|
||||
f"sea={res['sea_level']:.3f} ({dt:.1f}s)")
|
||||
elif res["status"] == "gas_giant":
|
||||
n_gas += 1
|
||||
else:
|
||||
n_err += 1
|
||||
print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR: {res.get('message','')}")
|
||||
|
||||
print(f"\n Done in {time.time()-t0:.0f}s — baked={n_baked} gas_giant={n_gas} errors={n_err}")
|
||||
print(" Stage: git add wiki/star-systems (reliefmap.png renames + heightmap.png)")
|
||||
if n_err:
|
||||
sys.exit(0) # per-body errors are non-fatal; reported above
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user