From 3b2cd6914eeaa5e6b99bdf89993e1e71dfcc80ce Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Sat, 23 May 2026 08:22:50 +0200 Subject: [PATCH] feat(tooling): rewrite import_heightmaps as the 16-bit heightmap + relief bake (#963) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the dead atlas_body_heightmaps DB-BLOB importer (Tyre review B2 — it asserted 256×512 and wrote a dropped table) with the canonical asset bake: - one-time rename of the legacy color heightmap.png → reliefmap.png for ALL bodies (incl. Sol — display-file rename only, no re-sim); - for each non-Sol inhabited body: simulate() at 1024×512 → clean reliefmap.png (render_heightmap, no painted features) + 16-bit grayscale heightmap.png with sea_level in a tEXt chunk; - no systems.db writes; uses parse_system for proper body defs; run via uv. Co-Authored-By: Claude Opus 4.7 (1M context) --- tooling/planet-gen/import_heightmaps.py | 290 ++++++++++-------------- 1 file changed, 120 insertions(+), 170 deletions(-) diff --git a/tooling/planet-gen/import_heightmaps.py b/tooling/planet-gen/import_heightmaps.py index a6829b1a8..b7658387c 100644 --- a/tooling/planet-gen/import_heightmaps.py +++ b/tooling/planet-gen/import_heightmaps.py @@ -1,218 +1,168 @@ #!/usr/bin/env python3 """ -import_heightmaps.py — Import terrain elevation grids into atlas_body_heightmaps. +import_heightmaps.py — bake per-body canonical elevation assets (D-202 amended, #963). -For each inhabited body with a terrain_reference, simulates the terrain via -planet_simulation.simulate() and stores the float32 LE elevation BLOB plus -sea_level metadata in atlas_body_heightmaps (#906, D-202). +This is the build-time elevation producer. For each NON-Sol inhabited body it +runs planet_simulation.simulate() at the canonical 1024×512 grid and writes two +files into the body's wiki directory: -The BLOB format matches the Rust loader spec (D-202): - - float32 little-endian, row-major - - width × height values, each in [0.0, 1.0] - - width = GRID_W (512), height = GRID_H (256) + reliefmap.png — clean color hypsometric render (display / Atlas), 1024×512 + heightmap.png — 16-bit GRAYSCALE elevation, the canonical cascade input, + with sea_level stored in a PNG tEXt chunk -Incremental: bodies that already have a row in atlas_body_heightmaps are -skipped unless --force is passed. +It also performs the one-time rename of the legacy color `heightmap.png` +(which was a relief render) → `reliefmap.png` across ALL bodies, including Sol +(GJ-0) — Sol's terrain is real-geography (sol_import) and is NOT re-simulated, +only its display file is renamed. -Usage: - tooling/planet-gen/import_heightmaps.py - tooling/planet-gen/import_heightmaps.py --body GJ380c - tooling/planet-gen/import_heightmaps.py --force - tooling/planet-gen/import_heightmaps.py --dry-run +No systems.db writes: the PNG is the store (the atlas_body_heightmaps BLOB +table is dropped). Run with uv (numpy/scipy/Pillow): -Exit codes: - 0 completed (possibly with skipped or errored bodies) - 1 fatal error (missing DB, schema error) + uv run python tooling/planet-gen/import_heightmaps.py # full bake + uv run python tooling/planet-gen/import_heightmaps.py --limit 3 # smoke test + uv run python tooling/planet-gen/import_heightmaps.py --dry-run + +Exit codes: 0 = completed (possibly with per-body errors), 1 = fatal. """ import argparse +import glob +import sqlite3 import sys import time from pathlib import Path +import numpy as np +from PIL import Image +from PIL.PngImagePlugin import PngInfo + TOOLING_DIR = Path(__file__).resolve().parent REPO_ROOT = (TOOLING_DIR / ".." / "..").resolve() +sys.path.insert(0, str(TOOLING_DIR)) -_venv_python = REPO_ROOT / ".venv" / "bin" / "python" -if _venv_python.exists() and Path(sys.executable).resolve() != _venv_python.resolve(): - import os - os.execv(str(_venv_python), [str(_venv_python)] + sys.argv) +from body_definition_parser import parse_system # noqa: E402 +from planet_simulation import simulate # noqa: E402 +from render_heightmap import render_heightmap # noqa: E402 -import numpy as np -import sqlite3 - -from atlas_common import ( - GRID_W, - GRID_H, - DB_PATH, - ensure_atlas_schema, - load_body_def, - query_inhabited_bodies, -) -from planet_simulation import simulate +DB_PATH = REPO_ROOT / "server" / "data" / "systems.db" +WIKI_SYSTEMS = REPO_ROOT / "wiki" / "star-systems" -def _elevation_to_blob(elevation: np.ndarray) -> bytes: - """Convert a float32 elevation grid to a little-endian BLOB.""" - arr = elevation.astype(" int: + """One-time: legacy color `heightmap.png` (a relief render) → `reliefmap.png`, + for every body. Bodies the bake re-simulates get their `reliefmap.png` + overwritten with a fresh 1024 render afterwards. Idempotent: skips bodies + that already have `reliefmap.png`.""" + n = 0 + for hm in sorted(glob.glob(str(WIKI_SYSTEMS / "*" / "bodies" / "*" / "heightmap.png"))): + relief = Path(hm).with_name("reliefmap.png") + if relief.exists(): + continue # already migrated (or freshly baked) + if not dry_run: + Path(hm).rename(relief) + n += 1 + return n -def import_body( - body_info: dict, - conn: sqlite3.Connection, - force: bool, - dry_run: bool, - verbose: bool, -) -> dict: - """Import heightmap BLOB for one body. - - Returns a dict with: - status: 'imported' | 'skipped' | 'gas_giant' | 'error' - message: detail (on error or skip) - """ - body_id = body_info["body_id"] - terrain_ref = body_info["terrain_reference"] - - # Incremental check — skip if already imported - if not force: - existing = conn.execute( - "SELECT 1 FROM atlas_body_heightmaps WHERE body_id = ?", (body_id,) - ).fetchone() - if existing: - return {"status": "skipped", "message": "already imported"} - - body_dir = REPO_ROOT / Path(terrain_ref).parent - if not body_dir.exists(): - return {"status": "error", "message": f"body_dir not found: {body_dir}"} - - bd = load_body_def(body_dir) - if not bd: - return {"status": "error", "message": f"no body definition found in {body_dir}"} - +def bake_body(bd: dict, body_dir: Path, dry_run: bool) -> dict: + """Simulate one body at 1024×512 and write reliefmap.png + 16-bit heightmap.png.""" try: terrain = simulate(bd) - except Exception as exc: - return {"status": "error", "message": f"simulate() failed: {exc}"} - - if not terrain: + except Exception as exc: # noqa: BLE001 + return {"status": "error", "message": f"simulate failed: {exc}"} + if not terrain or "elevation" not in terrain: return {"status": "gas_giant"} - elevation = terrain.get("elevation") - if elevation is None: - return {"status": "error", "message": "terrain dict missing 'elevation' key"} - + elevation = np.clip(np.asarray(terrain["elevation"], dtype=np.float32), 0.0, 1.0) sea_level = float(terrain.get("sea_level", 0.0)) - blob = _elevation_to_blob(elevation) - - if verbose: - land_pct = float(np.mean(elevation >= sea_level)) * 100 - print(f" {body_id}: {GRID_W}x{GRID_H} grid, sea_level={sea_level:.3f}, " - f"land={land_pct:.1f}%, blob={len(blob)} bytes") if not dry_run: - conn.execute( - """INSERT INTO atlas_body_heightmaps - (body_id, width, height, data, sea_level, imported_at) - VALUES (?, ?, ?, ?, ?, datetime('now')) - ON CONFLICT(body_id) DO UPDATE SET - width = excluded.width, - height = excluded.height, - data = excluded.data, - sea_level = excluded.sea_level, - imported_at = excluded.imported_at""", - (body_id, GRID_W, GRID_H, blob, sea_level), - ) + # Clean color relief (no painted features — cascade/Atlas overlay those). + relief = render_heightmap(bd, terrain, render_mode="cartographic", chrome=False) + relief.save(body_dir / "reliefmap.png") + # Canonical 16-bit grayscale elevation; sea_level rides in a tEXt chunk. + u16 = (elevation * 65535.0).astype(np.uint16) + meta = PngInfo() + meta.add_text("sea_level", f"{sea_level:.6f}") + Image.fromarray(u16).save(body_dir / "heightmap.png", pnginfo=meta) - return {"status": "imported"} + return {"status": "baked", "shape": elevation.shape, "sea_level": sea_level} def main() -> None: - parser = argparse.ArgumentParser( - description="Import terrain elevation BLOBs into atlas_body_heightmaps (#906)" - ) - parser.add_argument("--db", default=str(DB_PATH), help="Path to systems.db") - parser.add_argument("--body", help="Process only this body_id") - parser.add_argument("--force", action="store_true", - help="Re-import even if a row already exists") - parser.add_argument("--dry-run", action="store_true", - help="Simulate without writing to DB") - parser.add_argument("--verbose", action="store_true", - help="Print per-body detail") - args = parser.parse_args() + ap = argparse.ArgumentParser(description="Bake canonical heightmap/reliefmap assets (#963)") + ap.add_argument("--db", default=str(DB_PATH)) + ap.add_argument("--body", help="Bake only this body_id") + ap.add_argument("--limit", type=int, help="Bake at most N bodies (smoke test)") + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--skip-rename", action="store_true", help="Skip the legacy rename pass") + args = ap.parse_args() db_path = Path(args.db) if not db_path.exists(): print(f"error: {db_path} not found", file=sys.stderr) sys.exit(1) - print(f"\n Heightmap BLOB Import (#906)") - print(f" DB: {db_path}") + print("\n Heightmap bake (#963, D-202 amended)") if args.dry_run: - print(f" Mode: DRY RUN (no DB writes)") - if args.force: - print(f" Force: enabled (will overwrite existing rows)") - print() + print(" Mode: DRY RUN (no files written)") + # 1. Universal rename of the legacy color heightmap.png → reliefmap.png. + if not args.skip_rename: + n_renamed = rename_legacy_heightmaps(args.dry_run) + print(f" Renamed legacy heightmap.png → reliefmap.png: {n_renamed} bodies") + + # 2. Bake non-Sol inhabited bodies. conn = sqlite3.connect(str(db_path)) - conn.execute("PRAGMA foreign_keys=ON") - ensure_atlas_schema(conn) - - bodies = query_inhabited_bodies(conn) - if args.body: - bodies = [b for b in bodies if b["body_id"] == args.body] - if not bodies: - print(f"error: body '{args.body}' not found or has no terrain_reference", - file=sys.stderr) - conn.close() - sys.exit(1) - - print(f" {len(bodies)} inhabited bodies with terrain_reference\n") - - t_total = time.time() - n_imported = 0 - n_skipped = 0 - n_gas = 0 - n_errors = 0 - - for i, body_info in enumerate(bodies): - body_id = body_info["body_id"] - t0 = time.time() - - result = import_body(body_info, conn, args.force, args.dry_run, args.verbose) - elapsed = time.time() - t0 - status = result["status"] - - if status == "imported": - n_imported += 1 - print(f" [{i+1}/{len(bodies)}] {body_id:20s} imported ({elapsed:.1f}s)") - elif status == "skipped": - n_skipped += 1 - if args.verbose: - print(f" [{i+1}/{len(bodies)}] {body_id:20s} skipped (already imported)") - elif status == "gas_giant": - n_gas += 1 - if args.verbose: - print(f" [{i+1}/{len(bodies)}] {body_id:20s} gas giant — no surface") - elif status == "error": - n_errors += 1 - print(f" [{i+1}/{len(bodies)}] {body_id:20s} ERROR: {result.get('message', '')}") - - if not args.dry_run: - conn.commit() - + rows = conn.execute( + "SELECT body_id, terrain_reference FROM bodies " + "WHERE inhabited=1 AND terrain_reference IS NOT NULL AND population>0 " + "AND system_id != 'GJ 0' ORDER BY body_id" + ).fetchall() conn.close() + if args.body: + rows = [r for r in rows if r[0] == args.body] + if args.limit: + rows = rows[: args.limit] - elapsed_total = time.time() - t_total - print(f"\n Done in {elapsed_total:.1f}s") - print(f" imported={n_imported} skipped={n_skipped} " - f"gas_giant={n_gas} errors={n_errors}") + print(f" Baking {len(rows)} non-Sol inhabited bodies\n") - if n_errors > 0: - print(f"\n {n_errors} error(s) — check output above", file=sys.stderr) + parsed: dict[str, list] = {} + t0 = time.time() + n_baked = n_gas = n_err = 0 + for i, (body_id, tref) in enumerate(rows): + body_dir = REPO_ROOT / Path(tref).parent + sys_index = str(REPO_ROOT / Path(tref.split("/bodies/")[0]) / "index.md") + try: + defs = parsed.get(sys_index) or parse_system(sys_index) + parsed[sys_index] = defs + except Exception as exc: # noqa: BLE001 + print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR parse_system: {exc}") + n_err += 1 + continue + bd = next((d for d in defs if d.get("id") == body_id), None) + if bd is None: + print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR: not in system defs") + n_err += 1 + continue + ts = time.time() + res = bake_body(bd, body_dir, args.dry_run) + dt = time.time() - ts + if res["status"] == "baked": + n_baked += 1 + if (i + 1) % 25 == 0 or args.limit: + print(f" [{i+1}/{len(rows)}] {body_id:18s} baked {res['shape']} " + f"sea={res['sea_level']:.3f} ({dt:.1f}s)") + elif res["status"] == "gas_giant": + n_gas += 1 + else: + n_err += 1 + print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR: {res.get('message','')}") + + print(f"\n Done in {time.time()-t0:.0f}s — baked={n_baked} gas_giant={n_gas} errors={n_err}") + print(" Stage: git add wiki/star-systems (reliefmap.png renames + heightmap.png)") + if n_err: + sys.exit(0) # per-body errors are non-fatal; reported above if __name__ == "__main__":