feat(tooling): rewrite import_heightmaps as the 16-bit heightmap + relief bake (#963)
Replaces the dead atlas_body_heightmaps DB-BLOB importer (Tyre review B2 — it asserted 256×512 and wrote a dropped table) with the canonical asset bake: - one-time rename of the legacy color heightmap.png → reliefmap.png for ALL bodies (incl. Sol — display-file rename only, no re-sim); - for each non-Sol inhabited body: simulate() at 1024×512 → clean reliefmap.png (render_heightmap, no painted features) + 16-bit grayscale heightmap.png with sea_level in a tEXt chunk; - no systems.db writes; uses parse_system for proper body defs; run via uv. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -1,218 +1,168 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
import_heightmaps.py — Import terrain elevation grids into atlas_body_heightmaps.
|
import_heightmaps.py — bake per-body canonical elevation assets (D-202 amended, #963).
|
||||||
|
|
||||||
For each inhabited body with a terrain_reference, simulates the terrain via
|
This is the build-time elevation producer. For each NON-Sol inhabited body it
|
||||||
planet_simulation.simulate() and stores the float32 LE elevation BLOB plus
|
runs planet_simulation.simulate() at the canonical 1024×512 grid and writes two
|
||||||
sea_level metadata in atlas_body_heightmaps (#906, D-202).
|
files into the body's wiki directory:
|
||||||
|
|
||||||
The BLOB format matches the Rust loader spec (D-202):
|
reliefmap.png — clean color hypsometric render (display / Atlas), 1024×512
|
||||||
- float32 little-endian, row-major
|
heightmap.png — 16-bit GRAYSCALE elevation, the canonical cascade input,
|
||||||
- width × height values, each in [0.0, 1.0]
|
with sea_level stored in a PNG tEXt chunk
|
||||||
- width = GRID_W (512), height = GRID_H (256)
|
|
||||||
|
|
||||||
Incremental: bodies that already have a row in atlas_body_heightmaps are
|
It also performs the one-time rename of the legacy color `heightmap.png`
|
||||||
skipped unless --force is passed.
|
(which was a relief render) → `reliefmap.png` across ALL bodies, including Sol
|
||||||
|
(GJ-0) — Sol's terrain is real-geography (sol_import) and is NOT re-simulated,
|
||||||
|
only its display file is renamed.
|
||||||
|
|
||||||
Usage:
|
No systems.db writes: the PNG is the store (the atlas_body_heightmaps BLOB
|
||||||
tooling/planet-gen/import_heightmaps.py
|
table is dropped). Run with uv (numpy/scipy/Pillow):
|
||||||
tooling/planet-gen/import_heightmaps.py --body GJ380c
|
|
||||||
tooling/planet-gen/import_heightmaps.py --force
|
|
||||||
tooling/planet-gen/import_heightmaps.py --dry-run
|
|
||||||
|
|
||||||
Exit codes:
|
uv run python tooling/planet-gen/import_heightmaps.py # full bake
|
||||||
0 completed (possibly with skipped or errored bodies)
|
uv run python tooling/planet-gen/import_heightmaps.py --limit 3 # smoke test
|
||||||
1 fatal error (missing DB, schema error)
|
uv run python tooling/planet-gen/import_heightmaps.py --dry-run
|
||||||
|
|
||||||
|
Exit codes: 0 = completed (possibly with per-body errors), 1 = fatal.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import glob
|
||||||
|
import sqlite3
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from PIL import Image
|
||||||
|
from PIL.PngImagePlugin import PngInfo
|
||||||
|
|
||||||
TOOLING_DIR = Path(__file__).resolve().parent
|
TOOLING_DIR = Path(__file__).resolve().parent
|
||||||
REPO_ROOT = (TOOLING_DIR / ".." / "..").resolve()
|
REPO_ROOT = (TOOLING_DIR / ".." / "..").resolve()
|
||||||
|
sys.path.insert(0, str(TOOLING_DIR))
|
||||||
|
|
||||||
_venv_python = REPO_ROOT / ".venv" / "bin" / "python"
|
from body_definition_parser import parse_system # noqa: E402
|
||||||
if _venv_python.exists() and Path(sys.executable).resolve() != _venv_python.resolve():
|
from planet_simulation import simulate # noqa: E402
|
||||||
import os
|
from render_heightmap import render_heightmap # noqa: E402
|
||||||
os.execv(str(_venv_python), [str(_venv_python)] + sys.argv)
|
|
||||||
|
|
||||||
import numpy as np
|
DB_PATH = REPO_ROOT / "server" / "data" / "systems.db"
|
||||||
import sqlite3
|
WIKI_SYSTEMS = REPO_ROOT / "wiki" / "star-systems"
|
||||||
|
|
||||||
from atlas_common import (
|
|
||||||
GRID_W,
|
|
||||||
GRID_H,
|
|
||||||
DB_PATH,
|
|
||||||
ensure_atlas_schema,
|
|
||||||
load_body_def,
|
|
||||||
query_inhabited_bodies,
|
|
||||||
)
|
|
||||||
from planet_simulation import simulate
|
|
||||||
|
|
||||||
|
|
||||||
def _elevation_to_blob(elevation: np.ndarray) -> bytes:
|
def rename_legacy_heightmaps(dry_run: bool) -> int:
|
||||||
"""Convert a float32 elevation grid to a little-endian BLOB."""
|
"""One-time: legacy color `heightmap.png` (a relief render) → `reliefmap.png`,
|
||||||
arr = elevation.astype("<f4") # float32 LE, explicit
|
for every body. Bodies the bake re-simulates get their `reliefmap.png`
|
||||||
assert arr.shape == (GRID_H, GRID_W), (
|
overwritten with a fresh 1024 render afterwards. Idempotent: skips bodies
|
||||||
f"elevation shape {arr.shape} does not match expected ({GRID_H}, {GRID_W})"
|
that already have `reliefmap.png`."""
|
||||||
)
|
n = 0
|
||||||
return arr.tobytes()
|
for hm in sorted(glob.glob(str(WIKI_SYSTEMS / "*" / "bodies" / "*" / "heightmap.png"))):
|
||||||
|
relief = Path(hm).with_name("reliefmap.png")
|
||||||
|
if relief.exists():
|
||||||
|
continue # already migrated (or freshly baked)
|
||||||
|
if not dry_run:
|
||||||
|
Path(hm).rename(relief)
|
||||||
|
n += 1
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
def import_body(
|
def bake_body(bd: dict, body_dir: Path, dry_run: bool) -> dict:
|
||||||
body_info: dict,
|
"""Simulate one body at 1024×512 and write reliefmap.png + 16-bit heightmap.png."""
|
||||||
conn: sqlite3.Connection,
|
|
||||||
force: bool,
|
|
||||||
dry_run: bool,
|
|
||||||
verbose: bool,
|
|
||||||
) -> dict:
|
|
||||||
"""Import heightmap BLOB for one body.
|
|
||||||
|
|
||||||
Returns a dict with:
|
|
||||||
status: 'imported' | 'skipped' | 'gas_giant' | 'error'
|
|
||||||
message: detail (on error or skip)
|
|
||||||
"""
|
|
||||||
body_id = body_info["body_id"]
|
|
||||||
terrain_ref = body_info["terrain_reference"]
|
|
||||||
|
|
||||||
# Incremental check — skip if already imported
|
|
||||||
if not force:
|
|
||||||
existing = conn.execute(
|
|
||||||
"SELECT 1 FROM atlas_body_heightmaps WHERE body_id = ?", (body_id,)
|
|
||||||
).fetchone()
|
|
||||||
if existing:
|
|
||||||
return {"status": "skipped", "message": "already imported"}
|
|
||||||
|
|
||||||
body_dir = REPO_ROOT / Path(terrain_ref).parent
|
|
||||||
if not body_dir.exists():
|
|
||||||
return {"status": "error", "message": f"body_dir not found: {body_dir}"}
|
|
||||||
|
|
||||||
bd = load_body_def(body_dir)
|
|
||||||
if not bd:
|
|
||||||
return {"status": "error", "message": f"no body definition found in {body_dir}"}
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
terrain = simulate(bd)
|
terrain = simulate(bd)
|
||||||
except Exception as exc:
|
except Exception as exc: # noqa: BLE001
|
||||||
return {"status": "error", "message": f"simulate() failed: {exc}"}
|
return {"status": "error", "message": f"simulate failed: {exc}"}
|
||||||
|
if not terrain or "elevation" not in terrain:
|
||||||
if not terrain:
|
|
||||||
return {"status": "gas_giant"}
|
return {"status": "gas_giant"}
|
||||||
|
|
||||||
elevation = terrain.get("elevation")
|
elevation = np.clip(np.asarray(terrain["elevation"], dtype=np.float32), 0.0, 1.0)
|
||||||
if elevation is None:
|
|
||||||
return {"status": "error", "message": "terrain dict missing 'elevation' key"}
|
|
||||||
|
|
||||||
sea_level = float(terrain.get("sea_level", 0.0))
|
sea_level = float(terrain.get("sea_level", 0.0))
|
||||||
blob = _elevation_to_blob(elevation)
|
|
||||||
|
|
||||||
if verbose:
|
|
||||||
land_pct = float(np.mean(elevation >= sea_level)) * 100
|
|
||||||
print(f" {body_id}: {GRID_W}x{GRID_H} grid, sea_level={sea_level:.3f}, "
|
|
||||||
f"land={land_pct:.1f}%, blob={len(blob)} bytes")
|
|
||||||
|
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
conn.execute(
|
# Clean color relief (no painted features — cascade/Atlas overlay those).
|
||||||
"""INSERT INTO atlas_body_heightmaps
|
relief = render_heightmap(bd, terrain, render_mode="cartographic", chrome=False)
|
||||||
(body_id, width, height, data, sea_level, imported_at)
|
relief.save(body_dir / "reliefmap.png")
|
||||||
VALUES (?, ?, ?, ?, ?, datetime('now'))
|
# Canonical 16-bit grayscale elevation; sea_level rides in a tEXt chunk.
|
||||||
ON CONFLICT(body_id) DO UPDATE SET
|
u16 = (elevation * 65535.0).astype(np.uint16)
|
||||||
width = excluded.width,
|
meta = PngInfo()
|
||||||
height = excluded.height,
|
meta.add_text("sea_level", f"{sea_level:.6f}")
|
||||||
data = excluded.data,
|
Image.fromarray(u16).save(body_dir / "heightmap.png", pnginfo=meta)
|
||||||
sea_level = excluded.sea_level,
|
|
||||||
imported_at = excluded.imported_at""",
|
|
||||||
(body_id, GRID_W, GRID_H, blob, sea_level),
|
|
||||||
)
|
|
||||||
|
|
||||||
return {"status": "imported"}
|
return {"status": "baked", "shape": elevation.shape, "sea_level": sea_level}
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(
|
ap = argparse.ArgumentParser(description="Bake canonical heightmap/reliefmap assets (#963)")
|
||||||
description="Import terrain elevation BLOBs into atlas_body_heightmaps (#906)"
|
ap.add_argument("--db", default=str(DB_PATH))
|
||||||
)
|
ap.add_argument("--body", help="Bake only this body_id")
|
||||||
parser.add_argument("--db", default=str(DB_PATH), help="Path to systems.db")
|
ap.add_argument("--limit", type=int, help="Bake at most N bodies (smoke test)")
|
||||||
parser.add_argument("--body", help="Process only this body_id")
|
ap.add_argument("--dry-run", action="store_true")
|
||||||
parser.add_argument("--force", action="store_true",
|
ap.add_argument("--skip-rename", action="store_true", help="Skip the legacy rename pass")
|
||||||
help="Re-import even if a row already exists")
|
args = ap.parse_args()
|
||||||
parser.add_argument("--dry-run", action="store_true",
|
|
||||||
help="Simulate without writing to DB")
|
|
||||||
parser.add_argument("--verbose", action="store_true",
|
|
||||||
help="Print per-body detail")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
db_path = Path(args.db)
|
db_path = Path(args.db)
|
||||||
if not db_path.exists():
|
if not db_path.exists():
|
||||||
print(f"error: {db_path} not found", file=sys.stderr)
|
print(f"error: {db_path} not found", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
print(f"\n Heightmap BLOB Import (#906)")
|
print("\n Heightmap bake (#963, D-202 amended)")
|
||||||
print(f" DB: {db_path}")
|
|
||||||
if args.dry_run:
|
if args.dry_run:
|
||||||
print(f" Mode: DRY RUN (no DB writes)")
|
print(" Mode: DRY RUN (no files written)")
|
||||||
if args.force:
|
|
||||||
print(f" Force: enabled (will overwrite existing rows)")
|
|
||||||
print()
|
|
||||||
|
|
||||||
|
# 1. Universal rename of the legacy color heightmap.png → reliefmap.png.
|
||||||
|
if not args.skip_rename:
|
||||||
|
n_renamed = rename_legacy_heightmaps(args.dry_run)
|
||||||
|
print(f" Renamed legacy heightmap.png → reliefmap.png: {n_renamed} bodies")
|
||||||
|
|
||||||
|
# 2. Bake non-Sol inhabited bodies.
|
||||||
conn = sqlite3.connect(str(db_path))
|
conn = sqlite3.connect(str(db_path))
|
||||||
conn.execute("PRAGMA foreign_keys=ON")
|
rows = conn.execute(
|
||||||
ensure_atlas_schema(conn)
|
"SELECT body_id, terrain_reference FROM bodies "
|
||||||
|
"WHERE inhabited=1 AND terrain_reference IS NOT NULL AND population>0 "
|
||||||
bodies = query_inhabited_bodies(conn)
|
"AND system_id != 'GJ 0' ORDER BY body_id"
|
||||||
if args.body:
|
).fetchall()
|
||||||
bodies = [b for b in bodies if b["body_id"] == args.body]
|
|
||||||
if not bodies:
|
|
||||||
print(f"error: body '{args.body}' not found or has no terrain_reference",
|
|
||||||
file=sys.stderr)
|
|
||||||
conn.close()
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
print(f" {len(bodies)} inhabited bodies with terrain_reference\n")
|
|
||||||
|
|
||||||
t_total = time.time()
|
|
||||||
n_imported = 0
|
|
||||||
n_skipped = 0
|
|
||||||
n_gas = 0
|
|
||||||
n_errors = 0
|
|
||||||
|
|
||||||
for i, body_info in enumerate(bodies):
|
|
||||||
body_id = body_info["body_id"]
|
|
||||||
t0 = time.time()
|
|
||||||
|
|
||||||
result = import_body(body_info, conn, args.force, args.dry_run, args.verbose)
|
|
||||||
elapsed = time.time() - t0
|
|
||||||
status = result["status"]
|
|
||||||
|
|
||||||
if status == "imported":
|
|
||||||
n_imported += 1
|
|
||||||
print(f" [{i+1}/{len(bodies)}] {body_id:20s} imported ({elapsed:.1f}s)")
|
|
||||||
elif status == "skipped":
|
|
||||||
n_skipped += 1
|
|
||||||
if args.verbose:
|
|
||||||
print(f" [{i+1}/{len(bodies)}] {body_id:20s} skipped (already imported)")
|
|
||||||
elif status == "gas_giant":
|
|
||||||
n_gas += 1
|
|
||||||
if args.verbose:
|
|
||||||
print(f" [{i+1}/{len(bodies)}] {body_id:20s} gas giant — no surface")
|
|
||||||
elif status == "error":
|
|
||||||
n_errors += 1
|
|
||||||
print(f" [{i+1}/{len(bodies)}] {body_id:20s} ERROR: {result.get('message', '')}")
|
|
||||||
|
|
||||||
if not args.dry_run:
|
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
if args.body:
|
||||||
|
rows = [r for r in rows if r[0] == args.body]
|
||||||
|
if args.limit:
|
||||||
|
rows = rows[: args.limit]
|
||||||
|
|
||||||
elapsed_total = time.time() - t_total
|
print(f" Baking {len(rows)} non-Sol inhabited bodies\n")
|
||||||
print(f"\n Done in {elapsed_total:.1f}s")
|
|
||||||
print(f" imported={n_imported} skipped={n_skipped} "
|
|
||||||
f"gas_giant={n_gas} errors={n_errors}")
|
|
||||||
|
|
||||||
if n_errors > 0:
|
parsed: dict[str, list] = {}
|
||||||
print(f"\n {n_errors} error(s) — check output above", file=sys.stderr)
|
t0 = time.time()
|
||||||
|
n_baked = n_gas = n_err = 0
|
||||||
|
for i, (body_id, tref) in enumerate(rows):
|
||||||
|
body_dir = REPO_ROOT / Path(tref).parent
|
||||||
|
sys_index = str(REPO_ROOT / Path(tref.split("/bodies/")[0]) / "index.md")
|
||||||
|
try:
|
||||||
|
defs = parsed.get(sys_index) or parse_system(sys_index)
|
||||||
|
parsed[sys_index] = defs
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR parse_system: {exc}")
|
||||||
|
n_err += 1
|
||||||
|
continue
|
||||||
|
bd = next((d for d in defs if d.get("id") == body_id), None)
|
||||||
|
if bd is None:
|
||||||
|
print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR: not in system defs")
|
||||||
|
n_err += 1
|
||||||
|
continue
|
||||||
|
ts = time.time()
|
||||||
|
res = bake_body(bd, body_dir, args.dry_run)
|
||||||
|
dt = time.time() - ts
|
||||||
|
if res["status"] == "baked":
|
||||||
|
n_baked += 1
|
||||||
|
if (i + 1) % 25 == 0 or args.limit:
|
||||||
|
print(f" [{i+1}/{len(rows)}] {body_id:18s} baked {res['shape']} "
|
||||||
|
f"sea={res['sea_level']:.3f} ({dt:.1f}s)")
|
||||||
|
elif res["status"] == "gas_giant":
|
||||||
|
n_gas += 1
|
||||||
|
else:
|
||||||
|
n_err += 1
|
||||||
|
print(f" [{i+1}/{len(rows)}] {body_id:18s} ERROR: {res.get('message','')}")
|
||||||
|
|
||||||
|
print(f"\n Done in {time.time()-t0:.0f}s — baked={n_baked} gas_giant={n_gas} errors={n_err}")
|
||||||
|
print(" Stage: git add wiki/star-systems (reliefmap.png renames + heightmap.png)")
|
||||||
|
if n_err:
|
||||||
|
sys.exit(0) # per-body errors are non-fatal; reported above
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Reference in New Issue
Block a user