feat(simulation): load canonical 16-bit grayscale heightmap.png + downsample (D-202 #963)

Replace the atlas_body_heightmaps DB-BLOB loader with a per-body file
loader (D-202 amendment): heightmap.rs reads the 16-bit grayscale
heightmap.png (via the new png dep) from the body's terrain_reference
path, normalizes to f32 [0,1], and rejects RGB so a reliefmap can't be
misread as elevation. Adds BodyHeightmap::downsample (box-average,
deterministic) so Layer 1 drops the high-res stored heightmap to the
512x256 working resolution. sea_level becomes body metadata carried
alongside, not in the PNG. 4 loader tests pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-23 00:35:58 +02:00
co-authored by Claude Opus 4.7
parent 98658a512d
commit 3886cea46e
3 changed files with 241 additions and 136 deletions
+77 -7
View File
@@ -2,6 +2,12 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "ahash"
version = "0.8.12"
@@ -190,7 +196,7 @@ dependencies = [
"bevy_reflect",
"bevy_tasks",
"bevy_utils",
"bitflags",
"bitflags 2.10.0",
"bumpalo",
"concurrent-queue",
"derive_more",
@@ -325,6 +331,12 @@ dependencies = [
"thread_local",
]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.10.0"
@@ -462,6 +474,15 @@ dependencies = [
"unicode-segmentation",
]
[[package]]
name = "crc32fast"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
[[package]]
name = "critical-section"
version = "1.2.0"
@@ -563,7 +584,7 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec"
dependencies = [
"bitflags",
"bitflags 2.10.0",
"block2",
"libc",
"objc2",
@@ -654,6 +675,15 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fdeflate"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
dependencies = [
"simd-adler32",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
@@ -666,6 +696,16 @@ version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "flate2"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "foldhash"
version = "0.2.0"
@@ -886,13 +926,23 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]]
name = "nix"
version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225e7cfe711e0ba79a68baeddb2982723e4235247aefce1482f2f16c27865b66"
dependencies = [
"bitflags",
"bitflags 2.10.0",
"cfg-if",
"cfg_aliases",
"libc",
@@ -946,7 +996,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
dependencies = [
"bitflags",
"bitflags 2.10.0",
]
[[package]]
@@ -1035,6 +1085,19 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "png"
version = "0.17.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526"
dependencies = [
"bitflags 1.3.2",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide",
]
[[package]]
name = "portable-atomic"
version = "1.13.1"
@@ -1175,7 +1238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
dependencies = [
"base64",
"bitflags",
"bitflags 2.10.0",
"serde",
"serde_derive",
]
@@ -1186,7 +1249,7 @@ version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"
dependencies = [
"bitflags",
"bitflags 2.10.0",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
@@ -1305,6 +1368,7 @@ dependencies = [
"crossbeam-channel",
"econ-sim",
"pathfinding",
"png",
"rand",
"rand_chacha",
"rayon",
@@ -1336,6 +1400,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "simd-adler32"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
[[package]]
name = "slab"
version = "0.4.12"
@@ -1747,7 +1817,7 @@ version = "27.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afdcf84c395990db737f2dd91628706cb31e86d72e53482320d368e52b5da5eb"
dependencies = [
"bitflags",
"bitflags 2.10.0",
"bytemuck",
"js-sys",
"log",
+1
View File
@@ -29,6 +29,7 @@ sysinfo = "0.35"
serde_json = "1"
rusqlite = { version = "0.32", features = ["bundled"] }
bytemuck = "1"
png = "0.17"
toml = "0.8"
aho-corasick = "1"
# Economics simulation — Leontief + tâtonnement + D-180 event port (#821)
+163 -129
View File
@@ -1,19 +1,26 @@
//! Heightmap BLOB loader — reads float32 LE elevation grids from systems.db.
//! Heightmap loader — reads the canonical 16-bit grayscale elevation PNG (D-202).
//!
//! Implements the Rust side of D-202. The Python pipeline stores each body's
//! elevation grid as a contiguous float32 little-endian BLOB in
//! `atlas_body_heightmaps.data`. This module loads that BLOB via `rusqlite`
//! and reinterprets the bytes into a `Vec<f32>` using `bytemuck`.
//! Per D-202 (amended #963), each body's canonical elevation is a per-body
//! **16-bit grayscale `heightmap.png`** stored next to the color `reliefmap.png`
//! in the body's wiki directory (path from `bodies.terrain_reference`). The
//! grayscale luminance is the normalized elevation; this is the single source
//! of truth — no `systems.db` BLOB. The file is high-resolution (canonically
//! 2048×1024) so the lower cascade layers (region/block/tile) have real local
//! detail; **Layer 1** (continental drainage/basins/mountain-ranges) calls
//! [`BodyHeightmap::downsample`] to `GRID_W × GRID_H` first, decoupling the
//! continental compute cost from the stored resolution.
//!
//! Values are normalized elevation in [0.0, 1.0]. `sea_level` is the fraction
//! below which terrain is underwater (0.0 = no ocean).
//!
//! Canonical grid size: 512 × 256 (GRID_W × GRID_H), row-major.
//! Elevation values are normalized to [0.0, 1.0]. `sea_level` is body metadata
//! (carried alongside, not in the PNG) below which terrain is ocean.
use std::fs::File;
use std::io::Read;
use std::path::Path;
use rusqlite::{params, Connection};
use thiserror::Error;
/// Canonical grid dimensions matching the Python pipeline (atlas_common.py).
/// Layer-1 working resolution: continental drainage/feature extraction run at
/// this size (the stored heightmap is higher-res and downsampled to it).
pub const GRID_W: u32 = 512;
pub const GRID_H: u32 = 256;
@@ -45,155 +52,182 @@ impl BodyHeightmap {
pub fn is_land(&self, row: u32, col: u32) -> bool {
self.get(row, col).is_some_and(|e| e >= self.sea_level)
}
/// Box-average downsample to `target_w × target_h` (used by Layer 1 to drop
/// the high-res stored heightmap to the continental working resolution).
/// Deterministic (f64 accumulation, fixed footprint). Returns a clone when
/// the target is not strictly smaller (no upsampling here).
pub fn downsample(&self, target_w: u32, target_h: u32) -> BodyHeightmap {
if target_w == 0 || target_h == 0 || (target_w >= self.width && target_h >= self.height) {
return self.clone();
}
let (sw, sh) = (self.width as u64, self.height as u64);
let mut out = vec![0f32; (target_w * target_h) as usize];
for ty in 0..target_h {
let sy0 = (ty as u64 * sh / target_h as u64) as u32;
let sy1 = (((ty as u64 + 1) * sh / target_h as u64).max(sy0 as u64 + 1)) as u32;
for tx in 0..target_w {
let sx0 = (tx as u64 * sw / target_w as u64) as u32;
let sx1 = (((tx as u64 + 1) * sw / target_w as u64).max(sx0 as u64 + 1)) as u32;
let mut sum = 0f64;
let mut cnt = 0u32;
for sy in sy0..sy1 {
for sx in sx0..sx1 {
sum += self.data[(sy * self.width + sx) as usize] as f64;
cnt += 1;
}
}
out[(ty * target_w + tx) as usize] = (sum / cnt as f64) as f32;
}
}
BodyHeightmap {
body_id: self.body_id.clone(),
width: target_w,
height: target_h,
data: out,
sea_level: self.sea_level,
}
}
}
#[derive(Debug, Error)]
pub enum HeightmapLoadError {
#[error("no heightmap row for body '{0}'")]
NotFound(String),
#[error("BLOB size {actual} does not match declared grid {w}×{h}×4 = {expected}")]
BlobSizeMismatch {
actual: usize,
w: u32,
h: u32,
expected: usize,
#[error("heightmap file unreadable: {0}")]
Io(#[from] std::io::Error),
#[error("PNG decode error: {0}")]
Png(#[from] png::DecodingError),
#[error("unsupported heightmap PNG: {bit_depth} {color_type} (expected 16-bit grayscale)")]
UnsupportedFormat {
bit_depth: String,
color_type: String,
},
#[error("SQLite error: {0}")]
Sql(#[from] rusqlite::Error),
}
/// Load the heightmap for `body_id` from the open `conn`.
/// Load the canonical heightmap PNG for `body_id` from `path`.
///
/// The BLOB is reinterpreted in-place via `bytemuck::cast_slice` — no copy
/// beyond the initial `Vec<u8>` read from SQLite. On little-endian hosts
/// (all current targets) this is a zero-cost reinterpret. On big-endian hosts
/// the bytes are already stored LE, so each f32 would be byte-swapped; this
/// function does not perform that swap — big-endian support is deferred.
pub fn load_heightmap(
conn: &Connection,
/// `sea_level` is body metadata (e.g. from the `bodies` table / body def), not
/// stored in the PNG.
pub fn load_heightmap_png(
path: &Path,
body_id: &str,
sea_level: f32,
) -> Result<BodyHeightmap, HeightmapLoadError> {
let result = conn.query_row(
"SELECT width, height, data, sea_level \
FROM atlas_body_heightmaps WHERE body_id = ?1",
params![body_id],
|row| {
let width: u32 = row.get(0)?;
let height: u32 = row.get(1)?;
let blob: Vec<u8> = row.get(2)?;
let sea_level: f64 = row.get(3)?;
Ok((width, height, blob, sea_level as f32))
},
);
let file = File::open(path)?;
load_heightmap_reader(file, body_id, sea_level)
}
match result {
Err(rusqlite::Error::QueryReturnedNoRows) => {
Err(HeightmapLoadError::NotFound(body_id.to_string()))
/// Decode a heightmap PNG from any reader. 16-bit grayscale is the canonical
/// format; 8-bit grayscale is accepted (coarse — viewable/test only).
pub fn load_heightmap_reader<R: Read>(
reader: R,
body_id: &str,
sea_level: f32,
) -> Result<BodyHeightmap, HeightmapLoadError> {
let mut png_reader = png::Decoder::new(reader).read_info()?;
let (width, height, bit_depth, color_type) = {
let info = png_reader.info();
(info.width, info.height, info.bit_depth, info.color_type)
};
let mut buf = vec![0u8; png_reader.output_buffer_size()];
let frame = png_reader.next_frame(&mut buf)?;
let bytes = &buf[..frame.buffer_size()];
let data: Vec<f32> = match (bit_depth, color_type) {
(png::BitDepth::Sixteen, png::ColorType::Grayscale) => bytes
.chunks_exact(2)
.map(|p| u16::from_be_bytes([p[0], p[1]]) as f32 / 65535.0)
.collect(),
(png::BitDepth::Eight, png::ColorType::Grayscale) => {
bytes.iter().map(|&b| b as f32 / 255.0).collect()
}
Err(e) => Err(HeightmapLoadError::Sql(e)),
Ok((width, height, blob, sea_level)) => {
let expected = (width * height * 4) as usize;
if blob.len() != expected {
return Err(HeightmapLoadError::BlobSizeMismatch {
actual: blob.len(),
w: width,
h: height,
expected,
});
}
// Reinterpret the LE bytes as f32 values. bytemuck::cast_slice
// is safe here: we verified the length is a multiple of 4, and
// f32 has no invalid bit patterns.
let floats: &[f32] = bytemuck::cast_slice(&blob);
let data = floats.to_vec();
Ok(BodyHeightmap {
body_id: body_id.to_string(),
width,
height,
data,
sea_level,
(bd, ct) => {
return Err(HeightmapLoadError::UnsupportedFormat {
bit_depth: format!("{bd:?}"),
color_type: format!("{ct:?}"),
})
}
}
};
Ok(BodyHeightmap {
body_id: body_id.to_string(),
width,
height,
data,
sea_level,
})
}
#[cfg(test)]
mod tests {
use super::*;
use rusqlite::Connection;
fn make_test_db() -> Connection {
let conn = Connection::open_in_memory().unwrap();
conn.execute_batch(
"CREATE TABLE atlas_body_heightmaps (
body_id TEXT PRIMARY KEY,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
data BLOB NOT NULL,
sea_level REAL NOT NULL DEFAULT 0.0,
imported_at TEXT NOT NULL DEFAULT (datetime('now'))
);",
)
.unwrap();
conn
}
fn insert_heightmap(conn: &Connection, body_id: &str, w: u32, h: u32, sea_level: f32) {
let floats: Vec<f32> = (0..(w * h)).map(|i| i as f32 / (w * h) as f32).collect();
let bytes: &[u8] = bytemuck::cast_slice(&floats);
conn.execute(
"INSERT INTO atlas_body_heightmaps (body_id, width, height, data, sea_level)
VALUES (?1, ?2, ?3, ?4, ?5)",
params![body_id, w, h, bytes, sea_level],
)
.unwrap();
/// Encode a 16-bit grayscale PNG (row-major u16 elevation) to bytes.
fn encode_gray16(w: u32, h: u32, vals: &[u16]) -> Vec<u8> {
let mut out = Vec::new();
{
let mut enc = png::Encoder::new(&mut out, w, h);
enc.set_color(png::ColorType::Grayscale);
enc.set_depth(png::BitDepth::Sixteen);
let mut writer = enc.write_header().unwrap();
// png expects big-endian 16-bit samples.
let mut be = Vec::with_capacity(vals.len() * 2);
for &v in vals {
be.extend_from_slice(&v.to_be_bytes());
}
writer.write_image_data(&be).unwrap();
}
out
}
#[test]
fn round_trip_canonical_size() {
let conn = make_test_db();
insert_heightmap(&conn, "TestBody", GRID_W, GRID_H, 0.3);
let hm = load_heightmap(&conn, "TestBody").unwrap();
assert_eq!(hm.width, GRID_W);
assert_eq!(hm.height, GRID_H);
assert_eq!(hm.data.len(), (GRID_W * GRID_H) as usize);
assert!((hm.sea_level - 0.3).abs() < 1e-6);
// First cell is 0.0, last approaches 1.0
assert_eq!(hm.data[0], 0.0);
assert!(hm.data.last().copied().unwrap() < 1.0);
fn round_trips_16bit_grayscale() {
let (w, h) = (4u32, 2u32);
// Elevations 0, 1/7, ... 7/7 across 8 cells.
let vals: Vec<u16> = (0..(w * h))
.map(|i| (i as f32 / (w * h - 1) as f32 * 65535.0) as u16)
.collect();
let png_bytes = encode_gray16(w, h, &vals);
let hm = load_heightmap_reader(png_bytes.as_slice(), "T", 0.3).unwrap();
assert_eq!((hm.width, hm.height), (w, h));
assert_eq!(hm.data.len(), 8);
assert!((hm.data[0] - 0.0).abs() < 1e-6);
assert!((hm.data[7] - 1.0).abs() < 1e-4);
assert!(hm.is_land(1, 3)); // last cell = 1.0 > sea 0.3
assert!(!hm.is_land(0, 0)); // first cell = 0.0 < sea
}
#[test]
fn get_and_is_land() {
let conn = make_test_db();
insert_heightmap(&conn, "LandBody", 4, 2, 0.5);
let hm = load_heightmap(&conn, "LandBody").unwrap();
// First cell (index 0) = 0.0 / 8 = 0.0 — below sea level
assert!(!hm.is_land(0, 0));
// Last cell (index 7) = 7.0 / 8 = 0.875 — above sea level
assert!(hm.is_land(1, 3));
// Out-of-bounds returns false
assert!(!hm.is_land(99, 99));
fn downsample_halves_and_averages() {
// 4×2 → 2×1: each output cell averages a 2×2 block.
let vals: Vec<u16> = vec![0, 0, 65535, 65535, 0, 0, 65535, 65535];
let hm = load_heightmap_reader(encode_gray16(4, 2, &vals).as_slice(), "T", 0.3).unwrap();
let ds = hm.downsample(2, 1);
assert_eq!((ds.width, ds.height), (2, 1));
assert!((ds.data[0] - 0.0).abs() < 1e-4); // left 2×2 block all 0
assert!((ds.data[1] - 1.0).abs() < 1e-4); // right 2×2 block all 1
}
#[test]
fn not_found_error() {
let conn = make_test_db();
let err = load_heightmap(&conn, "Ghost").unwrap_err();
assert!(matches!(err, HeightmapLoadError::NotFound(_)));
fn downsample_is_deterministic() {
let vals: Vec<u16> = (0..(16 * 8)).map(|i| (i * 257) as u16).collect();
let hm = load_heightmap_reader(encode_gray16(16, 8, &vals).as_slice(), "T", 0.3).unwrap();
let a = hm.downsample(4, 2);
let b = hm.downsample(4, 2);
assert_eq!(a.data, b.data);
}
#[test]
fn blob_size_mismatch_error() {
let conn = make_test_db();
// Insert a truncated BLOB
conn.execute(
"INSERT INTO atlas_body_heightmaps (body_id, width, height, data, sea_level)
VALUES ('BadBlob', 4, 4, X'DEADBEEF', 0.0)",
[],
)
.unwrap();
let err = load_heightmap(&conn, "BadBlob").unwrap_err();
assert!(matches!(err, HeightmapLoadError::BlobSizeMismatch { .. }));
fn rejects_rgb() {
// An RGB PNG (like the reliefmap) must be rejected, not misread.
let mut out = Vec::new();
{
let mut enc = png::Encoder::new(&mut out, 2, 2);
enc.set_color(png::ColorType::Rgb);
enc.set_depth(png::BitDepth::Eight);
let mut w = enc.write_header().unwrap();
w.write_image_data(&[0u8; 12]).unwrap();
}
let err = load_heightmap_reader(out.as_slice(), "T", 0.3).unwrap_err();
assert!(matches!(err, HeightmapLoadError::UnsupportedFormat { .. }));
}
}