diff --git a/server/Cargo.lock b/server/Cargo.lock index d596d0208..99522c5c4 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -190,7 +196,7 @@ dependencies = [ "bevy_reflect", "bevy_tasks", "bevy_utils", - "bitflags", + "bitflags 2.10.0", "bumpalo", "concurrent-queue", "derive_more", @@ -325,6 +331,12 @@ dependencies = [ "thread_local", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.10.0" @@ -462,6 +474,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -563,7 +584,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" dependencies = [ - "bitflags", + "bitflags 2.10.0", "block2", "libc", "objc2", @@ -654,6 +675,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -666,6 +696,16 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "foldhash" version = "0.2.0" @@ -886,13 +926,23 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "nix" version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225e7cfe711e0ba79a68baeddb2982723e4235247aefce1482f2f16c27865b66" dependencies = [ - "bitflags", + "bitflags 2.10.0", "cfg-if", "cfg_aliases", "libc", @@ -946,7 +996,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.10.0", ] [[package]] @@ -1035,6 +1085,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -1175,7 +1238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64", - "bitflags", + "bitflags 2.10.0", "serde", "serde_derive", ] @@ -1186,7 +1249,7 @@ version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ - "bitflags", + "bitflags 2.10.0", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -1305,6 +1368,7 @@ dependencies = [ "crossbeam-channel", "econ-sim", "pathfinding", + "png", "rand", "rand_chacha", "rayon", @@ -1336,6 +1400,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "slab" version = "0.4.12" @@ -1747,7 +1817,7 @@ version = "27.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afdcf84c395990db737f2dd91628706cb31e86d72e53482320d368e52b5da5eb" dependencies = [ - "bitflags", + "bitflags 2.10.0", "bytemuck", "js-sys", "log", diff --git a/server/Cargo.toml b/server/Cargo.toml index 68deee0c7..a20341bc3 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -29,6 +29,7 @@ sysinfo = "0.35" serde_json = "1" rusqlite = { version = "0.32", features = ["bundled"] } bytemuck = "1" +png = "0.17" toml = "0.8" aho-corasick = "1" # Economics simulation — Leontief + tâtonnement + D-180 event port (#821) diff --git a/server/src/atlas/heightmap.rs b/server/src/atlas/heightmap.rs index f64a76260..57aedfcc0 100644 --- a/server/src/atlas/heightmap.rs +++ b/server/src/atlas/heightmap.rs @@ -1,19 +1,26 @@ -//! Heightmap BLOB loader — reads float32 LE elevation grids from systems.db. +//! Heightmap loader — reads the canonical 16-bit grayscale elevation PNG (D-202). //! -//! Implements the Rust side of D-202. The Python pipeline stores each body's -//! elevation grid as a contiguous float32 little-endian BLOB in -//! `atlas_body_heightmaps.data`. This module loads that BLOB via `rusqlite` -//! and reinterprets the bytes into a `Vec` using `bytemuck`. +//! Per D-202 (amended #963), each body's canonical elevation is a per-body +//! **16-bit grayscale `heightmap.png`** stored next to the color `reliefmap.png` +//! in the body's wiki directory (path from `bodies.terrain_reference`). The +//! grayscale luminance is the normalized elevation; this is the single source +//! of truth — no `systems.db` BLOB. The file is high-resolution (canonically +//! 2048×1024) so the lower cascade layers (region/block/tile) have real local +//! detail; **Layer 1** (continental drainage/basins/mountain-ranges) calls +//! [`BodyHeightmap::downsample`] to `GRID_W × GRID_H` first, decoupling the +//! continental compute cost from the stored resolution. //! -//! Values are normalized elevation in [0.0, 1.0]. `sea_level` is the fraction -//! below which terrain is underwater (0.0 = no ocean). -//! -//! Canonical grid size: 512 × 256 (GRID_W × GRID_H), row-major. +//! Elevation values are normalized to [0.0, 1.0]. `sea_level` is body metadata +//! (carried alongside, not in the PNG) below which terrain is ocean. + +use std::fs::File; +use std::io::Read; +use std::path::Path; -use rusqlite::{params, Connection}; use thiserror::Error; -/// Canonical grid dimensions matching the Python pipeline (atlas_common.py). +/// Layer-1 working resolution: continental drainage/feature extraction run at +/// this size (the stored heightmap is higher-res and downsampled to it). pub const GRID_W: u32 = 512; pub const GRID_H: u32 = 256; @@ -45,155 +52,182 @@ impl BodyHeightmap { pub fn is_land(&self, row: u32, col: u32) -> bool { self.get(row, col).is_some_and(|e| e >= self.sea_level) } + + /// Box-average downsample to `target_w × target_h` (used by Layer 1 to drop + /// the high-res stored heightmap to the continental working resolution). + /// Deterministic (f64 accumulation, fixed footprint). Returns a clone when + /// the target is not strictly smaller (no upsampling here). + pub fn downsample(&self, target_w: u32, target_h: u32) -> BodyHeightmap { + if target_w == 0 || target_h == 0 || (target_w >= self.width && target_h >= self.height) { + return self.clone(); + } + let (sw, sh) = (self.width as u64, self.height as u64); + let mut out = vec![0f32; (target_w * target_h) as usize]; + for ty in 0..target_h { + let sy0 = (ty as u64 * sh / target_h as u64) as u32; + let sy1 = (((ty as u64 + 1) * sh / target_h as u64).max(sy0 as u64 + 1)) as u32; + for tx in 0..target_w { + let sx0 = (tx as u64 * sw / target_w as u64) as u32; + let sx1 = (((tx as u64 + 1) * sw / target_w as u64).max(sx0 as u64 + 1)) as u32; + let mut sum = 0f64; + let mut cnt = 0u32; + for sy in sy0..sy1 { + for sx in sx0..sx1 { + sum += self.data[(sy * self.width + sx) as usize] as f64; + cnt += 1; + } + } + out[(ty * target_w + tx) as usize] = (sum / cnt as f64) as f32; + } + } + BodyHeightmap { + body_id: self.body_id.clone(), + width: target_w, + height: target_h, + data: out, + sea_level: self.sea_level, + } + } } #[derive(Debug, Error)] pub enum HeightmapLoadError { - #[error("no heightmap row for body '{0}'")] - NotFound(String), - #[error("BLOB size {actual} does not match declared grid {w}×{h}×4 = {expected}")] - BlobSizeMismatch { - actual: usize, - w: u32, - h: u32, - expected: usize, + #[error("heightmap file unreadable: {0}")] + Io(#[from] std::io::Error), + #[error("PNG decode error: {0}")] + Png(#[from] png::DecodingError), + #[error("unsupported heightmap PNG: {bit_depth} {color_type} (expected 16-bit grayscale)")] + UnsupportedFormat { + bit_depth: String, + color_type: String, }, - #[error("SQLite error: {0}")] - Sql(#[from] rusqlite::Error), } -/// Load the heightmap for `body_id` from the open `conn`. +/// Load the canonical heightmap PNG for `body_id` from `path`. /// -/// The BLOB is reinterpreted in-place via `bytemuck::cast_slice` — no copy -/// beyond the initial `Vec` read from SQLite. On little-endian hosts -/// (all current targets) this is a zero-cost reinterpret. On big-endian hosts -/// the bytes are already stored LE, so each f32 would be byte-swapped; this -/// function does not perform that swap — big-endian support is deferred. -pub fn load_heightmap( - conn: &Connection, +/// `sea_level` is body metadata (e.g. from the `bodies` table / body def), not +/// stored in the PNG. +pub fn load_heightmap_png( + path: &Path, body_id: &str, + sea_level: f32, ) -> Result { - let result = conn.query_row( - "SELECT width, height, data, sea_level \ - FROM atlas_body_heightmaps WHERE body_id = ?1", - params![body_id], - |row| { - let width: u32 = row.get(0)?; - let height: u32 = row.get(1)?; - let blob: Vec = row.get(2)?; - let sea_level: f64 = row.get(3)?; - Ok((width, height, blob, sea_level as f32)) - }, - ); + let file = File::open(path)?; + load_heightmap_reader(file, body_id, sea_level) +} - match result { - Err(rusqlite::Error::QueryReturnedNoRows) => { - Err(HeightmapLoadError::NotFound(body_id.to_string())) +/// Decode a heightmap PNG from any reader. 16-bit grayscale is the canonical +/// format; 8-bit grayscale is accepted (coarse — viewable/test only). +pub fn load_heightmap_reader( + reader: R, + body_id: &str, + sea_level: f32, +) -> Result { + let mut png_reader = png::Decoder::new(reader).read_info()?; + let (width, height, bit_depth, color_type) = { + let info = png_reader.info(); + (info.width, info.height, info.bit_depth, info.color_type) + }; + let mut buf = vec![0u8; png_reader.output_buffer_size()]; + let frame = png_reader.next_frame(&mut buf)?; + let bytes = &buf[..frame.buffer_size()]; + + let data: Vec = match (bit_depth, color_type) { + (png::BitDepth::Sixteen, png::ColorType::Grayscale) => bytes + .chunks_exact(2) + .map(|p| u16::from_be_bytes([p[0], p[1]]) as f32 / 65535.0) + .collect(), + (png::BitDepth::Eight, png::ColorType::Grayscale) => { + bytes.iter().map(|&b| b as f32 / 255.0).collect() } - Err(e) => Err(HeightmapLoadError::Sql(e)), - Ok((width, height, blob, sea_level)) => { - let expected = (width * height * 4) as usize; - if blob.len() != expected { - return Err(HeightmapLoadError::BlobSizeMismatch { - actual: blob.len(), - w: width, - h: height, - expected, - }); - } - // Reinterpret the LE bytes as f32 values. bytemuck::cast_slice - // is safe here: we verified the length is a multiple of 4, and - // f32 has no invalid bit patterns. - let floats: &[f32] = bytemuck::cast_slice(&blob); - let data = floats.to_vec(); - Ok(BodyHeightmap { - body_id: body_id.to_string(), - width, - height, - data, - sea_level, + (bd, ct) => { + return Err(HeightmapLoadError::UnsupportedFormat { + bit_depth: format!("{bd:?}"), + color_type: format!("{ct:?}"), }) } - } + }; + + Ok(BodyHeightmap { + body_id: body_id.to_string(), + width, + height, + data, + sea_level, + }) } #[cfg(test)] mod tests { use super::*; - use rusqlite::Connection; - fn make_test_db() -> Connection { - let conn = Connection::open_in_memory().unwrap(); - conn.execute_batch( - "CREATE TABLE atlas_body_heightmaps ( - body_id TEXT PRIMARY KEY, - width INTEGER NOT NULL, - height INTEGER NOT NULL, - data BLOB NOT NULL, - sea_level REAL NOT NULL DEFAULT 0.0, - imported_at TEXT NOT NULL DEFAULT (datetime('now')) - );", - ) - .unwrap(); - conn - } - - fn insert_heightmap(conn: &Connection, body_id: &str, w: u32, h: u32, sea_level: f32) { - let floats: Vec = (0..(w * h)).map(|i| i as f32 / (w * h) as f32).collect(); - let bytes: &[u8] = bytemuck::cast_slice(&floats); - conn.execute( - "INSERT INTO atlas_body_heightmaps (body_id, width, height, data, sea_level) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![body_id, w, h, bytes, sea_level], - ) - .unwrap(); + /// Encode a 16-bit grayscale PNG (row-major u16 elevation) to bytes. + fn encode_gray16(w: u32, h: u32, vals: &[u16]) -> Vec { + let mut out = Vec::new(); + { + let mut enc = png::Encoder::new(&mut out, w, h); + enc.set_color(png::ColorType::Grayscale); + enc.set_depth(png::BitDepth::Sixteen); + let mut writer = enc.write_header().unwrap(); + // png expects big-endian 16-bit samples. + let mut be = Vec::with_capacity(vals.len() * 2); + for &v in vals { + be.extend_from_slice(&v.to_be_bytes()); + } + writer.write_image_data(&be).unwrap(); + } + out } #[test] - fn round_trip_canonical_size() { - let conn = make_test_db(); - insert_heightmap(&conn, "TestBody", GRID_W, GRID_H, 0.3); - let hm = load_heightmap(&conn, "TestBody").unwrap(); - assert_eq!(hm.width, GRID_W); - assert_eq!(hm.height, GRID_H); - assert_eq!(hm.data.len(), (GRID_W * GRID_H) as usize); - assert!((hm.sea_level - 0.3).abs() < 1e-6); - // First cell is 0.0, last approaches 1.0 - assert_eq!(hm.data[0], 0.0); - assert!(hm.data.last().copied().unwrap() < 1.0); + fn round_trips_16bit_grayscale() { + let (w, h) = (4u32, 2u32); + // Elevations 0, 1/7, ... 7/7 across 8 cells. + let vals: Vec = (0..(w * h)) + .map(|i| (i as f32 / (w * h - 1) as f32 * 65535.0) as u16) + .collect(); + let png_bytes = encode_gray16(w, h, &vals); + let hm = load_heightmap_reader(png_bytes.as_slice(), "T", 0.3).unwrap(); + assert_eq!((hm.width, hm.height), (w, h)); + assert_eq!(hm.data.len(), 8); + assert!((hm.data[0] - 0.0).abs() < 1e-6); + assert!((hm.data[7] - 1.0).abs() < 1e-4); + assert!(hm.is_land(1, 3)); // last cell = 1.0 > sea 0.3 + assert!(!hm.is_land(0, 0)); // first cell = 0.0 < sea } #[test] - fn get_and_is_land() { - let conn = make_test_db(); - insert_heightmap(&conn, "LandBody", 4, 2, 0.5); - let hm = load_heightmap(&conn, "LandBody").unwrap(); - // First cell (index 0) = 0.0 / 8 = 0.0 — below sea level - assert!(!hm.is_land(0, 0)); - // Last cell (index 7) = 7.0 / 8 = 0.875 — above sea level - assert!(hm.is_land(1, 3)); - // Out-of-bounds returns false - assert!(!hm.is_land(99, 99)); + fn downsample_halves_and_averages() { + // 4×2 → 2×1: each output cell averages a 2×2 block. + let vals: Vec = vec![0, 0, 65535, 65535, 0, 0, 65535, 65535]; + let hm = load_heightmap_reader(encode_gray16(4, 2, &vals).as_slice(), "T", 0.3).unwrap(); + let ds = hm.downsample(2, 1); + assert_eq!((ds.width, ds.height), (2, 1)); + assert!((ds.data[0] - 0.0).abs() < 1e-4); // left 2×2 block all 0 + assert!((ds.data[1] - 1.0).abs() < 1e-4); // right 2×2 block all 1 } #[test] - fn not_found_error() { - let conn = make_test_db(); - let err = load_heightmap(&conn, "Ghost").unwrap_err(); - assert!(matches!(err, HeightmapLoadError::NotFound(_))); + fn downsample_is_deterministic() { + let vals: Vec = (0..(16 * 8)).map(|i| (i * 257) as u16).collect(); + let hm = load_heightmap_reader(encode_gray16(16, 8, &vals).as_slice(), "T", 0.3).unwrap(); + let a = hm.downsample(4, 2); + let b = hm.downsample(4, 2); + assert_eq!(a.data, b.data); } #[test] - fn blob_size_mismatch_error() { - let conn = make_test_db(); - // Insert a truncated BLOB - conn.execute( - "INSERT INTO atlas_body_heightmaps (body_id, width, height, data, sea_level) - VALUES ('BadBlob', 4, 4, X'DEADBEEF', 0.0)", - [], - ) - .unwrap(); - let err = load_heightmap(&conn, "BadBlob").unwrap_err(); - assert!(matches!(err, HeightmapLoadError::BlobSizeMismatch { .. })); + fn rejects_rgb() { + // An RGB PNG (like the reliefmap) must be rejected, not misread. + let mut out = Vec::new(); + { + let mut enc = png::Encoder::new(&mut out, 2, 2); + enc.set_color(png::ColorType::Rgb); + enc.set_depth(png::BitDepth::Eight); + let mut w = enc.write_header().unwrap(); + w.write_image_data(&[0u8; 12]).unwrap(); + } + let err = load_heightmap_reader(out.as_slice(), "T", 0.3).unwrap_err(); + assert!(matches!(err, HeightmapLoadError::UnsupportedFormat { .. })); } }