//! T-1179 — wire-size table for step-canvas encodings (body-map-viewer //! workshop, measurement ④). //! //! Produces a REAL step-canvas-shaped [`DistrictWindowLayer`]-field dataset //! (elev_q, morphology, temp_dc, moisture_q, vegetation, glaciation — the //! exact six arrays that struct ships today, D-226 T-1124 amendment §4) at //! ~330K/2.07M/8.3M gridunits, via the SAME derivation call //! `build_district_window_layer`'s row-chunked `par_iter` uses //! (`derive_at_metres`, district spacing, no octave cutoff, no river-course //! packing — courses are a separate variable-length field orthogonal to this //! raster wire-size question). Real derived data (not synthetic noise or //! constant fills) so RLE/PNG compression ratios reflect genuine spatial //! coherence — see the workshop brief's measurement ④ scope note. //! //! Candidate encodings measured on the SAME canvas: //! (a) raw dense `u8`/`i16` arrays through `rmp_serde` (today's wire format) //! (b) bit-packed (sub-byte field widths, see `pack_bits` doc) //! (c) per-field run-length encoding //! (d) PNG-encoded raster per field (the `png` crate — already a main //! dependency, `server/Cargo.toml`; no new dependency added) //! (e) PNG applied to the bit-packed planes (cheap combination of b+d) //! //! Run: `cargo test --release --test wire_encoding_bench -- --ignored --nocapture` //! (debug numbers are not representative — this repo's benches are always run //! `--release`, matching `zoom_ladder_bench.rs`'s convention). //! //! Body/seed: GJ338Bd, `--seed yolo` (`seed_to_u64("yolo")`) — the same //! body+seed pair `aliveness_probe`'s doc example and the believability //! harness default to (`server/src/atlas/believability.rs`). use std::io::Cursor; use std::time::Instant; use serde::{Deserialize, Serialize}; use settled_reach_server::atlas::believability::seed_to_u64; use settled_reach_server::atlas::district_profile::{ derive_at_metres, BodyParams, ClimateConstants, }; use settled_reach_server::atlas::drainage; use settled_reach_server::atlas::features::TerrainAnalysis; use settled_reach_server::atlas::heightmap::load_heightmap_png; use settled_reach_server::atlas::layer_proxy::REGION_TEMP_NONE_DC; use settled_reach_server::atlas::scale; use settled_reach_server::seed::SeedChain; /// The six wire arrays `DistrictWindowLayer` ships today (layer_proxy.rs), /// derived at full canvas size rather than the 4,096-cell window cap. #[derive(Serialize, Deserialize)] struct WireCanvas { cols: u32, rows: u32, morphology: Vec, elev_q: Vec, temp_dc: Vec, moisture_q: Vec, vegetation: Vec, glaciation: Vec, } fn load_gj338bd() -> (BodyParams, TerrainAnalysis, SeedChain) { let src = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../wiki/star-systems/GJ-338B/bodies/GJ338Bd/heightmap.png"); let heightmap = load_heightmap_png(&src, "GJ338Bd", 0.3).expect("decode committed GJ338Bd heightmap"); let small = heightmap.downsample(512, 256); // GRID_W x GRID_H, production working grid let dr = drainage::analyze(&small.data, small.width, small.height, small.sea_level); let ta = TerrainAnalysis::analyze(&small, &dr); let params = BodyParams { hydrosphere: Some("ocean".into()), atmosphere: Some("breathable".into()), planet_class: Some("temperate".into()), body_radius_km: Some(6371.0), ..Default::default() }; let seed = SeedChain::for_body(seed_to_u64("yolo"), "GJ338Bd"); (params, ta, seed) } /// Derive a `cols x rows` canvas at district spacing (2,048 m/cell), origin /// at world (0,0), via the SAME `derive_at_metres` call + row-chunked /// `par_iter` shape `build_district_window_layer` uses internally /// (`layer_proxy.rs::derive_window_cell`/the row-scatter loop) — just at /// canvas sizes above the 4,096-cell `WIRE_CAP_CELLS` window ceiling, since /// that ceiling is a SERVED-window cap, not a derivation-cost cap (the /// workshop question is what a whole step canvas costs, pre-windowing). fn derive_canvas( seed: SeedChain, params: &BodyParams, ta: &TerrainAnalysis, cols: u32, rows: u32, ) -> (WireCanvas, std::time::Duration) { use rayon::prelude::*; let climate = ClimateConstants::default(); let step_m = scale::DISTRICT_M as f64; let cells = (cols as usize) * (rows as usize); let t0 = Instant::now(); let row_results: Vec> = (0..rows) .into_par_iter() .map(|row| { (0..cols) .map(|col| { let wx = col as f64 * step_m; let wy = row as f64 * step_m; let prof = derive_at_metres(seed, "GJ338Bd", params, ta, wx, wy, &climate, 0.0, &[]); let temp_dc = match prof.temperature_c { Some(t) => ((t * 10.0).round() as i32) .clamp(i16::MIN as i32 + 1, i16::MAX as i32) as i16, None => REGION_TEMP_NONE_DC, }; ( prof.morphology_zone as u8, prof.elev_q.clamp(0, 100) as u8, temp_dc, prof.moisture_q.clamp(0, 100) as u8, prof.vegetation_class as u8, prof.glaciation_grade as u8, ) }) .collect() }) .collect(); let elapsed = t0.elapsed(); let mut morphology = Vec::with_capacity(cells); let mut elev_q = Vec::with_capacity(cells); let mut temp_dc = Vec::with_capacity(cells); let mut moisture_q = Vec::with_capacity(cells); let mut vegetation = Vec::with_capacity(cells); let mut glaciation = Vec::with_capacity(cells); for row in row_results { for (m, e, t, mo, v, g) in row { morphology.push(m); elev_q.push(e); temp_dc.push(t); moisture_q.push(mo); vegetation.push(v); glaciation.push(g); } } ( WireCanvas { cols, rows, morphology, elev_q, temp_dc, moisture_q, vegetation, glaciation, }, elapsed, ) } // --------------------------------------------------------------------------- // Encoding (a): raw dense arrays via rmp_serde — today's wire format. // --------------------------------------------------------------------------- fn encode_rmp(canvas: &WireCanvas) -> (Vec, std::time::Duration, std::time::Duration) { let t0 = Instant::now(); let bytes = rmp_serde::to_vec(canvas).expect("rmp_serde encode"); let enc_time = t0.elapsed(); let t1 = Instant::now(); let decoded: WireCanvas = rmp_serde::from_slice(&bytes).expect("rmp_serde decode"); let dec_time = t1.elapsed(); std::hint::black_box(decoded.morphology.len()); (bytes, enc_time, dec_time) } // --------------------------------------------------------------------------- // Encoding (b): bit-packed planes. // // Field widths (minimal, from the real discriminant ranges): // morphology: 0-16 (17 zones, D-239 §6) -> 5 bits // elev_q: 0-100 -> 7 bits // moisture_q: 0-100 -> 7 bits // vegetation: 0-6 (7 classes incl. Marine) -> 3 bits // glaciation: 0-4 (5 grades) -> 3 bits // temp_dc: i16 incl. REGION_TEMP_NONE_DC sentinel -> left at 16 bits // (full dynamic range is genuinely used across class bands + // the sentinel; no safe narrower width without a second // encoding scheme for the sentinel case, out of scope here) // --------------------------------------------------------------------------- /// Pack `values` (each `< 2^width`) into a bitstream, LSB-first within each /// byte, fields concatenated in stream order — the simplest fixed-width /// packing (no entropy coding). Returns the packed byte buffer. fn pack_bits(values: &[u8], width: u32) -> Vec { let mut out = Vec::with_capacity((values.len() * width as usize).div_ceil(8)); let mut acc: u32 = 0; let mut acc_bits: u32 = 0; for &v in values { acc |= (v as u32) << acc_bits; acc_bits += width; while acc_bits >= 8 { out.push((acc & 0xFF) as u8); acc >>= 8; acc_bits -= 8; } } if acc_bits > 0 { out.push((acc & 0xFF) as u8); } out } fn unpack_bits(packed: &[u8], width: u32, count: usize) -> Vec { let mut out = Vec::with_capacity(count); let mut acc: u32 = 0; let mut acc_bits: u32 = 0; let mask = (1u32 << width) - 1; let mut byte_iter = packed.iter(); while out.len() < count { while acc_bits < width { let Some(&b) = byte_iter.next() else { break }; acc |= (b as u32) << acc_bits; acc_bits += 8; } out.push((acc & mask) as u8); acc >>= width; acc_bits -= width; } out } #[derive(Serialize, Deserialize)] struct BitPacked { cols: u32, rows: u32, morphology_bits: Vec, // 5 bits/cell elev_q_bits: Vec, // 7 bits/cell temp_dc: Vec, // unpacked, full 16 bits (see doc above) moisture_q_bits: Vec, // 7 bits/cell vegetation_bits: Vec, // 3 bits/cell glaciation_bits: Vec, // 3 bits/cell } fn encode_bitpacked(canvas: &WireCanvas) -> (Vec, std::time::Duration, std::time::Duration) { let n = canvas.morphology.len(); let t0 = Instant::now(); let packed = BitPacked { cols: canvas.cols, rows: canvas.rows, morphology_bits: pack_bits(&canvas.morphology, 5), elev_q_bits: pack_bits(&canvas.elev_q, 7), temp_dc: canvas.temp_dc.clone(), moisture_q_bits: pack_bits(&canvas.moisture_q, 7), vegetation_bits: pack_bits(&canvas.vegetation, 3), glaciation_bits: pack_bits(&canvas.glaciation, 3), }; let bytes = rmp_serde::to_vec(&packed).expect("rmp_serde encode bitpacked"); let enc_time = t0.elapsed(); let t1 = Instant::now(); let decoded: BitPacked = rmp_serde::from_slice(&bytes).expect("rmp_serde decode bitpacked"); let morphology = unpack_bits(&decoded.morphology_bits, 5, n); let elev_q = unpack_bits(&decoded.elev_q_bits, 7, n); let moisture_q = unpack_bits(&decoded.moisture_q_bits, 7, n); let vegetation = unpack_bits(&decoded.vegetation_bits, 3, n); let glaciation = unpack_bits(&decoded.glaciation_bits, 3, n); let dec_time = t1.elapsed(); std::hint::black_box(( morphology.len(), elev_q.len(), moisture_q.len(), vegetation.len(), glaciation.len(), )); (bytes, enc_time, dec_time) } // --------------------------------------------------------------------------- // Encoding (c): per-field run-length encoding. // --------------------------------------------------------------------------- /// (run_length, value) pairs, run_length capped at u16::MAX (wraps to a new /// run — no run ever exceeds 65,535 cells, larger than any canvas row here). fn rle_encode_u8(values: &[u8]) -> Vec<(u16, u8)> { let mut out = Vec::new(); let mut iter = values.iter(); let Some(&first) = iter.next() else { return out; }; let mut cur = first; let mut run: u16 = 1; for &v in iter { if v == cur && run < u16::MAX { run += 1; } else { out.push((run, cur)); cur = v; run = 1; } } out.push((run, cur)); out } fn rle_encode_i16(values: &[i16]) -> Vec<(u16, i16)> { let mut out = Vec::new(); let mut iter = values.iter(); let Some(&first) = iter.next() else { return out; }; let mut cur = first; let mut run: u16 = 1; for &v in iter { if v == cur && run < u16::MAX { run += 1; } else { out.push((run, cur)); cur = v; run = 1; } } out.push((run, cur)); out } #[derive(Serialize, Deserialize)] struct RleCanvas { cols: u32, rows: u32, morphology: Vec<(u16, u8)>, elev_q: Vec<(u16, u8)>, temp_dc: Vec<(u16, i16)>, moisture_q: Vec<(u16, u8)>, vegetation: Vec<(u16, u8)>, glaciation: Vec<(u16, u8)>, } struct RlePerFieldRuns { morphology: usize, elev_q: usize, temp_dc: usize, moisture_q: usize, vegetation: usize, glaciation: usize, } fn encode_rle( canvas: &WireCanvas, ) -> ( Vec, std::time::Duration, std::time::Duration, RlePerFieldRuns, ) { let t0 = Instant::now(); let morphology = rle_encode_u8(&canvas.morphology); let elev_q = rle_encode_u8(&canvas.elev_q); let temp_dc = rle_encode_i16(&canvas.temp_dc); let moisture_q = rle_encode_u8(&canvas.moisture_q); let vegetation = rle_encode_u8(&canvas.vegetation); let glaciation = rle_encode_u8(&canvas.glaciation); let runs = RlePerFieldRuns { morphology: morphology.len(), elev_q: elev_q.len(), temp_dc: temp_dc.len(), moisture_q: moisture_q.len(), vegetation: vegetation.len(), glaciation: glaciation.len(), }; let rle = RleCanvas { cols: canvas.cols, rows: canvas.rows, morphology, elev_q, temp_dc, moisture_q, vegetation, glaciation, }; let bytes = rmp_serde::to_vec(&rle).expect("rmp_serde encode rle"); let enc_time = t0.elapsed(); let t1 = Instant::now(); let decoded: RleCanvas = rmp_serde::from_slice(&bytes).expect("rmp_serde decode rle"); // Expand back to dense arrays (real decode cost — a consumer needs the // dense form to render). let mut morphology_dense = Vec::with_capacity(canvas.morphology.len()); for (run, v) in &decoded.morphology { morphology_dense.extend(std::iter::repeat_n(*v, *run as usize)); } let dec_time = t1.elapsed(); std::hint::black_box(morphology_dense.len()); (bytes, enc_time, dec_time, runs) } // --------------------------------------------------------------------------- // Encoding (d): PNG-encoded raster per field. // --------------------------------------------------------------------------- fn png_encode_u8_plane(cols: u32, rows: u32, data: &[u8]) -> Vec { let mut out = Vec::new(); { let mut enc = png::Encoder::new(&mut out, cols, rows); enc.set_color(png::ColorType::Grayscale); enc.set_depth(png::BitDepth::Eight); let mut writer = enc.write_header().expect("png header"); writer.write_image_data(data).expect("png data"); } out } fn png_decode_u8_plane(bytes: &[u8]) -> Vec { let mut decoder = png::Decoder::new(Cursor::new(bytes)) .read_info() .expect("png read_info"); let mut buf = vec![0u8; decoder.output_buffer_size()]; let frame = decoder.next_frame(&mut buf).expect("png next_frame"); buf[..frame.buffer_size()].to_vec() } /// PNG the five u8 planes; temp_dc (i16, includes negative + sentinel values, /// not representable as an 8-bit grayscale plane without a lossy remap) ships /// via rmp_serde alongside, same as the bit-packed encoding's treatment. fn encode_png(canvas: &WireCanvas) -> (Vec, std::time::Duration, std::time::Duration) { let t0 = Instant::now(); let morphology_png = png_encode_u8_plane(canvas.cols, canvas.rows, &canvas.morphology); let elev_q_png = png_encode_u8_plane(canvas.cols, canvas.rows, &canvas.elev_q); let moisture_q_png = png_encode_u8_plane(canvas.cols, canvas.rows, &canvas.moisture_q); let vegetation_png = png_encode_u8_plane(canvas.cols, canvas.rows, &canvas.vegetation); let glaciation_png = png_encode_u8_plane(canvas.cols, canvas.rows, &canvas.glaciation); let temp_dc_bytes = rmp_serde::to_vec(&canvas.temp_dc).expect("rmp_serde encode temp_dc"); let total = morphology_png.len() + elev_q_png.len() + moisture_q_png.len() + vegetation_png.len() + glaciation_png.len() + temp_dc_bytes.len(); let enc_time = t0.elapsed(); let t1 = Instant::now(); let morphology_d = png_decode_u8_plane(&morphology_png); let elev_q_d = png_decode_u8_plane(&elev_q_png); let moisture_q_d = png_decode_u8_plane(&moisture_q_png); let vegetation_d = png_decode_u8_plane(&vegetation_png); let glaciation_d = png_decode_u8_plane(&glaciation_png); let temp_dc_d: Vec = rmp_serde::from_slice(&temp_dc_bytes).expect("rmp_serde decode temp_dc"); let dec_time = t1.elapsed(); std::hint::black_box(( morphology_d.len(), elev_q_d.len(), moisture_q_d.len(), vegetation_d.len(), glaciation_d.len(), temp_dc_d.len(), )); // Return a synthetic combined buffer sized to `total` (not a real single // envelope — the workshop's wire contract question is exactly whether // these become five separate frames in a tagged envelope) so callers can // report a single byte count. Filled with zero bytes; only `.len()` is // used by the reporting harness below. (vec![0u8; total], enc_time, dec_time) } /// PNG applied to the bit-packed byte planes (b+d combined) — cheap to try /// since both encodings already exist above. fn encode_png_of_bitpacked( canvas: &WireCanvas, ) -> (Vec, std::time::Duration, std::time::Duration) { let t0 = Instant::now(); let morphology_bits = pack_bits(&canvas.morphology, 5); let elev_q_bits = pack_bits(&canvas.elev_q, 7); let moisture_q_bits = pack_bits(&canvas.moisture_q, 7); let vegetation_bits = pack_bits(&canvas.vegetation, 3); let glaciation_bits = pack_bits(&canvas.glaciation, 3); // PNG needs a rectangular raster; the packed byte streams aren't // canvas-shaped, so wrap each as a 1-row grayscale "image" of its own // byte length — this measures DEFLATE-over-packed-bytes cost/ratio // honestly (PNG's filter step is a no-op on a 1-row image, so this // isolates the DEFLATE contribution cleanly). let png_plane = |bits: &[u8]| -> Vec { let mut out = Vec::new(); let mut enc = png::Encoder::new(&mut out, bits.len() as u32, 1); enc.set_color(png::ColorType::Grayscale); enc.set_depth(png::BitDepth::Eight); let mut writer = enc.write_header().expect("png header"); writer.write_image_data(bits).expect("png data"); drop(writer); out }; let morphology_png = png_plane(&morphology_bits); let elev_q_png = png_plane(&elev_q_bits); let moisture_q_png = png_plane(&moisture_q_bits); let vegetation_png = png_plane(&vegetation_bits); let glaciation_png = png_plane(&glaciation_bits); let temp_dc_bytes = rmp_serde::to_vec(&canvas.temp_dc).expect("rmp_serde encode temp_dc"); let total = morphology_png.len() + elev_q_png.len() + moisture_q_png.len() + vegetation_png.len() + glaciation_png.len() + temp_dc_bytes.len(); let enc_time = t0.elapsed(); let t1 = Instant::now(); let n = canvas.morphology.len(); let morphology_d = unpack_bits(&png_decode_u8_plane(&morphology_png), 5, n); let elev_q_d = unpack_bits(&png_decode_u8_plane(&elev_q_png), 7, n); let moisture_q_d = unpack_bits(&png_decode_u8_plane(&moisture_q_png), 7, n); let vegetation_d = unpack_bits(&png_decode_u8_plane(&vegetation_png), 3, n); let glaciation_d = unpack_bits(&png_decode_u8_plane(&glaciation_png), 3, n); let temp_dc_d: Vec = rmp_serde::from_slice(&temp_dc_bytes).expect("rmp_serde decode temp_dc"); let dec_time = t1.elapsed(); std::hint::black_box(( morphology_d.len(), elev_q_d.len(), moisture_q_d.len(), vegetation_d.len(), glaciation_d.len(), temp_dc_d.len(), )); (vec![0u8; total], enc_time, dec_time) } // --------------------------------------------------------------------------- // Reporting // --------------------------------------------------------------------------- fn report_row(label: &str, bytes: usize, raw_bytes: usize, enc_ms: f64, dec_ms: f64) { let ratio = bytes as f64 / raw_bytes as f64; let wire_cap_multiple = bytes as f64 / 30_000.0; // ~30 KB context row println!( " {label:<28} {bytes:>10} bytes {ratio:>6.3}x raw {wire_cap_multiple:>8.1}x (30KB cap) enc {enc_ms:>7.2} ms dec {dec_ms:>7.2} ms" ); } #[test] #[ignore] fn wire_size_table_330k() { let (params, ta, seed) = load_gj338bd(); run_canvas_report(¶ms, &ta, seed, 768, 432, "330K (768x432)"); } #[test] #[ignore] fn wire_size_table_2_07m() { let (params, ta, seed) = load_gj338bd(); run_canvas_report(¶ms, &ta, seed, 1920, 1080, "2.07M (1920x1080)"); } #[test] #[ignore] fn wire_size_table_8_3m() { let (params, ta, seed) = load_gj338bd(); run_canvas_report(¶ms, &ta, seed, 3840, 2160, "8.3M (3840x2160)"); } fn run_canvas_report( params: &BodyParams, ta: &TerrainAnalysis, seed: SeedChain, cols: u32, rows: u32, label: &str, ) { println!( "\n=== T-1179 wire-size table: {label} = {} gridunits ===", cols as u64 * rows as u64 ); let (canvas, derive_time) = derive_canvas(seed, params, ta, cols, rows); println!( " derive: {:.2} ms ({} cells, {} Rayon threads available)\n", derive_time.as_secs_f64() * 1000.0, canvas.morphology.len(), std::thread::available_parallelism() .map(|n| n.get()) .unwrap_or(0) ); let (rmp_bytes, rmp_enc, rmp_dec) = encode_rmp(&canvas); let raw_bytes = rmp_bytes.len(); report_row( "(a) raw dense rmp_serde", raw_bytes, raw_bytes, rmp_enc.as_secs_f64() * 1000.0, rmp_dec.as_secs_f64() * 1000.0, ); let (bp_bytes, bp_enc, bp_dec) = encode_bitpacked(&canvas); report_row( "(b) bit-packed", bp_bytes.len(), raw_bytes, bp_enc.as_secs_f64() * 1000.0, bp_dec.as_secs_f64() * 1000.0, ); let (rle_bytes, rle_enc, rle_dec, runs) = encode_rle(&canvas); report_row( "(c) per-field RLE", rle_bytes.len(), raw_bytes, rle_enc.as_secs_f64() * 1000.0, rle_dec.as_secs_f64() * 1000.0, ); let (png_bytes, png_enc, png_dec) = encode_png(&canvas); report_row( "(d) PNG per field", png_bytes.len(), raw_bytes, png_enc.as_secs_f64() * 1000.0, png_dec.as_secs_f64() * 1000.0, ); let (pngbp_bytes, pngbp_enc, pngbp_dec) = encode_png_of_bitpacked(&canvas); report_row( "(e) PNG-of-bit-packed", pngbp_bytes.len(), raw_bytes, pngbp_enc.as_secs_f64() * 1000.0, pngbp_dec.as_secs_f64() * 1000.0, ); let n = canvas.morphology.len(); println!("\n per-field RLE run counts (lower = more compressible; n={n} cells):"); println!( " morphology: {:>8} runs ({:.1}% of dense)", runs.morphology, 100.0 * runs.morphology as f64 / n as f64 ); println!( " elev_q: {:>8} runs ({:.1}% of dense)", runs.elev_q, 100.0 * runs.elev_q as f64 / n as f64 ); println!( " temp_dc: {:>8} runs ({:.1}% of dense)", runs.temp_dc, 100.0 * runs.temp_dc as f64 / n as f64 ); println!( " moisture_q: {:>8} runs ({:.1}% of dense)", runs.moisture_q, 100.0 * runs.moisture_q as f64 / n as f64 ); println!( " vegetation: {:>8} runs ({:.1}% of dense)", runs.vegetation, 100.0 * runs.vegetation as f64 / n as f64 ); println!( " glaciation: {:>8} runs ({:.1}% of dense)", runs.glaciation, 100.0 * runs.glaciation as f64 / n as f64 ); println!(); }