jpmschweitzerandClaude Opus 5.5
668772075c
The 30-file tree moves under atlas as its third rung (D-243), ten verbs fronting it. Each verb restates its module's options so `--help` describes something; tooling/test_planet_router.py hands every declared option to the module's own argparse and fails on drift, and now runs in make test-tooling. The 2026-09-02 half of this move had converted the top-level imports and the repo roots. Finishing it found what the half-move left: - Lazy in-function imports, and all of sol_data/, still named siblings bare. They resolved only through sys.path.insert hacks, so under reach the first globe render in generate, batch or sol-import would have raised ModuleNotFoundError. Qualified; the hacks are gone. - 247 print() calls and a stdout progress writer that fired once per 8 KB block. Report verbs (audit, quality) write through console.out, progress through console.event, and download progress is throttled to 10% steps so a job log is not tens of thousands of lines. - Every error exit raises ReachError with a fix. Two checks that could not fail: - batch --verify-determinism printed a warning and exited 0 on a mismatch. - import-provinces exited 0 with errors > 0. Both now raise. The 271-body bake is only safe to re-run because the first one holds. sol-import --body is action="append" in the module but the router took one value, so --body GJ0d --body GJ0e kept one. Now repeatable, and _flags repeats list options. test_conformance walked one level, so a nested group was reported as a verb missing @command and its ten verbs were never checked. It recurses now; proven by stripping @command from `planet quality` and watching it fail. Stray PNGs from the 2026-09-03 runaway router-test run are parked in .cache/t1288-stray-pngs/, not committed. Their reliefmaps differ from HEAD while the heightmap regenerated byte-identical — filed as T-1291. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
105 lines
3.3 KiB
Python
105 lines
3.3 KiB
Python
"""
|
|
Caching downloader for planetary science datasets.
|
|
|
|
Downloads are stored in sol_data/.cache/ and reused on subsequent runs.
|
|
Supports resume for large files and optional SHA-256 verification.
|
|
"""
|
|
|
|
import hashlib
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from tooling.core import console
|
|
|
|
CACHE_DIR = Path(__file__).resolve().parent / ".cache"
|
|
|
|
# urlretrieve calls the hook once per 8 KB block. The old `\r` rewrite made that
|
|
# free on a terminal; as stream events it would be tens of thousands of lines in
|
|
# a job log, so progress is reported per 10% step (or per 50 MB when the server
|
|
# sends no length) instead.
|
|
_STEP_PCT = 10
|
|
_STEP_MB = 50
|
|
_last_step = -1
|
|
|
|
|
|
def _progress_hook(block_num, block_size, total_size):
|
|
"""Report download progress, one event per step."""
|
|
global _last_step
|
|
if block_num == 0:
|
|
_last_step = -1
|
|
downloaded = block_num * block_size
|
|
mb = downloaded / (1024 * 1024)
|
|
if total_size > 0:
|
|
fraction = min(1.0, downloaded / total_size)
|
|
step = int(fraction * 100) // _STEP_PCT
|
|
if step != _last_step:
|
|
_last_step = step
|
|
total_mb = total_size / (1024 * 1024)
|
|
console.event(f"downloading: {mb:.1f}/{total_mb:.1f} MB", progress=fraction)
|
|
else:
|
|
step = int(mb) // _STEP_MB
|
|
if step != _last_step:
|
|
_last_step = step
|
|
console.event(f"downloading: {mb:.1f} MB")
|
|
|
|
|
|
def ensure_cached(url: str, filename: str, sha256: str = None) -> Path:
|
|
"""
|
|
Download a file if not already cached. Returns path to cached file.
|
|
|
|
Parameters
|
|
----------
|
|
url : download URL
|
|
filename : local filename within the cache directory
|
|
sha256 : optional hex digest for verification
|
|
"""
|
|
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
local_path = CACHE_DIR / filename
|
|
|
|
if local_path.exists():
|
|
if sha256:
|
|
actual = _sha256(local_path)
|
|
if actual != sha256:
|
|
console.event(f"checksum mismatch for {filename}, re-downloading", level="warn")
|
|
local_path.unlink()
|
|
else:
|
|
return local_path
|
|
else:
|
|
return local_path
|
|
|
|
console.event(f" fetching {filename} from {url[:80]}...")
|
|
tmp_path = local_path.with_suffix(".tmp")
|
|
|
|
try:
|
|
# Many government data servers (USGS, NOAA) require a User-Agent
|
|
opener = urllib.request.build_opener()
|
|
opener.addheaders = [
|
|
("User-Agent", "SettledReach-PlanetGen/1.0 (terrain pipeline)"),
|
|
]
|
|
urllib.request.install_opener(opener)
|
|
urllib.request.urlretrieve(url, str(tmp_path), reporthook=_progress_hook)
|
|
except Exception as e:
|
|
if tmp_path.exists():
|
|
tmp_path.unlink()
|
|
raise RuntimeError(f"Download failed for {filename}: {e}") from e
|
|
|
|
if sha256:
|
|
actual = _sha256(tmp_path)
|
|
if actual != sha256:
|
|
tmp_path.unlink()
|
|
raise RuntimeError(
|
|
f"Checksum mismatch for {filename}: "
|
|
f"expected {sha256[:16]}..., got {actual[:16]}..."
|
|
)
|
|
|
|
tmp_path.rename(local_path)
|
|
return local_path
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(8192), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|