""" Caching downloader for planetary science datasets. Downloads are stored in sol_data/.cache/ and reused on subsequent runs. Supports resume for large files and optional SHA-256 verification. """ import hashlib import sys import urllib.request from pathlib import Path CACHE_DIR = Path(__file__).resolve().parent / ".cache" def _progress_hook(block_num, block_size, total_size): """Print download progress.""" downloaded = block_num * block_size if total_size > 0: pct = min(100.0, downloaded * 100.0 / total_size) mb = downloaded / (1024 * 1024) total_mb = total_size / (1024 * 1024) sys.stdout.write(f"\r downloading: {mb:.1f}/{total_mb:.1f} MB ({pct:.0f}%)") else: mb = downloaded / (1024 * 1024) sys.stdout.write(f"\r downloading: {mb:.1f} MB") sys.stdout.flush() def ensure_cached(url: str, filename: str, sha256: str = None) -> Path: """ Download a file if not already cached. Returns path to cached file. Parameters ---------- url : download URL filename : local filename within the cache directory sha256 : optional hex digest for verification """ CACHE_DIR.mkdir(parents=True, exist_ok=True) local_path = CACHE_DIR / filename if local_path.exists(): if sha256: actual = _sha256(local_path) if actual != sha256: print(f" WARNING: checksum mismatch for {filename}, re-downloading") local_path.unlink() else: return local_path else: return local_path print(f" fetching {filename} from {url[:80]}...") tmp_path = local_path.with_suffix(".tmp") try: # Many government data servers (USGS, NOAA) require a User-Agent opener = urllib.request.build_opener() opener.addheaders = [ ("User-Agent", "SettledReach-PlanetGen/1.0 (terrain pipeline)"), ] urllib.request.install_opener(opener) urllib.request.urlretrieve(url, str(tmp_path), reporthook=_progress_hook) print() # newline after progress except Exception as e: if tmp_path.exists(): tmp_path.unlink() raise RuntimeError(f"Download failed for {filename}: {e}") from e if sha256: actual = _sha256(tmp_path) if actual != sha256: tmp_path.unlink() raise RuntimeError( f"Checksum mismatch for {filename}: " f"expected {sha256[:16]}..., got {actual[:16]}..." ) tmp_path.rename(local_path) return local_path def _sha256(path: Path) -> str: h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(8192), b""): h.update(chunk) return h.hexdigest()