jpmschweitzerandClaude Opus 4.6
d364e1907d
Ruff pre-push lint caught 17 unused imports across 7 files. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
90 lines
2.8 KiB
Python
90 lines
2.8 KiB
Python
"""
|
|
Caching downloader for planetary science datasets.
|
|
|
|
Downloads are stored in sol_data/.cache/ and reused on subsequent runs.
|
|
Supports resume for large files and optional SHA-256 verification.
|
|
"""
|
|
|
|
import hashlib
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
CACHE_DIR = Path(__file__).resolve().parent / ".cache"
|
|
|
|
|
|
def _progress_hook(block_num, block_size, total_size):
|
|
"""Print download progress."""
|
|
downloaded = block_num * block_size
|
|
if total_size > 0:
|
|
pct = min(100.0, downloaded * 100.0 / total_size)
|
|
mb = downloaded / (1024 * 1024)
|
|
total_mb = total_size / (1024 * 1024)
|
|
sys.stdout.write(f"\r downloading: {mb:.1f}/{total_mb:.1f} MB ({pct:.0f}%)")
|
|
else:
|
|
mb = downloaded / (1024 * 1024)
|
|
sys.stdout.write(f"\r downloading: {mb:.1f} MB")
|
|
sys.stdout.flush()
|
|
|
|
|
|
def ensure_cached(url: str, filename: str, sha256: str = None) -> Path:
|
|
"""
|
|
Download a file if not already cached. Returns path to cached file.
|
|
|
|
Parameters
|
|
----------
|
|
url : download URL
|
|
filename : local filename within the cache directory
|
|
sha256 : optional hex digest for verification
|
|
"""
|
|
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
local_path = CACHE_DIR / filename
|
|
|
|
if local_path.exists():
|
|
if sha256:
|
|
actual = _sha256(local_path)
|
|
if actual != sha256:
|
|
print(f" WARNING: checksum mismatch for {filename}, re-downloading")
|
|
local_path.unlink()
|
|
else:
|
|
return local_path
|
|
else:
|
|
return local_path
|
|
|
|
print(f" fetching {filename} from {url[:80]}...")
|
|
tmp_path = local_path.with_suffix(".tmp")
|
|
|
|
try:
|
|
# Many government data servers (USGS, NOAA) require a User-Agent
|
|
opener = urllib.request.build_opener()
|
|
opener.addheaders = [
|
|
("User-Agent", "SettledReach-PlanetGen/1.0 (terrain pipeline)"),
|
|
]
|
|
urllib.request.install_opener(opener)
|
|
urllib.request.urlretrieve(url, str(tmp_path), reporthook=_progress_hook)
|
|
print() # newline after progress
|
|
except Exception as e:
|
|
if tmp_path.exists():
|
|
tmp_path.unlink()
|
|
raise RuntimeError(f"Download failed for {filename}: {e}") from e
|
|
|
|
if sha256:
|
|
actual = _sha256(tmp_path)
|
|
if actual != sha256:
|
|
tmp_path.unlink()
|
|
raise RuntimeError(
|
|
f"Checksum mismatch for {filename}: "
|
|
f"expected {sha256[:16]}..., got {actual[:16]}..."
|
|
)
|
|
|
|
tmp_path.rename(local_path)
|
|
return local_path
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(8192), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|